Light, fluffy, and always free - The AWS Local Emulator alternative
feat(lambda): durable function callbacks (#5100)
TATSUNO “Taz” Yasuhiroa1ed65c51dfa
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.javaindex d43d0bb34f..5be56bd05a 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java@@ -22,6 +22,7 @@ class LambdaDurableFunctionsTest {private static final String ROLE = "arn:aws:iam::000000000000:role/lambda-role";private static final String FN = TestFixtures.uniqueName("fn-durable-exec");+ private static final String CALLBACK_FN = TestFixtures.uniqueName("fn-durable-callback");private static LambdaClient lambda;@@ -33,10 +34,12 @@ class LambdaDurableFunctionsTest {@AfterAllstatic void cleanup() {if (lambda != null) {- try {- lambda.deleteFunction(DeleteFunctionRequest.builder().functionName(FN).build());- } catch (Exception ignored) {- // Cleanup only. A failed delete must not hide the test result.+ for (String name : new String[] {FN, CALLBACK_FN}) {+ try {+ lambda.deleteFunction(DeleteFunctionRequest.builder().functionName(name).build());+ } catch (Exception ignored) {+ // Cleanup only. A failed delete must not hide the test result.+ }}lambda.close();}@@ -117,4 +120,63 @@ class LambdaDurableFunctionsTest {.isInstanceOf(InvalidParameterValueException.class).hasMessageContaining("You cannot invoke a durable function using an unqualified ARN.");}++ @Test+ @DisplayName("a durable callback is completed through the callback APIs and resumes the execution")+ void durableCallbackResumesTheExecution() throws InterruptedException {+ Assumptions.assumeTrue(TestFixtures.isLambdaDispatchAvailable(),+ "skipping: Lambda dispatch (Docker) not available in this environment");++ lambda.createFunction(CreateFunctionRequest.builder()+ .functionName(CALLBACK_FN)+ .runtime(Runtime.PYTHON3_14)+ .role(ROLE)+ .handler("lambda_function.handler")+ .timeout(30)+ .durableConfig(DurableConfig.builder().executionTimeout(120).retentionPeriodInDays(1).build())+ .code(FunctionCode.builder().zipFile(SdkBytes.fromByteArray(LambdaUtils.durablePythonZip())).build())+ .build());++ String arn = lambda.invoke(InvokeRequest.builder()+ .functionName(CALLBACK_FN + ":$LATEST")+ .invocationType(InvocationType.EVENT)+ .payload(SdkBytes.fromUtf8String("{\"callback\": true}"))+ .build()).durableExecutionArn();++ String callbackId = null;+ for (int i = 0; i < 60 && callbackId == null; i++) {+ callbackId = lambda.getDurableExecutionHistory(GetDurableExecutionHistoryRequest.builder()+ .durableExecutionArn(arn).build())+ .events().stream()+ .filter(event -> event.eventType() == EventType.CALLBACK_STARTED)+ .map(event -> event.callbackStartedDetails().callbackId())+ .findFirst().orElse(null);+ if (callbackId == null) {+ Thread.sleep(500);+ }+ }+ assertThat(callbackId).as("the function started a callback").isNotNull();+ String id = callbackId;++ lambda.sendDurableExecutionCallbackHeartbeat(SendDurableExecutionCallbackHeartbeatRequest.builder()+ .callbackId(id).build());+ lambda.sendDurableExecutionCallbackSuccess(SendDurableExecutionCallbackSuccessRequest.builder()+ .callbackId(id).result(SdkBytes.fromUtf8String("{\"approved\": true}")).build());++ GetDurableExecutionResponse execution = null;+ for (int i = 0; i < 60; i++) {+ execution = lambda.getDurableExecution(GetDurableExecutionRequest.builder()+ .durableExecutionArn(arn).includeExecutionData(true).build());+ if (execution.status() != ExecutionStatus.RUNNING) {+ break;+ }+ Thread.sleep(500);+ }+ assertThat(execution.status()).isEqualTo(ExecutionStatus.SUCCEEDED);+ assertThat(execution.result()).isEqualTo("{\"approved\": true}");++ assertThatThrownBy(() -> lambda.sendDurableExecutionCallbackSuccess(+ SendDurableExecutionCallbackSuccessRequest.builder().callbackId(id).build()))+ .isInstanceOf(CallbackTimeoutException.class);+ }}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.javaindex e93f56cc88..67d3545d50 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java@@ -284,6 +284,19 @@ public final class LambdaUtils {if request.get("fail"):return {"Status": "FAILED","Error": {"ErrorMessage": "asked to fail", "ErrorType": "TestFailure"}}+ if request.get("callback"):+ if "callback" not in operations:+ client.checkpoint_durable_execution(+ DurableExecutionArn=event["DurableExecutionArn"],+ CheckpointToken=event["CheckpointToken"],+ Updates=[{"Id": "callback", "Name": "approval", "Type": "CALLBACK",+ "SubType": "Callback", "Action": "START",+ "CallbackOptions": {"HeartbeatTimeoutSeconds": 60}}])+ return {"Status": "PENDING"}+ callback = operations["callback"]+ if callback["Status"] == "STARTED":+ return {"Status": "PENDING"}+ return {"Status": "SUCCEEDED", "Result": callback["CallbackDetails"].get("Result", "")}if "step" not in operations:client.checkpoint_durable_execution(DurableExecutionArn=event["DurableExecutionArn"],diff --git a/docs/services/index.md b/docs/services/index.mdindex 91561711b0..668f03b02f 100644--- a/docs/services/index.md+++ b/docs/services/index.md@@ -18,7 +18,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac| [S3 Tables](s3tables.md) | `/buckets`, `/namespaces/*`, `/tables/*` | REST JSON | 25 || [DynamoDB](dynamodb.md) | `POST /` + `X-Amz-Target: DynamoDB_20120810.*` | JSON 1.1 | 28 || [DynamoDB Streams](dynamodb.md#streams) | `POST /` + `X-Amz-Target: DynamoDBStreams_20120810.*` | JSON 1.1 | 4 |-| [Lambda](lambda.md) | `/2015-03-31/functions/...` + `/2025-12-01/durable-executions/...` | REST JSON | 52 |+| [Lambda](lambda.md) | `/2015-03-31/functions/...` + `/2025-12-01/durable-executions/...` | REST JSON | 55 || [Lambda MicroVMs](lambda-microvms.md) | `/2025-09-09/...` + `/2026-04-04/...` | REST JSON | 22 || [API Gateway v1](api-gateway.md) | `/restapis/...` | REST JSON | 79 || [API Gateway v2](api-gateway.md#v2) | `/v2/apis/...` | REST JSON | 53 + data-plane |diff --git a/docs/services/lambda.md b/docs/services/lambda.mdindex b0dbcb9c8b..73fefbc259 100644--- a/docs/services/lambda.md+++ b/docs/services/lambda.md@@ -61,6 +61,9 @@ Floci Lambda runs your function code locally inside real Docker containers - clo| `GetDurableExecutionHistory` | Event log of a durable execution, oldest or newest first || `ListDurableExecutionsByFunction` | Durable executions of a function or version, filtered by status, name and start time || `StopDurableExecution` | Stop a running durable execution with an error |+| `SendDurableExecutionCallbackSuccess` | Complete a durable callback with a result, up to 1 MB |+| `SendDurableExecutionCallbackFailure` | Fail a durable callback with an error |+| `SendDurableExecutionCallbackHeartbeat` | Keep a durable callback with a heartbeat timeout alive |`UpdateFunctionConfiguration` validates `MemorySize` and `Timeout` as whole numbers in theirsupported ranges before changing any stored settings. A rejected value returns@@ -165,24 +168,33 @@ background. Invocations through an ARN (EventBridge, Scheduler, function URLs, ddurable executions the same way.The checkpoint protocol under `/2025-12-01` is the real one, so the Durable Execution SDK works-unchanged for steps, waits, step retries and replay. Timers are persisted and fired by a+unchanged for steps, waits, step retries, callbacks and replay. Timers are persisted and fired by abackground sweep, so an execution survives a restart of Floci. A function that returns somethingother than the `{Status, Result, Error}` envelope fails its execution with`Invalid Status in invocation output.` A function that throws is re-invoked four more times, after1, 2, 4 and 8 seconds, and then fails the execution with its error. AWS does the same over a fewminutes.+A callback (`createCallback`, `waitForCallback`) gets a callback id that the+`SendDurableExecutionCallback*` APIs complete. Its `TimeoutSeconds` and `HeartbeatTimeoutSeconds`+fail it with `Callback.Timeout` or `Callback.Heartbeat`, and each heartbeat restarts the heartbeat+timeout. A failure sent with an empty request body fails the callback but does not invoke the+function again, as on AWS, so the execution keeps waiting. The SDKs send `{}` when `Error` is+omitted, which does invoke it.+`GetDurableExecutionHistory` leaves payloads out unless `IncludeExecutionData=true` is sent. TheAPI reference names `true` as the default, but AWS answers this way.+`SendDurableExecutionCallbackSuccess` takes a result of up to 1 MB. The API reference names 256 KB,+but AWS accepts 1 MB and rejects one byte more.+The public `public.ecr.aws/lambda` images do not bundle the SDK that the managed runtimes carry.Add `aws-durable-execution-sdk-python` or `@aws/durable-execution-sdk-js` to your deploymentpackage.Not emulated yet:-- Callbacks (`WaitForCallback` and the `SendDurableExecutionCallback*` APIs) and chained invokes- (`context.invoke`). Both are rejected at checkpoint time.+- Chained invokes (`context.invoke`). They are rejected at checkpoint time.- Durable executions started by Step Functions or by an event source mapping. These invoke thefunction as a plain one.- Payload encryption. `KMSKeyArn` is stored and returned only.@@ -387,8 +399,7 @@ These AWS Lambda operations have no handler in Floci. Calls will return `404` or- Layer permissions (`AddLayerVersionPermission`, `RemoveLayerVersionPermission`, `GetLayerVersionPolicy`)- Provisioned concurrency (`PutProvisionedConcurrencyConfig`, `GetProvisionedConcurrencyConfig`, `ListProvisionedConcurrencyConfigs`, `DeleteProvisionedConcurrencyConfig`)- `InvokeWithResponseStream`-- Durable function callbacks and chained invokes (`SendDurableExecutionCallbackSuccess`,- `SendDurableExecutionCallbackFailure`, `SendDurableExecutionCallbackHeartbeat`). See Durable Functions+- Durable function chained invokes. See Durable Functions- Code signing enforcement. A configuration is created, read, updated, deleted and listed, andnothing verifies a signature against it, so it never gates a deployment. Attaching one to afunction is not wired either: there is no `PutFunctionCodeSigningConfig`, sodiff --git a/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java b/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.javaindex bfbc8c6321..d84f082f2c 100644--- a/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java+++ b/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java@@ -68,6 +68,11 @@ public class IamActionRegistry {rule("lambda", "GET", ".*/durable-executions/.+/history$", "lambda:GetDurableExecutionHistory"),rule("lambda", "POST", ".*/durable-executions/.+/stop$", "lambda:StopDurableExecution"),rule("lambda", "GET", ".*/durable-executions/.+$", "lambda:GetDurableExecution"),+ rule("lambda", "POST", ".*/durable-execution-callbacks/.+/succeed$",+ "lambda:SendDurableExecutionCallbackSuccess"),+ rule("lambda", "POST", ".*/durable-execution-callbacks/.+/fail$", "lambda:SendDurableExecutionCallbackFailure"),+ rule("lambda", "POST", ".*/durable-execution-callbacks/.+/heartbeat$",+ "lambda:SendDurableExecutionCallbackHeartbeat"),// ── DynamoDB (JSON 1.1, action from X-Amz-Target handled separately) ──// Handled via Query-style action extraction in the filterdiff --git a/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java b/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.javaindex b18b26d224..9514801287 100644--- a/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java+++ b/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.core.common.AwsQueryServiceResolver;import io.github.hectorvent.floci.core.common.AwsRegions;import io.github.hectorvent.floci.services.iam.model.ServerCertificate;import io.github.hectorvent.floci.services.lambda.LambdaArnUtils;+import io.github.hectorvent.floci.services.lambda.durable.DurableTokens;import jakarta.enterprise.context.ApplicationScoped;import jakarta.enterprise.inject.Instance;import jakarta.inject.Inject;@@ -224,6 +225,9 @@ public class ResourceArnBuilder {// ── Lambda ──────────────────────────────────────────────────────────────────private String buildLambdaArn(String path, String region, String accountId) {String executionArn = durableExecutionArn(path);+ if (executionArn == null) {+ executionArn = durableCallbackExecutionArn(path);+ }if (executionArn != null) {return executionArn;}@@ -247,6 +251,19 @@ public class ResourceArnBuilder {return matcher.lookingAt() ? matcher.group() : null;}+ /** A callback id carries its execution ARN, so the callback actions are authorized against that execution. */+ private static String durableCallbackExecutionArn(String path) {+ String prefix = "/durable-execution-callbacks/";+ int callbacks = path.indexOf(prefix);+ int action = path.lastIndexOf('/');+ if (callbacks < 0 || action <= callbacks + prefix.length()) {+ return null;+ }+ return DurableTokens.callbackExecutionArn(path.substring(callbacks + prefix.length(), action))+ .filter(arn -> LambdaArnUtils.DURABLE_EXECUTION_ARN.matcher(arn).matches())+ .orElse(null);+ }+// ── SQS ─────────────────────────────────────────────────────────────────────private String buildSqsArn(ContainerRequestContext ctx, String region, String accountId) {String queueUrl = ctx.getUriInfo().getQueryParameters().getFirst("QueueUrl");diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.javaindex 315754cfa7..0051ccb852 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java@@ -48,7 +48,7 @@ final class DurableCheckpointApplier {return new Outcome(draft.closingStatus != null);}- /** Completes waits and step retries whose time has come. The sweeper and every checkpoint call it. */+ /** Completes waits, step retries and callback timeouts whose time has come. The sweeper and every checkpoint call it. */static boolean fireDueTimers(DurableExecution execution, long now) {boolean changed = false;for (DurableOperation operation : execution.getOperations().values()) {@@ -71,11 +71,38 @@ final class DurableCheckpointApplier {operation.setNextAttemptTimestamp(null);operation.setChangeSequence(execution.nextChangeSequence());changed = true;+ } else if (operation.getType() == DurableOperationType.CALLBACK+ && operation.getStatus() == DurableOperationStatus.STARTED) {+ changed |= timeOutCallback(execution, operation, now);}}return changed;}+ private static boolean timeOutCallback(DurableExecution execution, DurableOperation operation, long now) {+ boolean timedOut = operation.getCallbackDeadline() != null && operation.getCallbackDeadline() <= now;+ boolean heartbeatMissed = operation.getHeartbeatDeadline() != null && operation.getHeartbeatDeadline() <= now;+ if (!timedOut && !heartbeatMissed) {+ return false;+ }+ boolean heartbeatFirst = heartbeatMissed+ && (!timedOut || operation.getHeartbeatDeadline() < operation.getCallbackDeadline());+ DurableErrorObject error = heartbeatFirst+ ? DurableErrorObject.of("Callback timed out on heartbeat", "Callback.Heartbeat")+ : DurableErrorObject.of("Callback timed out", "Callback.Timeout");+ operation.setStatus(DurableOperationStatus.TIMED_OUT);+ operation.setError(error);+ operation.setEndTimestamp(now);+ operation.setCallbackDeadline(null);+ operation.setHeartbeatDeadline(null);+ operation.setChangeSequence(execution.nextChangeSequence());+ // The history event names only the error type.+ Map<String, Object> details = new LinkedHashMap<>();+ details.put("Error", DurableHistory.errorWrapper(DurableErrorObject.of(null, error.getErrorType())));+ DurableHistory.operationEvent(execution, operation, "CallbackTimedOut", now, details);+ return true;+ }+private static void validateBatch(List<DurableOperationUpdate> updates) {long executionUpdates = updates.stream().filter(update -> update.type() == DurableOperationType.EXECUTION).count();if (executionUpdates > 1) {@@ -93,7 +120,7 @@ final class DurableCheckpointApplier {DurableOperationUpdate previous = seen.put(update.id(), update);if (previous != null && !(previous.action() == DurableOperationAction.START&& previous.type() == update.type() && closesInSameBatch(update))) {- throw invalid("Cannot checkpoint multiple operations with the same ID.");+ throw invalid("Cannot update the same operation twice in a single request.");}}}@@ -170,7 +197,8 @@ final class DurableCheckpointApplier {case CONTEXT -> applyContext(update, existing);case STEP -> applyStep(update, existing);case WAIT -> applyWait(update, existing);- case CALLBACK, CHAINED_INVOKE -> throw invalid(update.type() + " operations are not supported yet");+ case CALLBACK -> applyCallback(update, existing);+ case CHAINED_INVOKE -> throw invalid(update.type() + " operations are not supported yet");default -> throw invalid("Unknown operation type.");}}@@ -340,6 +368,32 @@ final class DurableCheckpointApplier {}}+ /** Only the function starts a callback. SendDurableExecutionCallback* completes it. */+ private void applyCallback(DurableOperationUpdate update, DurableOperation existing) {+ if (update.action() != DurableOperationAction.START) {+ throw invalid("Invalid action for the given operation type.");+ }+ if (existing != null) {+ throw invalid("Cannot start a CALLBACK that already exist.");+ }+ DurableOperation operation = create(update);+ operation.setCallbackId(DurableTokens.callbackId(execution.getExecutionArn(), update.id()));+ Map<String, Object> details = new LinkedHashMap<>();+ details.put("CallbackId", operation.getCallbackId());+ Integer timeout = update.callbackTimeoutSeconds();+ if (timeout != null && timeout > 0) {+ operation.setCallbackDeadline(now + timeout * 1000L);+ details.put("Timeout", timeout);+ }+ Integer heartbeat = update.callbackHeartbeatTimeoutSeconds();+ if (heartbeat != null && heartbeat > 0) {+ operation.setHeartbeatTimeoutSeconds(heartbeat);+ operation.setHeartbeatDeadline(now + heartbeat * 1000L);+ details.put("HeartbeatTimeout", heartbeat);+ }+ event(operation, "CallbackStarted", details);+ }+private static void requireMatchingResult(DurableOperationUpdate update) {if (update.action() == DurableOperationAction.SUCCEED && update.error() != null) {throw invalid("Cannot provide an Error for SUCCEED action.");diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.javaindex fb9cb0e408..29ac1b37c2 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.java@@ -28,6 +28,7 @@ import jakarta.ws.rs.core.MediaType;import jakarta.ws.rs.core.Response;import java.io.IOException;+import java.nio.charset.StandardCharsets;import java.time.Instant;import java.time.LocalDate;import java.time.ZoneOffset;@@ -53,6 +54,7 @@ public class DurableExecutionController {/** Valid Statuses filters for a state Floci never puts an execution in. */private static final Set<String> UNMODELLED_STATUSES = Set.of("PAUSED", "PAUSING", "DELETING");+ private static final int MAX_CALLBACK_PAYLOAD_BYTES = 1024 * 1024;private final DurableExecutionService service;private final LambdaService lambdaService;@@ -179,6 +181,48 @@ public class DurableExecutionController {return Response.ok(response).build();}+ @POST+ @Path("/durable-execution-callbacks/{callbackId: .+}/succeed")+ public Response callbackSucceed(@Context HttpHeaders headers, @PathParam("callbackId") String callbackId,+ byte[] body) {+ if (body != null && MAX_CALLBACK_PAYLOAD_BYTES < body.length) {+ throw new AwsException("ValidationException", "1 validation error detected: Value at 'result' failed to "+ + "satisfy constraint: Member must have length less than or equal to " + MAX_CALLBACK_PAYLOAD_BYTES,+ 400);+ }+ String result = body == null || body.length == 0 ? null : new String(body, StandardCharsets.UTF_8);+ service.completeCallback(callbackId, regionResolver.getAccountId(), regionResolver.resolveRegion(headers), true,+ result, null);+ return Response.ok(objectMapper.createObjectNode()).build();+ }++ /**+ * Without a body the callback fails with no Error, and AWS does not invoke the function for it.+ * Any other body must be a JSON object.+ */+ @POST+ @Path("/durable-execution-callbacks/{callbackId: .+}/fail")+ public Response callbackFail(@Context HttpHeaders headers, @PathParam("callbackId") String callbackId,+ String body) {+ DurableErrorObject error = body == null || body.isEmpty() ? null : parseCallbackError(body);+ // AWS measures the error as compact JSON, whatever spacing the request used.+ if (error != null+ && MAX_CALLBACK_PAYLOAD_BYTES < DurableCheckpointApplier.utf8Length(DurableWire.error(error).toString())) {+ throw new AwsException("InvalidParameterValueException", "Error object size must be less than or equal to "+ + MAX_CALLBACK_PAYLOAD_BYTES + " bytes.", 400);+ }+ service.completeCallback(callbackId, regionResolver.getAccountId(), regionResolver.resolveRegion(headers), false,+ null, error);+ return Response.ok(objectMapper.createObjectNode()).build();+ }++ @POST+ @Path("/durable-execution-callbacks/{callbackId: .+}/heartbeat")+ public Response callbackHeartbeat(@Context HttpHeaders headers, @PathParam("callbackId") String callbackId) {+ service.heartbeatCallback(callbackId, regionResolver.getAccountId(), regionResolver.resolveRegion(headers));+ return Response.ok(objectMapper.createObjectNode()).build();+ }+/*** An execution of another account is not found. One of another region reads as "Function not found",* as AWS answers for an execution ARN of another region.@@ -194,6 +238,20 @@ public class DurableExecutionController {return arn;}+ /** AWS answers whitespace, null or broken JSON with a SerializationException that has no message. */+ private DurableErrorObject parseCallbackError(String body) {+ Map<String, Object> error;+ try {+ error = objectMapper.readValue(body, new TypeReference<Map<String, Object>>() {});+ } catch (IOException e) {+ error = null;+ }+ if (error == null) {+ throw new AwsException("SerializationException", null, 400);+ }+ return DurableWire.parseError(error);+ }+private Map<String, Object> readObject(String body) {if (body == null || body.isBlank()) {return Map.of();diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.javaindex 2a7b6a7770..9a482866de 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java@@ -34,6 +34,7 @@ import java.time.Clock;import java.time.Duration;import java.util.ArrayList;import java.util.Comparator;+import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Objects;@@ -75,6 +76,8 @@ public class DurableExecutionService implements Resettable {static final String INVALID_TOKEN = "Invalid checkpoint token";static final String TOKEN_FOR_OTHER_EXECUTION = "Checkpoint token is not valid for the durable execution ARN";static final String NOT_FOUND = "Durable Execution does not exist";+ static final String INVALID_CALLBACK_ID = "Invalid callback id";+ static final String CALLBACK_CLOSED = "The callback is either timed out or already completed";private final AccountAwareStorageBackend<DurableExecution> store;private final ObjectMapper objectMapper;@@ -300,6 +303,101 @@ public class DurableExecutionService implements Resettable {return execution;}+ /**+ * SendDurableExecutionCallbackSuccess when {@code succeeded}, else SendDurableExecutionCallbackFailure.+ * The callback's operation completes and the function is invoked again to see it, except for a+ * failure sent with no body: AWS records it but does not invoke the function.+ */+ public void completeCallback(String callbackId, String accountId, String region, boolean succeeded, String result,+ DurableErrorObject error) {+ DurableTokens.CallbackId id = parseCallbackId(callbackId);+ ArnParts arn = callbackArn(id, accountId, region);+ List<Runnable> effects = new ArrayList<>();+ synchronized (lockFor(arn)) {+ long now = clock.millis();+ DurableExecution execution = loadPendingCallbackExecution(arn, id, callbackId, now);+ DurableOperation operation = execution.getOperations().get(id.operationId());+ operation.setStatus(succeeded ? DurableOperationStatus.SUCCEEDED : DurableOperationStatus.FAILED);+ operation.setResult(succeeded ? result : null);+ operation.setError(succeeded ? null : error);+ operation.setEndTimestamp(now);+ operation.setCallbackDeadline(null);+ operation.setHeartbeatDeadline(null);+ operation.setChangeSequence(execution.nextChangeSequence());+ Map<String, Object> details = new LinkedHashMap<>();+ if (succeeded) {+ details.put("Result", DurableHistory.payloadWrapper(result));+ } else {+ details.put("Error", DurableHistory.errorWrapper(error));+ }+ DurableHistory.operationEvent(execution, operation, succeeded ? "CallbackSucceeded" : "CallbackFailed",+ now, details);+ if (succeeded || error != null) {+ trigger(execution, effects);+ }+ save(execution);+ }+ runEffects(effects);+ }++ /** SendDurableExecutionCallbackHeartbeat. It moves the heartbeat deadline when the callback has one. */+ public void heartbeatCallback(String callbackId, String accountId, String region) {+ DurableTokens.CallbackId id = parseCallbackId(callbackId);+ ArnParts arn = callbackArn(id, accountId, region);+ synchronized (lockFor(arn)) {+ long now = clock.millis();+ DurableExecution execution = loadPendingCallbackExecution(arn, id, callbackId, now);+ DurableOperation operation = execution.getOperations().get(id.operationId());+ if (operation.getHeartbeatTimeoutSeconds() != null) {+ operation.setHeartbeatDeadline(now + operation.getHeartbeatTimeoutSeconds() * 1000L);+ save(execution);+ }+ }+ }++ private static DurableTokens.CallbackId parseCallbackId(String callbackId) {+ return DurableTokens.parseCallbackId(callbackId)+ .orElseThrow(() -> new AwsException("InvalidParameterValueException", INVALID_CALLBACK_ID, 400));+ }++ /** A callback of another account or region is one this caller cannot complete. */+ private static ArnParts callbackArn(DurableTokens.CallbackId id, String accountId, String region) {+ ArnParts arn;+ try {+ arn = parseArn(id.executionArn());+ } catch (AwsException e) {+ throw new AwsException("InvalidParameterValueException", INVALID_CALLBACK_ID, 400);+ }+ if (!arn.accountId().equals(accountId) || !arn.region().equals(region)) {+ throw callbackClosed();+ }+ return arn;+ }++ /** A callback past its deadline is closed even before the sweeper records the timeout. */+ private DurableExecution loadPendingCallbackExecution(ArnParts arn, DurableTokens.CallbackId id, String callbackId,+ long now) {+ DurableExecution execution = store.getForAccount(arn.accountId(), arn.storeKey())+ .filter(candidate -> candidate.getExecutionArn().equals(arn.arn()))+ .orElseThrow(DurableExecutionService::callbackClosed);+ DurableOperation operation = execution.getOperations().get(id.operationId());+ if (execution.isClosed() || operation == null || operation.getType() != DurableOperationType.CALLBACK+ || operation.getStatus() != DurableOperationStatus.STARTED+ || !callbackId.equals(operation.getCallbackId())+ || isPast(operation.getCallbackDeadline(), now) || isPast(operation.getHeartbeatDeadline(), now)) {+ throw callbackClosed();+ }+ return execution;+ }++ private static boolean isPast(Long deadline, long now) {+ return deadline != null && deadline <= now;+ }++ private static AwsException callbackClosed() {+ return new AwsException("CallbackTimeoutException", CALLBACK_CLOSED, 400);+ }+/** One tick of {@link DurableExecutionSweeper}. It fires due timers, times out and expires executions. */public void sweep() {long now = clock.millis();diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.javaindex df4fda7a3e..eb135daa15 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.java@@ -3,13 +3,18 @@ package io.github.hectorvent.floci.services.lambda.durable;import java.nio.charset.StandardCharsets;import java.util.Base64;import java.util.Optional;+import java.util.UUID;+import java.util.regex.Pattern;/**- * The opaque CheckpointToken of the durable execution protocol, standard base64 as AWS documents it.+ * The opaque CheckpointToken and CallbackId of the durable execution protocol, standard base64 as+ * AWS documents them. A CallbackId carries its execution ARN, which IAM authorizes against.*/-final class DurableTokens {+public final class DurableTokens {private static final String SEPARATOR = "|";+ private static final Pattern CALLBACK_ID_PATTERN = Pattern.compile("[A-Za-z0-9+/]+={0,2}");+ private static final int MAX_CALLBACK_ID_LENGTH = 1024;private DurableTokens() {}@@ -17,6 +22,9 @@ final class DurableTokens {record CheckpointToken(String executionArn, String invocationId, long sequence) {}+ record CallbackId(String executionArn, String operationId) {+ }+static String checkpointToken(String executionArn, String invocationId, long sequence) {return encode(executionArn + SEPARATOR + invocationId + SEPARATOR + sequence);}@@ -41,6 +49,28 @@ final class DurableTokens {}}+ /** The nonce keeps an id from matching a later callback that reuses the operation id. */+ static String callbackId(String executionArn, String operationId) {+ return encode(executionArn + SEPARATOR + operationId + SEPARATOR + UUID.randomUUID());+ }++ static Optional<CallbackId> parseCallbackId(String callbackId) {+ if (callbackId == null || callbackId.length() > MAX_CALLBACK_ID_LENGTH+ || !CALLBACK_ID_PATTERN.matcher(callbackId).matches()) {+ return Optional.empty();+ }+ String decoded = decode(callbackId);+ String[] parts = decoded == null ? new String[0] : decoded.split("\\" + SEPARATOR, -1);+ if (parts.length != 3 || parts[0].isEmpty() || parts[1].isEmpty()) {+ return Optional.empty();+ }+ return Optional.of(new CallbackId(parts[0], parts[1]));+ }++ public static Optional<String> callbackExecutionArn(String callbackId) {+ return parseCallbackId(callbackId).map(CallbackId::executionArn);+ }+private static String encode(String value) {return Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8));}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.javaindex e49b9218f1..20a1196bb0 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java@@ -28,6 +28,7 @@ public final class DurableWire {private static final JsonNodeFactory NODES = JsonNodeFactory.instance;private static final long MAX_DELAY_SECONDS = 31_622_400L;+ private static final long MAX_CALLBACK_TIMEOUT_SECONDS = 99_999_999L;private DurableWire() {}@@ -118,6 +119,11 @@ public final class DurableWire {putTimestamp(details, "ScheduledEndTimestamp", operation.getScheduledEndTimestamp(), millis);}}+ case CALLBACK -> {+ ObjectNode details = node.putObject("CallbackDetails");+ putIfPresent(details, "CallbackId", operation.getCallbackId());+ putResultAndError(details, operation);+ }default -> {}}@@ -203,7 +209,7 @@ public final class DurableWire {if (!(item instanceof Map<?, ?> map)) {throw new AwsException("SerializationException", "Updates members must be JSON objects", 400);}- updates.add(parseUpdate(map));+ updates.add(parseUpdate(map, "updates." + (updates.size() + 1) + ".member."));}return updates;}@@ -250,12 +256,14 @@ public final class DurableWire {result != null && result.isTextual() ? result.asText() : null, error);}- private static DurableOperationUpdate parseUpdate(Map<?, ?> map) {+ /** {@code path} prefixes the member in a ValidationException, as in "updates.1.member.waitOptions.waitSeconds". */+ private static DurableOperationUpdate parseUpdate(Map<?, ?> map, String path) {DurableOperationType type = enumMember(map, "Type", DurableOperationType.class);DurableOperationAction action = enumMember(map, "Action", DurableOperationAction.class);Map<?, ?> stepOptions = structure(map, "StepOptions");Map<?, ?> waitOptions = structure(map, "WaitOptions");Map<?, ?> contextOptions = structure(map, "ContextOptions");+ Map<?, ?> callbackOptions = structure(map, "CallbackOptions");return new DurableOperationUpdate(string(map, "Id"),string(map, "ParentId"),@@ -265,9 +273,12 @@ public final class DurableWire {action,string(map, "Payload"),parseError(map.get("Error")),- stepOptions == null ? null : integer(stepOptions, "NextAttemptDelaySeconds", "StepOptions"),- waitOptions == null ? null : integer(waitOptions, "WaitSeconds", "WaitOptions"),- contextOptions == null ? null : bool(contextOptions, "ReplayChildren"));+ integer(stepOptions, "NextAttemptDelaySeconds", path + "stepOptions", 1, MAX_DELAY_SECONDS),+ integer(waitOptions, "WaitSeconds", path + "waitOptions", 1, MAX_DELAY_SECONDS),+ contextOptions == null ? null : bool(contextOptions, "ReplayChildren"),+ integer(callbackOptions, "TimeoutSeconds", path + "callbackOptions", 0, MAX_CALLBACK_TIMEOUT_SECONDS),+ integer(callbackOptions, "HeartbeatTimeoutSeconds", path + "callbackOptions", 0,+ MAX_CALLBACK_TIMEOUT_SECONDS));}private static <E extends Enum<E>> E enumMember(Map<?, ?> map, String member, Class<E> type) {@@ -305,20 +316,19 @@ public final class DurableWire {throw new AwsException("SerializationException", member + " must be a string", 400);}- private static Integer integer(Map<?, ?> map, String member, String structure) {- Object value = map.get(member);+ private static Integer integer(Map<?, ?> structure, String member, String structurePath, long min, long max) {+ Object value = structure == null ? null : structure.get(member);if (value == null) {return null;}+ String field = structurePath + "." + Character.toLowerCase(member.charAt(0)) + member.substring(1);if (!(value instanceof Number number && (number instanceof Byte || number instanceof Short|| number instanceof Integer || number instanceof Long))) {- throw new AwsException("SerializationException", structure + "." + member + " must be an integer", 400);+ throw new AwsException("SerializationException", field + " must be an integer", 400);}long seconds = number.longValue();- if (seconds < 1 || MAX_DELAY_SECONDS < seconds) {- String field = Character.toLowerCase(structure.charAt(0)) + structure.substring(1) + "."- + Character.toLowerCase(member.charAt(0)) + member.substring(1);- String bound = seconds < 1 ? "greater than or equal to 1" : "less than or equal to " + MAX_DELAY_SECONDS;+ if (seconds < min || max < seconds) {+ String bound = seconds < min ? "greater than or equal to " + min : "less than or equal to " + max;throw new AwsException("ValidationException", "1 validation error detected: Value '" + seconds+ "' at '" + field + "' failed to satisfy constraint: Member must have value " + bound, 400);}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.javaindex 9ff7e2beb6..bdaa1fb5a8 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java@@ -66,7 +66,7 @@ public class DurableExecution {return changeSequence > seenSequence;}- /** True while a wait or a step retry is still pending. */+ /** True while a wait, a step retry or a callback is still pending. */@JsonIgnorepublic boolean hasPendingOperations() {for (DurableOperation operation : operations.values()) {@@ -83,6 +83,11 @@ public class DurableExecution {return true;}}+ case CALLBACK -> {+ if (operationStatus == DurableOperationStatus.STARTED) {+ return true;+ }+ }default -> {}}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.javaindex 1ff6aa8f5c..827f9e976e 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java@@ -29,6 +29,11 @@ public class DurableOperation {private Integer attempt;private Long nextAttemptTimestamp;private Long scheduledEndTimestamp;+ private String callbackId;+ private Integer heartbeatTimeoutSeconds;+ private Long callbackDeadline;+ /** Moves forward on every SendDurableExecutionCallbackHeartbeat. */+ private Long heartbeatDeadline;/** Position in the execution's change log. The handler is only shown operations newer than it has seen. */private long changeSequence;@@ -52,6 +57,10 @@ public class DurableOperation {copy.attempt = attempt;copy.nextAttemptTimestamp = nextAttemptTimestamp;copy.scheduledEndTimestamp = scheduledEndTimestamp;+ copy.callbackId = callbackId;+ copy.heartbeatTimeoutSeconds = heartbeatTimeoutSeconds;+ copy.callbackDeadline = callbackDeadline;+ copy.heartbeatDeadline = heartbeatDeadline;copy.changeSequence = changeSequence;return copy;}@@ -101,6 +110,20 @@ public class DurableOperation {public Long getScheduledEndTimestamp() { return scheduledEndTimestamp; }public void setScheduledEndTimestamp(Long scheduledEndTimestamp) { this.scheduledEndTimestamp = scheduledEndTimestamp; }+ public String getCallbackId() { return callbackId; }+ public void setCallbackId(String callbackId) { this.callbackId = callbackId; }++ public Integer getHeartbeatTimeoutSeconds() { return heartbeatTimeoutSeconds; }+ public void setHeartbeatTimeoutSeconds(Integer heartbeatTimeoutSeconds) {+ this.heartbeatTimeoutSeconds = heartbeatTimeoutSeconds;+ }++ public Long getCallbackDeadline() { return callbackDeadline; }+ public void setCallbackDeadline(Long callbackDeadline) { this.callbackDeadline = callbackDeadline; }++ public Long getHeartbeatDeadline() { return heartbeatDeadline; }+ public void setHeartbeatDeadline(Long heartbeatDeadline) { this.heartbeatDeadline = heartbeatDeadline; }+public long getChangeSequence() { return changeSequence; }public void setChangeSequence(long changeSequence) { this.changeSequence = changeSequence; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.javaindex ca9b275f4e..e43c4f1860 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java@@ -14,5 +14,7 @@ public record DurableOperationUpdate(String id,DurableErrorObject error,Integer nextAttemptDelaySeconds,Integer waitSeconds,- Boolean replayChildren) {+ Boolean replayChildren,+ Integer callbackTimeoutSeconds,+ Integer callbackHeartbeatTimeoutSeconds) {}diff --git a/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.javaindex d9f8926544..94388e26ea 100644--- a/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java@@ -222,6 +222,10 @@ class IamActionRegistryTest {assertEquals("lambda:GetDurableExecution", lambdaAction("GET", base));assertEquals("lambda:ListDurableExecutionsByFunction",lambdaAction("GET", "/2025-12-01/functions/f/durable-executions"));+ String callback = "/2025-12-01/durable-execution-callbacks/QUJD%2BREVG%2FR0g%3D";+ assertEquals("lambda:SendDurableExecutionCallbackSuccess", lambdaAction("POST", callback + "/succeed"));+ assertEquals("lambda:SendDurableExecutionCallbackFailure", lambdaAction("POST", callback + "/fail"));+ assertEquals("lambda:SendDurableExecutionCallbackHeartbeat", lambdaAction("POST", callback + "/heartbeat"));}private String lambdaAction(String method, String path) {diff --git a/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.javaindex 42642f04d3..83888395b7 100644--- a/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java@@ -15,6 +15,7 @@ import java.io.ByteArrayInputStream;import java.io.IOException;import java.io.InputStream;import java.nio.charset.StandardCharsets;+import java.util.Base64;import java.util.HashMap;import java.util.List;import java.util.Map;@@ -605,6 +606,17 @@ class ResourceArnBuilderTest {arn);}+ @Test+ void lambdaDurableCallbackRoutesNameTheExecutionInTheCallbackId() {+ String executionArn = "arn:aws:lambda:us-east-1:000000000000:function:my-function:1/durable-execution/run-1/0b1c";+ String callbackId = Base64.getEncoder().encodeToString((executionArn + "|c1|nonce").getBytes(StandardCharsets.UTF_8));+ when(uriInfo.getPath()).thenReturn("/2025-12-01/durable-execution-callbacks/" + callbackId + "/succeed");+ assertEquals(executionArn, builder.build("lambda", ctx, "us-east-1", "000000000000"));++ when(uriInfo.getPath()).thenReturn("/2025-12-01/durable-execution-callbacks/not-a-callback/heartbeat");+ assertEquals("*", builder.build("lambda", ctx, "us-east-1", "000000000000"));+ }+@Testvoid lambdaDurableExecutionGetNamesTheExecution() {when(uriInfo.getPath()).thenReturn("/2025-12-01/durable-executions/arn:aws:lambda:us-east-1:000000000000:"diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.javaindex 65963c5b9d..400c80d89a 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java@@ -20,6 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows;class DurableCheckpointApplierTest {private static final long NOW = 1_700_000_000_000L;+ private static final String EXECUTION_ARN =+ "arn:aws:lambda:us-east-1:000000000000:function:fn:1/durable-execution/exec/exec-id";@Testvoid executionUpdateMustBeTheOnlyOneAndTheLast() {@@ -31,10 +33,10 @@ class DurableCheckpointApplierTest {void anExecutionCloseCarriesOnlyItsOwnResultKind() {DurableErrorObject error = DurableErrorObject.of("m", "T");assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,- DurableOperationAction.SUCCEED, "\"p\"", error, null, null, null)),+ DurableOperationAction.SUCCEED, "\"p\"", error, null, null, null, null, null)),"Cannot provide an Error for SUCCEED action.");assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,- DurableOperationAction.FAIL, "\"p\"", error, null, null, null)),+ DurableOperationAction.FAIL, "\"p\"", error, null, null, null, null, null)),"Cannot provide a Payload for FAIL action.");}@@ -42,15 +44,16 @@ class DurableCheckpointApplierTest {void anExecutionErrorHasTheSameSizeLimitAsAnOperationError() {DurableErrorObject large = DurableErrorObject.of("x".repeat(DurableCheckpointApplier.MAX_ERROR_BYTES), "T");assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,- DurableOperationAction.FAIL, null, large, null, null, null)),+ DurableOperationAction.FAIL, null, large, null, null, null, null, null)),"Error object size must be less than 32768 bytes.");}@Testvoid duplicateIdsAreRejectedUnlessAStartIsClosedInTheSameBatch() {assertRejected(List.of(stepStart("s1", null), stepStart("s1", null)),- "Cannot checkpoint multiple operations with the same ID.");- assertRejected(List.of(waitStart("w1"), waitStart("w1")), "Cannot checkpoint multiple operations with the same ID.");+ "Cannot update the same operation twice in a single request.");+ assertRejected(List.of(waitStart("w1"), waitStart("w1")),+ "Cannot update the same operation twice in a single request.");DurableExecution execution = execution();DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null), update("s1", null,DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1")), NOW);@@ -74,7 +77,7 @@ class DurableCheckpointApplierTest {assertEquals("Invalid current STEP state to start.", reject(execution, List.of(stepStart("s1", null))));assertEquals("Cannot provide an Error for SUCCEED action.", reject(execution, List.of(new DurableOperationUpdate("s1", null, null, DurableOperationType.STEP, null,- DurableOperationAction.SUCCEED, null, DurableErrorObject.of("x", "y"), null, null, null))));+ DurableOperationAction.SUCCEED, null, DurableErrorObject.of("x", "y"), null, null, null, null, null))));assertEquals("Invalid StepOptions for the given action.", reject(execution, List.of(update("s1", null, DurableOperationType.STEP, DurableOperationAction.RETRY, null))));DurableCheckpointApplier.apply(execution, List.of(update("s1", null, DurableOperationType.STEP,@@ -98,22 +101,68 @@ class DurableCheckpointApplierTest {}@Test- void delaysOutsideTheirRangeAreValidationErrors() {- AwsException rejected = assertThrows(AwsException.class, () -> DurableWire.parseUpdates(+ void optionsOutsideTheirRangeAreValidationErrorsNamingTheUpdate() {+ AwsException wait = assertThrows(AwsException.class, () -> DurableWire.parseUpdates(List.of(Map.of("Id", "w1", "Type", "WAIT", "Action", "START", "WaitOptions", Map.of("WaitSeconds", 0)))));- assertEquals("ValidationException", rejected.getErrorCode());- assertEquals("1 validation error detected: Value '0' at 'waitOptions.waitSeconds' failed to satisfy "- + "constraint: Member must have value greater than or equal to 1", rejected.getMessage());+ assertEquals("ValidationException", wait.getErrorCode());+ assertEquals("1 validation error detected: Value '0' at 'updates.1.member.waitOptions.waitSeconds' failed to "+ + "satisfy constraint: Member must have value greater than or equal to 1", wait.getMessage());++ AwsException callback = assertThrows(AwsException.class, () -> DurableWire.parseUpdates(List.of(+ Map.of("Id", "c1", "Type", "CALLBACK", "Action", "START"),+ Map.of("Id", "c2", "Type", "CALLBACK", "Action", "START",+ "CallbackOptions", Map.of("HeartbeatTimeoutSeconds", 100_000_000)))));+ assertEquals("1 validation error detected: Value '100000000' at "+ + "'updates.2.member.callbackOptions.heartbeatTimeoutSeconds' failed to satisfy constraint: Member "+ + "must have value less than or equal to 99999999", callback.getMessage());}@Test- void callbacksAndChainedInvokesAreNotSupportedYet() {- assertRejected(List.of(update("c1", null, DurableOperationType.CALLBACK, DurableOperationAction.START, null)),- "CALLBACK operations are not supported yet");+ void chainedInvokesAreNotSupportedYet() {assertRejected(List.of(update("i1", null, DurableOperationType.CHAINED_INVOKE, DurableOperationAction.START,null)), "CHAINED_INVOKE operations are not supported yet");}+ @Test+ void aCallbackStartsWithAnIdThatNamesItsExecution() {+ DurableExecution execution = execution();+ DurableCheckpointApplier.apply(execution, List.of(callbackStart("c1", 30, 10)), NOW);++ DurableOperation callback = execution.getOperations().get("c1");+ assertEquals(DurableOperationStatus.STARTED, callback.getStatus());+ assertEquals(EXECUTION_ARN, DurableTokens.callbackExecutionArn(callback.getCallbackId()).orElseThrow());+ assertEquals(NOW + 30_000, callback.getCallbackDeadline());+ assertEquals(NOW + 10_000, callback.getHeartbeatDeadline());+ assertEquals(Map.of("CallbackStartedDetails", Map.of("CallbackId", callback.getCallbackId(), "Timeout", 30,+ "HeartbeatTimeout", 10)), execution.getHistory().get(0).getDetails());++ assertEquals("Cannot start a CALLBACK that already exist.", reject(execution, List.of(callbackStart("c1", 0, 0))));+ assertEquals("Invalid action for the given operation type.", reject(execution, List.of(+ update("c1", null, DurableOperationType.CALLBACK, DurableOperationAction.SUCCEED, "\"r\""))));+ }++ @Test+ void aCallbackTimesOutOnTheDeadlineItMissesFirst() {+ DurableExecution execution = execution();+ DurableCheckpointApplier.apply(execution, List.of(callbackStart("slow", 5, 0), callbackStart("silent", 60, 3)),+ NOW);++ DurableCheckpointApplier.fireDueTimers(execution, NOW + 3_000);+ DurableOperation silent = execution.getOperations().get("silent");+ assertEquals(DurableOperationStatus.TIMED_OUT, silent.getStatus());+ assertEquals("Callback.Heartbeat", silent.getError().getErrorType());+ assertEquals("Callback timed out on heartbeat", silent.getError().getErrorMessage());+ assertEquals(DurableOperationStatus.STARTED, execution.getOperations().get("slow").getStatus());++ DurableCheckpointApplier.fireDueTimers(execution, NOW + 5_000);+ DurableOperation slow = execution.getOperations().get("slow");+ assertEquals("Callback.Timeout", slow.getError().getErrorType());+ assertEquals("Callback timed out", slow.getError().getErrorMessage());+ assertEquals(Map.of("CallbackTimedOutDetails", Map.of("Error", Map.of("Payload",+ Map.of("ErrorType", "Callback.Timeout"), "Truncated", false))),+ execution.getHistory().get(execution.getHistory().size() - 1).getDetails());+ }+@Testvoid aRejectedBatchLeavesTheExecutionUntouched() {DurableExecution execution = execution();@@ -140,6 +189,7 @@ class DurableCheckpointApplierTest {private static DurableExecution execution() {DurableExecution execution = new DurableExecution();execution.setExecutionId("exec-id");+ execution.setExecutionArn(EXECUTION_ARN);execution.setName("exec");execution.setMaxResultBytes(DurableExecutionService.ASYNC_PAYLOAD_LIMIT);DurableOperation root = new DurableOperation();@@ -158,7 +208,12 @@ class DurableCheckpointApplierTest {private static DurableOperationUpdate waitStart(String id) {return new DurableOperationUpdate(id, null, null, DurableOperationType.WAIT, null, DurableOperationAction.START,- null, null, null, 5, null);+ null, null, null, 5, null, null, null);+ }++ private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds, int heartbeatSeconds) {+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds);}private static DurableOperationUpdate executionSucceed() {@@ -167,6 +222,7 @@ class DurableCheckpointApplierTest {private static DurableOperationUpdate update(String id, String parentId, DurableOperationType type,DurableOperationAction action, String payload) {- return new DurableOperationUpdate(id, parentId, null, type, null, action, payload, null, null, null, null);+ return new DurableOperationUpdate(id, parentId, null, type, null, action, payload, null, null, null, null, null,+ null);}}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.javaindex b967e1ff07..45642c4f6b 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java@@ -93,6 +93,43 @@ class DurableExecutionServicePersistenceTest {execution.getHistory().stream().map(DurableHistoryEvent::getEventType).toList());}+ @Test+ void aPendingCallbackKeepsItsIdAndDeadlineAcrossARestart() {+ ScriptedInvoker before = new ScriptedInvoker();+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);+ DurableExecutionService first = newService(firstStorage, before);+ before.script((service, event) -> {+ service.checkpoint(arn(event), token(event), null, List.of(callbackStart("approve", 0),+ callbackStart("expire", 5)));+ return "{\"Status\":\"PENDING\"}";+ });+ String executionArn = first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{}",+ false)).getExecutionArn();+ String callbackId = first.get(executionArn).getOperations().get("approve").getCallbackId();+ firstStorage.flushAll();++ ScriptedInvoker after = new ScriptedInvoker();+ DurableExecutionService restarted = newService(new PersistentStorageFactory(directory), after);+ after.script((service, event) -> "{\"Status\":\"PENDING\"}");+ after.script((service, event) -> {+ assertEquals("SUCCEEDED", operation(event, "approve").get("Status").asText());+ assertEquals("STARTED", operation(event, "expire").get("Status").asText());+ return "{\"Status\":\"PENDING\"}";+ });+ after.script((service, event) -> {+ assertEquals("Callback.Timeout", operation(event, "expire").at("/CallbackDetails/Error/ErrorType").asText());+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"done\\\"\"}";+ });+ restarted.recoverAfterRestart();++ restarted.completeCallback(callbackId, ACCOUNT, "us-east-1", true, "\"yes\"", null);+ clock.advance(Duration.ofSeconds(5));+ restarted.sweep();++ assertEquals(DurableExecutionStatus.SUCCEEDED, restarted.get(executionArn).getStatus());+ assertEquals(3, after.events.size());+ }+private DurableExecutionService newService(StorageFactory storage, ScriptedInvoker invoker) {DurableExecutionService service = new DurableExecutionService(storage, MAPPER, clock, invoker, Runnable::run,Duration.ZERO);@@ -102,7 +139,12 @@ class DurableExecutionServicePersistenceTest {private static DurableOperationUpdate update(String id, DurableOperationType type, DurableOperationAction action,String payload, Integer waitSeconds) {- return new DurableOperationUpdate(id, null, null, type, null, action, payload, null, null, waitSeconds, null);+ return new DurableOperationUpdate(id, null, null, type, null, action, payload, null, null, waitSeconds, null, null, null);+ }++ private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds) {+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, null);}private static String arn(JsonNode event) {diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.javaindex 10ef6e45a7..6bdd5ced49 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java@@ -19,6 +19,7 @@ import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperationimport io.github.hectorvent.floci.testing.MutableClock;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.function.Executable;import java.io.IOException;import java.io.UncheckedIOException;@@ -513,8 +514,130 @@ class DurableExecutionServiceTest {assertEquals(2, invoker.events.size(), "no extra invocation after the retry ran");}+ @Test+ void aCallbackCompletedFromOutsideReinvokesTheFunctionWithItsResult() {+ invoker.script(event -> {+ CheckpointResult started = checkpoint(event, token(event), List.of(callbackStart("c1", 0, 0)));+ assertNotNull(started.newExecutionState().get(0).getCallbackId());+ return pending();+ });+ invoker.script(event -> {+ JsonNode callback = operation(event, "c1");+ assertEquals("SUCCEEDED", callback.get("Status").asText());+ assertEquals("\"approved\"", callback.at("/CallbackDetails/Result").asText());+ assertEquals("c1", event.get("UpdatedOperationIds").get(0).asText());+ return succeeded("\"done\"");+ });+ String arn = start("exec-1", "{}", false).getExecutionArn();+ String callbackId = service.get(arn).getOperations().get("c1").getCallbackId();++ service.completeCallback(callbackId, ACCOUNT, REGION, true, "\"approved\"", null);++ DurableExecution execution = service.get(arn);+ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());+ assertEquals(List.of("ExecutionStarted", "CallbackStarted", "InvocationCompleted", "CallbackSucceeded",+ "InvocationCompleted", "ExecutionSucceeded"), eventTypes(execution));+ assertCallbackClosed(() -> service.completeCallback(callbackId, ACCOUNT, REGION, false, null, null));+ assertCallbackClosed(() -> service.heartbeatCallback(callbackId, ACCOUNT, REGION));+ }++ @Test+ void aFailedCallbackCarriesTheErrorItWasSent() {+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(callbackStart("c1", 0, 0)));+ return pending();+ });+ invoker.script(event -> {+ JsonNode callback = operation(event, "c1");+ assertEquals("FAILED", callback.get("Status").asText());+ assertEquals("Rejected", callback.at("/CallbackDetails/Error/ErrorType").asText());+ return succeeded("\"handled\"");+ });+ String arn = start("exec-1", "{}", false).getExecutionArn();++ service.completeCallback(service.get(arn).getOperations().get("c1").getCallbackId(), ACCOUNT, REGION, false,+ null, DurableErrorObject.of("denied", "Rejected"));++ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(arn).getStatus());+ }++ @Test+ void aFailureWithoutAnErrorIsRecordedButDoesNotInvokeTheFunction() {+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(callbackStart("c1", 0, 0)));+ return pending();+ });+ String arn = start("exec-1", "{}", false).getExecutionArn();+ String callbackId = service.get(arn).getOperations().get("c1").getCallbackId();++ service.completeCallback(callbackId, ACCOUNT, REGION, false, null, null);++ DurableExecution execution = service.get(arn);+ assertEquals(1, invoker.events.size());+ assertEquals(DurableExecutionStatus.RUNNING, execution.getStatus());+ assertEquals(DurableOperationStatus.FAILED, execution.getOperations().get("c1").getStatus());+ assertEquals("CallbackFailed", eventTypes(execution).get(eventTypes(execution).size() - 1));+ assertCallbackClosed(() -> service.completeCallback(callbackId, ACCOUNT, REGION, true, "\"x\"", null));+ }++ @Test+ void heartbeatsKeepACallbackAliveUntilTheyStop() {+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(callbackStart("c1", 0, 3)));+ return pending();+ });+ invoker.script(event -> {+ assertEquals("Callback.Heartbeat", operation(event, "c1").at("/CallbackDetails/Error/ErrorType").asText());+ return succeeded("\"gave up\"");+ });+ String arn = start("exec-1", "{}", false).getExecutionArn();+ String callbackId = service.get(arn).getOperations().get("c1").getCallbackId();++ clock.advance(Duration.ofSeconds(2));+ service.heartbeatCallback(callbackId, ACCOUNT, REGION);+ clock.advance(Duration.ofSeconds(2));+ service.sweep();+ assertEquals(DurableOperationStatus.STARTED, service.get(arn).getOperations().get("c1").getStatus());++ clock.advance(Duration.ofSeconds(1));+ assertCallbackClosed(() -> service.heartbeatCallback(callbackId, ACCOUNT, REGION));+ service.sweep();++ assertEquals(DurableOperationStatus.TIMED_OUT, service.get(arn).getOperations().get("c1").getStatus());+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(arn).getStatus());+ }++ @Test+ void callbackIdsThatMatchNoOpenCallbackAreRejected() {+ invoker.script(event -> {+ checkpoint(event, token(event), List.of(callbackStart("c1", 0, 0)));+ return pending();+ });+ String arn = start("exec-1", "{}", false).getExecutionArn();+ String callbackId = service.get(arn).getOperations().get("c1").getCallbackId();++ for (String malformed : List.of("QUJD", "not-valid!")) {+ AwsException rejected = assertThrows(AwsException.class,+ () -> service.heartbeatCallback(malformed, ACCOUNT, REGION));+ assertEquals("InvalidParameterValueException", rejected.getErrorCode());+ assertEquals("Invalid callback id", rejected.getMessage());+ }+ assertCallbackClosed(() -> service.heartbeatCallback(callbackId, ACCOUNT, "eu-west-1"));+ assertCallbackClosed(() -> service.heartbeatCallback(callbackId, "111111111111", REGION));++ service.stop(arn, null);+ assertCallbackClosed(() -> service.completeCallback(callbackId, ACCOUNT, REGION, true, "\"late\"", null));+ }+// ──────────────────────────── helpers ────────────────────────────+ private static void assertCallbackClosed(Executable call) {+ AwsException rejected = assertThrows(AwsException.class, call);+ assertEquals("CallbackTimeoutException", rejected.getErrorCode());+ assertEquals("The callback is either timed out or already completed", rejected.getMessage());+ assertEquals(400, rejected.getHttpStatus());+ }+private static DurableExecutionService newService(InMemoryStorageFactory storage, ScriptedInvoker invoker,MutableClock clock) {return new DurableExecutionService(storage, MAPPER, clock, invoker, Runnable::run, Duration.ZERO);@@ -553,22 +676,27 @@ class DurableExecutionServiceTest {private static DurableOperationUpdate step(String id, DurableOperationAction action, String payload,DurableErrorObject error) {return new DurableOperationUpdate(id, null, null, DurableOperationType.STEP, "Step", action, payload, error,- null, null, null);+ null, null, null, null, null);}private static DurableOperationUpdate stepRetry(String id, int delaySeconds, DurableErrorObject error) {return new DurableOperationUpdate(id, null, null, DurableOperationType.STEP, "Step",- DurableOperationAction.RETRY, null, error, delaySeconds, null, null);+ DurableOperationAction.RETRY, null, error, delaySeconds, null, null, null, null);+ }++ private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds, int heartbeatSeconds) {+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds);}private static DurableOperationUpdate waitStart(String id, int seconds) {return new DurableOperationUpdate(id, null, null, DurableOperationType.WAIT, "Wait",- DurableOperationAction.START, null, null, null, seconds, null);+ DurableOperationAction.START, null, null, null, seconds, null, null, null);}private static DurableOperationUpdate executionSucceed(String payload) {return new DurableOperationUpdate("execution-result", null, null, DurableOperationType.EXECUTION, null,- DurableOperationAction.SUCCEED, payload, null, null, null, null);+ DurableOperationAction.SUCCEED, payload, null, null, null, null, null, null);}private static DurableFunctionInvoker.DurableInvocationResult succeeded(String result) {diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.javaindex 4ab96caaea..cc0a3ab4fa 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java@@ -312,6 +312,135 @@ class LambdaDurableProtocolIntegrationTest {.body("Events[0].EventType", equalTo("ExecutionStopped"));}+ @Test+ void aCallbackCompletedOverTheApiResumesTheExecution() throws Exception {+ Response accepted = given()+ .header("X-Amz-Invocation-Type", "Event")+ .header("X-Amz-Durable-Execution-Name", "callback-1")+ .body("{}")+ .post(LAMBDA + "/functions/" + FUNCTION + ":$LATEST/invocations");+ String arn = accepted.getHeader("X-Amz-Durable-Execution-Arn");++ Invocation first = invoker.awaitInvocation(Duration.ofSeconds(30));+ JsonNode event = MAPPER.readTree(first.payload());+ Response started = given()+ .urlEncodingEnabled(false)+ .contentType("application/json")+ .body("""+ {+ "CheckpointToken": "%s",+ "Updates": [{"Id": "c1", "Name": "approval", "Type": "CALLBACK", "SubType": "Callback",+ "Action": "START",+ "CallbackOptions": {"TimeoutSeconds": 300, "HeartbeatTimeoutSeconds": 120}}]+ }+ """.formatted(event.get("CheckpointToken").asText()))+ .post(DURABLE + "/durable-executions/" + arn + "/checkpoint");+ started.then()+ .statusCode(200)+ .body("NewExecutionState.Operations[0].Status", equalTo("STARTED"))+ .body("NewExecutionState.Operations[0].CallbackDetails.CallbackId", notNullValue());+ String callbackId = started.jsonPath().getString("NewExecutionState.Operations[0].CallbackDetails.CallbackId");+ first.response().complete(handlerResponse("{\"Status\": \"PENDING\"}"));++ // The SDKs percent-encode the base64 id. The CLI and curl send it raw.+ String encodedId = callbackId.replace("+", "%2B").replace("/", "%2F").replace("=", "%3D");+ given()+ .urlEncodingEnabled(false)+ .when()+ .post(DURABLE + "/durable-execution-callbacks/" + callbackId + "/heartbeat")+ .then()+ .statusCode(200)+ .body(equalTo("{}"));++ given()+ .urlEncodingEnabled(false)+ .contentType("application/octet-stream")+ .body(new byte[1024 * 1024 + 1])+ .when()+ .post(DURABLE + "/durable-execution-callbacks/" + encodedId + "/succeed")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("1 validation error detected: Value at 'result' failed to satisfy constraint: "+ + "Member must have length less than or equal to 1048576"));++ for (String notAnObject : List.of(" ", "null", "{")) {+ given()+ .urlEncodingEnabled(false)+ .contentType("application/json")+ .body(notAnObject)+ .when()+ .post(DURABLE + "/durable-execution-callbacks/" + encodedId + "/fail")+ .then()+ .statusCode(400)+ .body("__type", equalTo("SerializationException"));+ }++ String oversizedError = "{\"ErrorType\":\"Big\",\"ErrorMessage\":\"" + "m".repeat(1024 * 1024) + "\"}";+ given()+ .urlEncodingEnabled(false)+ .contentType("application/json")+ .body(oversizedError)+ .when()+ .post(DURABLE + "/durable-execution-callbacks/" + encodedId + "/fail")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidParameterValueException"))+ .body("message", equalTo("Error object size must be less than or equal to 1048576 bytes."));++ given()+ .urlEncodingEnabled(false)+ .contentType("application/octet-stream")+ .body("{\"approved\":true}".getBytes(StandardCharsets.UTF_8))+ .when()+ .post(DURABLE + "/durable-execution-callbacks/" + encodedId + "/succeed")+ .then()+ .statusCode(200)+ .body(equalTo("{}"));++ Invocation second = invoker.awaitInvocation(Duration.ofSeconds(30));+ JsonNode resumed = MAPPER.readTree(second.payload());+ JsonNode callback = resumed.at("/InitialExecutionState/Operations/1");+ assertEquals("SUCCEEDED", callback.get("Status").asText());+ assertEquals(callbackId, callback.at("/CallbackDetails/CallbackId").asText());+ assertEquals("{\"approved\":true}", callback.at("/CallbackDetails/Result").asText());+ second.response().complete(handlerResponse("{\"Status\": \"SUCCEEDED\", \"Result\": \"\\\"ok\\\"\"}"));++ given()+ .urlEncodingEnabled(false)+ .contentType("application/json")+ .body("{\"ErrorMessage\": \"too late\"}")+ .when()+ .post(DURABLE + "/durable-execution-callbacks/" + encodedId + "/fail")+ .then()+ .statusCode(400)+ .body("__type", equalTo("CallbackTimeoutException"))+ .body("message", equalTo("The callback is either timed out or already completed"));++ given()+ .urlEncodingEnabled(false)+ .when()+ .post(DURABLE + "/durable-execution-callbacks/not-valid!/heartbeat")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidParameterValueException"))+ .body("message", equalTo("Invalid callback id"));++ given()+ .urlEncodingEnabled(false)+ .when()+ .get(DURABLE + "/durable-executions/" + arn + "/history?IncludeExecutionData=true")+ .then()+ .statusCode(200)+ .body("Events.EventType", equalTo(List.of("ExecutionStarted", "CallbackStarted", "InvocationCompleted",+ "CallbackSucceeded", "InvocationCompleted", "ExecutionSucceeded")))+ .body("Events[1].Name", equalTo("approval"))+ .body("Events[1].CallbackStartedDetails.CallbackId", equalTo(callbackId))+ .body("Events[1].CallbackStartedDetails.Timeout", equalTo(300))+ .body("Events[1].CallbackStartedDetails.HeartbeatTimeout", equalTo(120))+ .body("Events[3].CallbackSucceededDetails.Result.Payload", equalTo("{\"approved\":true}"));+ }+private static DurableInvocationResult handlerResponse(String json) {return new DurableInvocationResult("req-" + System.nanoTime(), json.getBytes(StandardCharsets.UTF_8), null);}