Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(lambda): durable function callbacks (#5100)

TATSUNO “Taz” Yasuhiroa1ed65c51dfa
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java
index d43d0bb34f..5be56bd05a 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaDurableFunctionsTest.java
@@ -22,6 +22,7 @@ class LambdaDurableFunctionsTest {
private static final String ROLE = "arn:aws:iam::000000000000:role/lambda-role";
private static final String FN = TestFixtures.uniqueName("fn-durable-exec");
+ private static final String CALLBACK_FN = TestFixtures.uniqueName("fn-durable-callback");
private static LambdaClient lambda;
@@ -33,10 +34,12 @@ class LambdaDurableFunctionsTest {
@AfterAll
static void cleanup() {
if (lambda != null) {
- try {
- lambda.deleteFunction(DeleteFunctionRequest.builder().functionName(FN).build());
- } catch (Exception ignored) {
- // Cleanup only. A failed delete must not hide the test result.
+ for (String name : new String[] {FN, CALLBACK_FN}) {
+ try {
+ lambda.deleteFunction(DeleteFunctionRequest.builder().functionName(name).build());
+ } catch (Exception ignored) {
+ // Cleanup only. A failed delete must not hide the test result.
+ }
}
lambda.close();
}
@@ -117,4 +120,63 @@ class LambdaDurableFunctionsTest {
.isInstanceOf(InvalidParameterValueException.class)
.hasMessageContaining("You cannot invoke a durable function using an unqualified ARN.");
}
+
+ @Test
+ @DisplayName("a durable callback is completed through the callback APIs and resumes the execution")
+ void durableCallbackResumesTheExecution() throws InterruptedException {
+ Assumptions.assumeTrue(TestFixtures.isLambdaDispatchAvailable(),
+ "skipping: Lambda dispatch (Docker) not available in this environment");
+
+ lambda.createFunction(CreateFunctionRequest.builder()
+ .functionName(CALLBACK_FN)
+ .runtime(Runtime.PYTHON3_14)
+ .role(ROLE)
+ .handler("lambda_function.handler")
+ .timeout(30)
+ .durableConfig(DurableConfig.builder().executionTimeout(120).retentionPeriodInDays(1).build())
+ .code(FunctionCode.builder().zipFile(SdkBytes.fromByteArray(LambdaUtils.durablePythonZip())).build())
+ .build());
+
+ String arn = lambda.invoke(InvokeRequest.builder()
+ .functionName(CALLBACK_FN + ":$LATEST")
+ .invocationType(InvocationType.EVENT)
+ .payload(SdkBytes.fromUtf8String("{\"callback\": true}"))
+ .build()).durableExecutionArn();
+
+ String callbackId = null;
+ for (int i = 0; i < 60 && callbackId == null; i++) {
+ callbackId = lambda.getDurableExecutionHistory(GetDurableExecutionHistoryRequest.builder()
+ .durableExecutionArn(arn).build())
+ .events().stream()
+ .filter(event -> event.eventType() == EventType.CALLBACK_STARTED)
+ .map(event -> event.callbackStartedDetails().callbackId())
+ .findFirst().orElse(null);
+ if (callbackId == null) {
+ Thread.sleep(500);
+ }
+ }
+ assertThat(callbackId).as("the function started a callback").isNotNull();
+ String id = callbackId;
+
+ lambda.sendDurableExecutionCallbackHeartbeat(SendDurableExecutionCallbackHeartbeatRequest.builder()
+ .callbackId(id).build());
+ lambda.sendDurableExecutionCallbackSuccess(SendDurableExecutionCallbackSuccessRequest.builder()
+ .callbackId(id).result(SdkBytes.fromUtf8String("{\"approved\": true}")).build());
+
+ GetDurableExecutionResponse execution = null;
+ for (int i = 0; i < 60; i++) {
+ execution = lambda.getDurableExecution(GetDurableExecutionRequest.builder()
+ .durableExecutionArn(arn).includeExecutionData(true).build());
+ if (execution.status() != ExecutionStatus.RUNNING) {
+ break;
+ }
+ Thread.sleep(500);
+ }
+ assertThat(execution.status()).isEqualTo(ExecutionStatus.SUCCEEDED);
+ assertThat(execution.result()).isEqualTo("{\"approved\": true}");
+
+ assertThatThrownBy(() -> lambda.sendDurableExecutionCallbackSuccess(
+ SendDurableExecutionCallbackSuccessRequest.builder().callbackId(id).build()))
+ .isInstanceOf(CallbackTimeoutException.class);
+ }
}
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
index e93f56cc88..67d3545d50 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
@@ -284,6 +284,19 @@ public final class LambdaUtils {
if request.get("fail"):
return {"Status": "FAILED",
"Error": {"ErrorMessage": "asked to fail", "ErrorType": "TestFailure"}}
+ if request.get("callback"):
+ if "callback" not in operations:
+ client.checkpoint_durable_execution(
+ DurableExecutionArn=event["DurableExecutionArn"],
+ CheckpointToken=event["CheckpointToken"],
+ Updates=[{"Id": "callback", "Name": "approval", "Type": "CALLBACK",
+ "SubType": "Callback", "Action": "START",
+ "CallbackOptions": {"HeartbeatTimeoutSeconds": 60}}])
+ return {"Status": "PENDING"}
+ callback = operations["callback"]
+ if callback["Status"] == "STARTED":
+ return {"Status": "PENDING"}
+ return {"Status": "SUCCEEDED", "Result": callback["CallbackDetails"].get("Result", "")}
if "step" not in operations:
client.checkpoint_durable_execution(
DurableExecutionArn=event["DurableExecutionArn"],
diff --git a/docs/services/index.md b/docs/services/index.md
index 91561711b0..668f03b02f 100644
--- a/docs/services/index.md
+++ b/docs/services/index.md
@@ -18,7 +18,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
| [S3 Tables](s3tables.md) | `/buckets`, `/namespaces/*`, `/tables/*` | REST JSON | 25 |
| [DynamoDB](dynamodb.md) | `POST /` + `X-Amz-Target: DynamoDB_20120810.*` | JSON 1.1 | 28 |
| [DynamoDB Streams](dynamodb.md#streams) | `POST /` + `X-Amz-Target: DynamoDBStreams_20120810.*` | JSON 1.1 | 4 |
-| [Lambda](lambda.md) | `/2015-03-31/functions/...` + `/2025-12-01/durable-executions/...` | REST JSON | 52 |
+| [Lambda](lambda.md) | `/2015-03-31/functions/...` + `/2025-12-01/durable-executions/...` | REST JSON | 55 |
| [Lambda MicroVMs](lambda-microvms.md) | `/2025-09-09/...` + `/2026-04-04/...` | REST JSON | 22 |
| [API Gateway v1](api-gateway.md) | `/restapis/...` | REST JSON | 79 |
| [API Gateway v2](api-gateway.md#v2) | `/v2/apis/...` | REST JSON | 53 + data-plane |
diff --git a/docs/services/lambda.md b/docs/services/lambda.md
index b0dbcb9c8b..73fefbc259 100644
--- a/docs/services/lambda.md
+++ b/docs/services/lambda.md
@@ -61,6 +61,9 @@ Floci Lambda runs your function code locally inside real Docker containers - clo
| `GetDurableExecutionHistory` | Event log of a durable execution, oldest or newest first |
| `ListDurableExecutionsByFunction` | Durable executions of a function or version, filtered by status, name and start time |
| `StopDurableExecution` | Stop a running durable execution with an error |
+| `SendDurableExecutionCallbackSuccess` | Complete a durable callback with a result, up to 1 MB |
+| `SendDurableExecutionCallbackFailure` | Fail a durable callback with an error |
+| `SendDurableExecutionCallbackHeartbeat` | Keep a durable callback with a heartbeat timeout alive |
`UpdateFunctionConfiguration` validates `MemorySize` and `Timeout` as whole numbers in their
supported ranges before changing any stored settings. A rejected value returns
@@ -165,24 +168,33 @@ background. Invocations through an ARN (EventBridge, Scheduler, function URLs, d
durable executions the same way.
The checkpoint protocol under `/2025-12-01` is the real one, so the Durable Execution SDK works
-unchanged for steps, waits, step retries and replay. Timers are persisted and fired by a
+unchanged for steps, waits, step retries, callbacks and replay. Timers are persisted and fired by a
background sweep, so an execution survives a restart of Floci. A function that returns something
other than the `{Status, Result, Error}` envelope fails its execution with
`Invalid Status in invocation output.` A function that throws is re-invoked four more times, after
1, 2, 4 and 8 seconds, and then fails the execution with its error. AWS does the same over a few
minutes.
+A callback (`createCallback`, `waitForCallback`) gets a callback id that the
+`SendDurableExecutionCallback*` APIs complete. Its `TimeoutSeconds` and `HeartbeatTimeoutSeconds`
+fail it with `Callback.Timeout` or `Callback.Heartbeat`, and each heartbeat restarts the heartbeat
+timeout. A failure sent with an empty request body fails the callback but does not invoke the
+function again, as on AWS, so the execution keeps waiting. The SDKs send `{}` when `Error` is
+omitted, which does invoke it.
+
`GetDurableExecutionHistory` leaves payloads out unless `IncludeExecutionData=true` is sent. The
API reference names `true` as the default, but AWS answers this way.
+`SendDurableExecutionCallbackSuccess` takes a result of up to 1 MB. The API reference names 256 KB,
+but AWS accepts 1 MB and rejects one byte more.
+
The public `public.ecr.aws/lambda` images do not bundle the SDK that the managed runtimes carry.
Add `aws-durable-execution-sdk-python` or `@aws/durable-execution-sdk-js` to your deployment
package.
Not emulated yet:
-- Callbacks (`WaitForCallback` and the `SendDurableExecutionCallback*` APIs) and chained invokes
- (`context.invoke`). Both are rejected at checkpoint time.
+- Chained invokes (`context.invoke`). They are rejected at checkpoint time.
- Durable executions started by Step Functions or by an event source mapping. These invoke the
function as a plain one.
- Payload encryption. `KMSKeyArn` is stored and returned only.
@@ -387,8 +399,7 @@ These AWS Lambda operations have no handler in Floci. Calls will return `404` or
- Layer permissions (`AddLayerVersionPermission`, `RemoveLayerVersionPermission`, `GetLayerVersionPolicy`)
- Provisioned concurrency (`PutProvisionedConcurrencyConfig`, `GetProvisionedConcurrencyConfig`, `ListProvisionedConcurrencyConfigs`, `DeleteProvisionedConcurrencyConfig`)
- `InvokeWithResponseStream`
-- Durable function callbacks and chained invokes (`SendDurableExecutionCallbackSuccess`,
- `SendDurableExecutionCallbackFailure`, `SendDurableExecutionCallbackHeartbeat`). See Durable Functions
+- Durable function chained invokes. See Durable Functions
- Code signing enforcement. A configuration is created, read, updated, deleted and listed, and
nothing verifies a signature against it, so it never gates a deployment. Attaching one to a
function is not wired either: there is no `PutFunctionCodeSigningConfig`, so
diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java b/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java
index bfbc8c6321..d84f082f2c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iam/IamActionRegistry.java
@@ -68,6 +68,11 @@ public class IamActionRegistry {
rule("lambda", "GET", ".*/durable-executions/.+/history$", "lambda:GetDurableExecutionHistory"),
rule("lambda", "POST", ".*/durable-executions/.+/stop$", "lambda:StopDurableExecution"),
rule("lambda", "GET", ".*/durable-executions/.+$", "lambda:GetDurableExecution"),
+ rule("lambda", "POST", ".*/durable-execution-callbacks/.+/succeed$",
+ "lambda:SendDurableExecutionCallbackSuccess"),
+ rule("lambda", "POST", ".*/durable-execution-callbacks/.+/fail$", "lambda:SendDurableExecutionCallbackFailure"),
+ rule("lambda", "POST", ".*/durable-execution-callbacks/.+/heartbeat$",
+ "lambda:SendDurableExecutionCallbackHeartbeat"),
// ── DynamoDB (JSON 1.1, action from X-Amz-Target handled separately) ──
// Handled via Query-style action extraction in the filter
diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java b/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java
index b18b26d224..9514801287 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java
@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.core.common.AwsQueryServiceResolver;
import io.github.hectorvent.floci.core.common.AwsRegions;
import io.github.hectorvent.floci.services.iam.model.ServerCertificate;
import io.github.hectorvent.floci.services.lambda.LambdaArnUtils;
+import io.github.hectorvent.floci.services.lambda.durable.DurableTokens;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.inject.Instance;
import jakarta.inject.Inject;
@@ -224,6 +225,9 @@ public class ResourceArnBuilder {
// ── Lambda ──────────────────────────────────────────────────────────────────
private String buildLambdaArn(String path, String region, String accountId) {
String executionArn = durableExecutionArn(path);
+ if (executionArn == null) {
+ executionArn = durableCallbackExecutionArn(path);
+ }
if (executionArn != null) {
return executionArn;
}
@@ -247,6 +251,19 @@ public class ResourceArnBuilder {
return matcher.lookingAt() ? matcher.group() : null;
}
+ /** A callback id carries its execution ARN, so the callback actions are authorized against that execution. */
+ private static String durableCallbackExecutionArn(String path) {
+ String prefix = "/durable-execution-callbacks/";
+ int callbacks = path.indexOf(prefix);
+ int action = path.lastIndexOf('/');
+ if (callbacks < 0 || action <= callbacks + prefix.length()) {
+ return null;
+ }
+ return DurableTokens.callbackExecutionArn(path.substring(callbacks + prefix.length(), action))
+ .filter(arn -> LambdaArnUtils.DURABLE_EXECUTION_ARN.matcher(arn).matches())
+ .orElse(null);
+ }
+
// ── SQS ─────────────────────────────────────────────────────────────────────
private String buildSqsArn(ContainerRequestContext ctx, String region, String accountId) {
String queueUrl = ctx.getUriInfo().getQueryParameters().getFirst("QueueUrl");
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
index 315754cfa7..0051ccb852 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplier.java
@@ -48,7 +48,7 @@ final class DurableCheckpointApplier {
return new Outcome(draft.closingStatus != null);
}
- /** Completes waits and step retries whose time has come. The sweeper and every checkpoint call it. */
+ /** Completes waits, step retries and callback timeouts whose time has come. The sweeper and every checkpoint call it. */
static boolean fireDueTimers(DurableExecution execution, long now) {
boolean changed = false;
for (DurableOperation operation : execution.getOperations().values()) {
@@ -71,11 +71,38 @@ final class DurableCheckpointApplier {
operation.setNextAttemptTimestamp(null);
operation.setChangeSequence(execution.nextChangeSequence());
changed = true;
+ } else if (operation.getType() == DurableOperationType.CALLBACK
+ && operation.getStatus() == DurableOperationStatus.STARTED) {
+ changed |= timeOutCallback(execution, operation, now);
}
}
return changed;
}
+ private static boolean timeOutCallback(DurableExecution execution, DurableOperation operation, long now) {
+ boolean timedOut = operation.getCallbackDeadline() != null && operation.getCallbackDeadline() <= now;
+ boolean heartbeatMissed = operation.getHeartbeatDeadline() != null && operation.getHeartbeatDeadline() <= now;
+ if (!timedOut && !heartbeatMissed) {
+ return false;
+ }
+ boolean heartbeatFirst = heartbeatMissed
+ && (!timedOut || operation.getHeartbeatDeadline() < operation.getCallbackDeadline());
+ DurableErrorObject error = heartbeatFirst
+ ? DurableErrorObject.of("Callback timed out on heartbeat", "Callback.Heartbeat")
+ : DurableErrorObject.of("Callback timed out", "Callback.Timeout");
+ operation.setStatus(DurableOperationStatus.TIMED_OUT);
+ operation.setError(error);
+ operation.setEndTimestamp(now);
+ operation.setCallbackDeadline(null);
+ operation.setHeartbeatDeadline(null);
+ operation.setChangeSequence(execution.nextChangeSequence());
+ // The history event names only the error type.
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("Error", DurableHistory.errorWrapper(DurableErrorObject.of(null, error.getErrorType())));
+ DurableHistory.operationEvent(execution, operation, "CallbackTimedOut", now, details);
+ return true;
+ }
+
private static void validateBatch(List<DurableOperationUpdate> updates) {
long executionUpdates = updates.stream().filter(update -> update.type() == DurableOperationType.EXECUTION).count();
if (executionUpdates > 1) {
@@ -93,7 +120,7 @@ final class DurableCheckpointApplier {
DurableOperationUpdate previous = seen.put(update.id(), update);
if (previous != null && !(previous.action() == DurableOperationAction.START
&& previous.type() == update.type() && closesInSameBatch(update))) {
- throw invalid("Cannot checkpoint multiple operations with the same ID.");
+ throw invalid("Cannot update the same operation twice in a single request.");
}
}
}
@@ -170,7 +197,8 @@ final class DurableCheckpointApplier {
case CONTEXT -> applyContext(update, existing);
case STEP -> applyStep(update, existing);
case WAIT -> applyWait(update, existing);
- case CALLBACK, CHAINED_INVOKE -> throw invalid(update.type() + " operations are not supported yet");
+ case CALLBACK -> applyCallback(update, existing);
+ case CHAINED_INVOKE -> throw invalid(update.type() + " operations are not supported yet");
default -> throw invalid("Unknown operation type.");
}
}
@@ -340,6 +368,32 @@ final class DurableCheckpointApplier {
}
}
+ /** Only the function starts a callback. SendDurableExecutionCallback* completes it. */
+ private void applyCallback(DurableOperationUpdate update, DurableOperation existing) {
+ if (update.action() != DurableOperationAction.START) {
+ throw invalid("Invalid action for the given operation type.");
+ }
+ if (existing != null) {
+ throw invalid("Cannot start a CALLBACK that already exist.");
+ }
+ DurableOperation operation = create(update);
+ operation.setCallbackId(DurableTokens.callbackId(execution.getExecutionArn(), update.id()));
+ Map<String, Object> details = new LinkedHashMap<>();
+ details.put("CallbackId", operation.getCallbackId());
+ Integer timeout = update.callbackTimeoutSeconds();
+ if (timeout != null && timeout > 0) {
+ operation.setCallbackDeadline(now + timeout * 1000L);
+ details.put("Timeout", timeout);
+ }
+ Integer heartbeat = update.callbackHeartbeatTimeoutSeconds();
+ if (heartbeat != null && heartbeat > 0) {
+ operation.setHeartbeatTimeoutSeconds(heartbeat);
+ operation.setHeartbeatDeadline(now + heartbeat * 1000L);
+ details.put("HeartbeatTimeout", heartbeat);
+ }
+ event(operation, "CallbackStarted", details);
+ }
+
private static void requireMatchingResult(DurableOperationUpdate update) {
if (update.action() == DurableOperationAction.SUCCEED && update.error() != null) {
throw invalid("Cannot provide an Error for SUCCEED action.");
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.java
index fb9cb0e408..29ac1b37c2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionController.java
@@ -28,6 +28,7 @@ import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneOffset;
@@ -53,6 +54,7 @@ public class DurableExecutionController {
/** Valid Statuses filters for a state Floci never puts an execution in. */
private static final Set<String> UNMODELLED_STATUSES = Set.of("PAUSED", "PAUSING", "DELETING");
+ private static final int MAX_CALLBACK_PAYLOAD_BYTES = 1024 * 1024;
private final DurableExecutionService service;
private final LambdaService lambdaService;
@@ -179,6 +181,48 @@ public class DurableExecutionController {
return Response.ok(response).build();
}
+ @POST
+ @Path("/durable-execution-callbacks/{callbackId: .+}/succeed")
+ public Response callbackSucceed(@Context HttpHeaders headers, @PathParam("callbackId") String callbackId,
+ byte[] body) {
+ if (body != null && MAX_CALLBACK_PAYLOAD_BYTES < body.length) {
+ throw new AwsException("ValidationException", "1 validation error detected: Value at 'result' failed to "
+ + "satisfy constraint: Member must have length less than or equal to " + MAX_CALLBACK_PAYLOAD_BYTES,
+ 400);
+ }
+ String result = body == null || body.length == 0 ? null : new String(body, StandardCharsets.UTF_8);
+ service.completeCallback(callbackId, regionResolver.getAccountId(), regionResolver.resolveRegion(headers), true,
+ result, null);
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
+ /**
+ * Without a body the callback fails with no Error, and AWS does not invoke the function for it.
+ * Any other body must be a JSON object.
+ */
+ @POST
+ @Path("/durable-execution-callbacks/{callbackId: .+}/fail")
+ public Response callbackFail(@Context HttpHeaders headers, @PathParam("callbackId") String callbackId,
+ String body) {
+ DurableErrorObject error = body == null || body.isEmpty() ? null : parseCallbackError(body);
+ // AWS measures the error as compact JSON, whatever spacing the request used.
+ if (error != null
+ && MAX_CALLBACK_PAYLOAD_BYTES < DurableCheckpointApplier.utf8Length(DurableWire.error(error).toString())) {
+ throw new AwsException("InvalidParameterValueException", "Error object size must be less than or equal to "
+ + MAX_CALLBACK_PAYLOAD_BYTES + " bytes.", 400);
+ }
+ service.completeCallback(callbackId, regionResolver.getAccountId(), regionResolver.resolveRegion(headers), false,
+ null, error);
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
+ @POST
+ @Path("/durable-execution-callbacks/{callbackId: .+}/heartbeat")
+ public Response callbackHeartbeat(@Context HttpHeaders headers, @PathParam("callbackId") String callbackId) {
+ service.heartbeatCallback(callbackId, regionResolver.getAccountId(), regionResolver.resolveRegion(headers));
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
/**
* An execution of another account is not found. One of another region reads as "Function not found",
* as AWS answers for an execution ARN of another region.
@@ -194,6 +238,20 @@ public class DurableExecutionController {
return arn;
}
+ /** AWS answers whitespace, null or broken JSON with a SerializationException that has no message. */
+ private DurableErrorObject parseCallbackError(String body) {
+ Map<String, Object> error;
+ try {
+ error = objectMapper.readValue(body, new TypeReference<Map<String, Object>>() {});
+ } catch (IOException e) {
+ error = null;
+ }
+ if (error == null) {
+ throw new AwsException("SerializationException", null, 400);
+ }
+ return DurableWire.parseError(error);
+ }
+
private Map<String, Object> readObject(String body) {
if (body == null || body.isBlank()) {
return Map.of();
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
index 2a7b6a7770..9a482866de 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionService.java
@@ -34,6 +34,7 @@ import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Comparator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -75,6 +76,8 @@ public class DurableExecutionService implements Resettable {
static final String INVALID_TOKEN = "Invalid checkpoint token";
static final String TOKEN_FOR_OTHER_EXECUTION = "Checkpoint token is not valid for the durable execution ARN";
static final String NOT_FOUND = "Durable Execution does not exist";
+ static final String INVALID_CALLBACK_ID = "Invalid callback id";
+ static final String CALLBACK_CLOSED = "The callback is either timed out or already completed";
private final AccountAwareStorageBackend<DurableExecution> store;
private final ObjectMapper objectMapper;
@@ -300,6 +303,101 @@ public class DurableExecutionService implements Resettable {
return execution;
}
+ /**
+ * SendDurableExecutionCallbackSuccess when {@code succeeded}, else SendDurableExecutionCallbackFailure.
+ * The callback's operation completes and the function is invoked again to see it, except for a
+ * failure sent with no body: AWS records it but does not invoke the function.
+ */
+ public void completeCallback(String callbackId, String accountId, String region, boolean succeeded, String result,
+ DurableErrorObject error) {
+ DurableTokens.CallbackId id = parseCallbackId(callbackId);
+ ArnParts arn = callbackArn(id, accountId, region);
+ List<Runnable> effects = new ArrayList<>();
+ synchronized (lockFor(arn)) {
+ long now = clock.millis();
+ DurableExecution execution = loadPendingCallbackExecution(arn, id, callbackId, now);
+ DurableOperation operation = execution.getOperations().get(id.operationId());
+ operation.setStatus(succeeded ? DurableOperationStatus.SUCCEEDED : DurableOperationStatus.FAILED);
+ operation.setResult(succeeded ? result : null);
+ operation.setError(succeeded ? null : error);
+ operation.setEndTimestamp(now);
+ operation.setCallbackDeadline(null);
+ operation.setHeartbeatDeadline(null);
+ operation.setChangeSequence(execution.nextChangeSequence());
+ Map<String, Object> details = new LinkedHashMap<>();
+ if (succeeded) {
+ details.put("Result", DurableHistory.payloadWrapper(result));
+ } else {
+ details.put("Error", DurableHistory.errorWrapper(error));
+ }
+ DurableHistory.operationEvent(execution, operation, succeeded ? "CallbackSucceeded" : "CallbackFailed",
+ now, details);
+ if (succeeded || error != null) {
+ trigger(execution, effects);
+ }
+ save(execution);
+ }
+ runEffects(effects);
+ }
+
+ /** SendDurableExecutionCallbackHeartbeat. It moves the heartbeat deadline when the callback has one. */
+ public void heartbeatCallback(String callbackId, String accountId, String region) {
+ DurableTokens.CallbackId id = parseCallbackId(callbackId);
+ ArnParts arn = callbackArn(id, accountId, region);
+ synchronized (lockFor(arn)) {
+ long now = clock.millis();
+ DurableExecution execution = loadPendingCallbackExecution(arn, id, callbackId, now);
+ DurableOperation operation = execution.getOperations().get(id.operationId());
+ if (operation.getHeartbeatTimeoutSeconds() != null) {
+ operation.setHeartbeatDeadline(now + operation.getHeartbeatTimeoutSeconds() * 1000L);
+ save(execution);
+ }
+ }
+ }
+
+ private static DurableTokens.CallbackId parseCallbackId(String callbackId) {
+ return DurableTokens.parseCallbackId(callbackId)
+ .orElseThrow(() -> new AwsException("InvalidParameterValueException", INVALID_CALLBACK_ID, 400));
+ }
+
+ /** A callback of another account or region is one this caller cannot complete. */
+ private static ArnParts callbackArn(DurableTokens.CallbackId id, String accountId, String region) {
+ ArnParts arn;
+ try {
+ arn = parseArn(id.executionArn());
+ } catch (AwsException e) {
+ throw new AwsException("InvalidParameterValueException", INVALID_CALLBACK_ID, 400);
+ }
+ if (!arn.accountId().equals(accountId) || !arn.region().equals(region)) {
+ throw callbackClosed();
+ }
+ return arn;
+ }
+
+ /** A callback past its deadline is closed even before the sweeper records the timeout. */
+ private DurableExecution loadPendingCallbackExecution(ArnParts arn, DurableTokens.CallbackId id, String callbackId,
+ long now) {
+ DurableExecution execution = store.getForAccount(arn.accountId(), arn.storeKey())
+ .filter(candidate -> candidate.getExecutionArn().equals(arn.arn()))
+ .orElseThrow(DurableExecutionService::callbackClosed);
+ DurableOperation operation = execution.getOperations().get(id.operationId());
+ if (execution.isClosed() || operation == null || operation.getType() != DurableOperationType.CALLBACK
+ || operation.getStatus() != DurableOperationStatus.STARTED
+ || !callbackId.equals(operation.getCallbackId())
+ || isPast(operation.getCallbackDeadline(), now) || isPast(operation.getHeartbeatDeadline(), now)) {
+ throw callbackClosed();
+ }
+ return execution;
+ }
+
+ private static boolean isPast(Long deadline, long now) {
+ return deadline != null && deadline <= now;
+ }
+
+ private static AwsException callbackClosed() {
+ return new AwsException("CallbackTimeoutException", CALLBACK_CLOSED, 400);
+ }
+
/** One tick of {@link DurableExecutionSweeper}. It fires due timers, times out and expires executions. */
public void sweep() {
long now = clock.millis();
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.java
index df4fda7a3e..eb135daa15 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableTokens.java
@@ -3,13 +3,18 @@ package io.github.hectorvent.floci.services.lambda.durable;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Optional;
+import java.util.UUID;
+import java.util.regex.Pattern;
/**
- * The opaque CheckpointToken of the durable execution protocol, standard base64 as AWS documents it.
+ * The opaque CheckpointToken and CallbackId of the durable execution protocol, standard base64 as
+ * AWS documents them. A CallbackId carries its execution ARN, which IAM authorizes against.
*/
-final class DurableTokens {
+public final class DurableTokens {
private static final String SEPARATOR = "|";
+ private static final Pattern CALLBACK_ID_PATTERN = Pattern.compile("[A-Za-z0-9+/]+={0,2}");
+ private static final int MAX_CALLBACK_ID_LENGTH = 1024;
private DurableTokens() {
}
@@ -17,6 +22,9 @@ final class DurableTokens {
record CheckpointToken(String executionArn, String invocationId, long sequence) {
}
+ record CallbackId(String executionArn, String operationId) {
+ }
+
static String checkpointToken(String executionArn, String invocationId, long sequence) {
return encode(executionArn + SEPARATOR + invocationId + SEPARATOR + sequence);
}
@@ -41,6 +49,28 @@ final class DurableTokens {
}
}
+ /** The nonce keeps an id from matching a later callback that reuses the operation id. */
+ static String callbackId(String executionArn, String operationId) {
+ return encode(executionArn + SEPARATOR + operationId + SEPARATOR + UUID.randomUUID());
+ }
+
+ static Optional<CallbackId> parseCallbackId(String callbackId) {
+ if (callbackId == null || callbackId.length() > MAX_CALLBACK_ID_LENGTH
+ || !CALLBACK_ID_PATTERN.matcher(callbackId).matches()) {
+ return Optional.empty();
+ }
+ String decoded = decode(callbackId);
+ String[] parts = decoded == null ? new String[0] : decoded.split("\\" + SEPARATOR, -1);
+ if (parts.length != 3 || parts[0].isEmpty() || parts[1].isEmpty()) {
+ return Optional.empty();
+ }
+ return Optional.of(new CallbackId(parts[0], parts[1]));
+ }
+
+ public static Optional<String> callbackExecutionArn(String callbackId) {
+ return parseCallbackId(callbackId).map(CallbackId::executionArn);
+ }
+
private static String encode(String value) {
return Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8));
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java
index e49b9218f1..20a1196bb0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/DurableWire.java
@@ -28,6 +28,7 @@ public final class DurableWire {
private static final JsonNodeFactory NODES = JsonNodeFactory.instance;
private static final long MAX_DELAY_SECONDS = 31_622_400L;
+ private static final long MAX_CALLBACK_TIMEOUT_SECONDS = 99_999_999L;
private DurableWire() {
}
@@ -118,6 +119,11 @@ public final class DurableWire {
putTimestamp(details, "ScheduledEndTimestamp", operation.getScheduledEndTimestamp(), millis);
}
}
+ case CALLBACK -> {
+ ObjectNode details = node.putObject("CallbackDetails");
+ putIfPresent(details, "CallbackId", operation.getCallbackId());
+ putResultAndError(details, operation);
+ }
default -> {
}
}
@@ -203,7 +209,7 @@ public final class DurableWire {
if (!(item instanceof Map<?, ?> map)) {
throw new AwsException("SerializationException", "Updates members must be JSON objects", 400);
}
- updates.add(parseUpdate(map));
+ updates.add(parseUpdate(map, "updates." + (updates.size() + 1) + ".member."));
}
return updates;
}
@@ -250,12 +256,14 @@ public final class DurableWire {
result != null && result.isTextual() ? result.asText() : null, error);
}
- private static DurableOperationUpdate parseUpdate(Map<?, ?> map) {
+ /** {@code path} prefixes the member in a ValidationException, as in "updates.1.member.waitOptions.waitSeconds". */
+ private static DurableOperationUpdate parseUpdate(Map<?, ?> map, String path) {
DurableOperationType type = enumMember(map, "Type", DurableOperationType.class);
DurableOperationAction action = enumMember(map, "Action", DurableOperationAction.class);
Map<?, ?> stepOptions = structure(map, "StepOptions");
Map<?, ?> waitOptions = structure(map, "WaitOptions");
Map<?, ?> contextOptions = structure(map, "ContextOptions");
+ Map<?, ?> callbackOptions = structure(map, "CallbackOptions");
return new DurableOperationUpdate(
string(map, "Id"),
string(map, "ParentId"),
@@ -265,9 +273,12 @@ public final class DurableWire {
action,
string(map, "Payload"),
parseError(map.get("Error")),
- stepOptions == null ? null : integer(stepOptions, "NextAttemptDelaySeconds", "StepOptions"),
- waitOptions == null ? null : integer(waitOptions, "WaitSeconds", "WaitOptions"),
- contextOptions == null ? null : bool(contextOptions, "ReplayChildren"));
+ integer(stepOptions, "NextAttemptDelaySeconds", path + "stepOptions", 1, MAX_DELAY_SECONDS),
+ integer(waitOptions, "WaitSeconds", path + "waitOptions", 1, MAX_DELAY_SECONDS),
+ contextOptions == null ? null : bool(contextOptions, "ReplayChildren"),
+ integer(callbackOptions, "TimeoutSeconds", path + "callbackOptions", 0, MAX_CALLBACK_TIMEOUT_SECONDS),
+ integer(callbackOptions, "HeartbeatTimeoutSeconds", path + "callbackOptions", 0,
+ MAX_CALLBACK_TIMEOUT_SECONDS));
}
private static <E extends Enum<E>> E enumMember(Map<?, ?> map, String member, Class<E> type) {
@@ -305,20 +316,19 @@ public final class DurableWire {
throw new AwsException("SerializationException", member + " must be a string", 400);
}
- private static Integer integer(Map<?, ?> map, String member, String structure) {
- Object value = map.get(member);
+ private static Integer integer(Map<?, ?> structure, String member, String structurePath, long min, long max) {
+ Object value = structure == null ? null : structure.get(member);
if (value == null) {
return null;
}
+ String field = structurePath + "." + Character.toLowerCase(member.charAt(0)) + member.substring(1);
if (!(value instanceof Number number && (number instanceof Byte || number instanceof Short
|| number instanceof Integer || number instanceof Long))) {
- throw new AwsException("SerializationException", structure + "." + member + " must be an integer", 400);
+ throw new AwsException("SerializationException", field + " must be an integer", 400);
}
long seconds = number.longValue();
- if (seconds < 1 || MAX_DELAY_SECONDS < seconds) {
- String field = Character.toLowerCase(structure.charAt(0)) + structure.substring(1) + "."
- + Character.toLowerCase(member.charAt(0)) + member.substring(1);
- String bound = seconds < 1 ? "greater than or equal to 1" : "less than or equal to " + MAX_DELAY_SECONDS;
+ if (seconds < min || max < seconds) {
+ String bound = seconds < min ? "greater than or equal to " + min : "less than or equal to " + max;
throw new AwsException("ValidationException", "1 validation error detected: Value '" + seconds
+ "' at '" + field + "' failed to satisfy constraint: Member must have value " + bound, 400);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
index 9ff7e2beb6..bdaa1fb5a8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableExecution.java
@@ -66,7 +66,7 @@ public class DurableExecution {
return changeSequence > seenSequence;
}
- /** True while a wait or a step retry is still pending. */
+ /** True while a wait, a step retry or a callback is still pending. */
@JsonIgnore
public boolean hasPendingOperations() {
for (DurableOperation operation : operations.values()) {
@@ -83,6 +83,11 @@ public class DurableExecution {
return true;
}
}
+ case CALLBACK -> {
+ if (operationStatus == DurableOperationStatus.STARTED) {
+ return true;
+ }
+ }
default -> {
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java
index 1ff6aa8f5c..827f9e976e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperation.java
@@ -29,6 +29,11 @@ public class DurableOperation {
private Integer attempt;
private Long nextAttemptTimestamp;
private Long scheduledEndTimestamp;
+ private String callbackId;
+ private Integer heartbeatTimeoutSeconds;
+ private Long callbackDeadline;
+ /** Moves forward on every SendDurableExecutionCallbackHeartbeat. */
+ private Long heartbeatDeadline;
/** Position in the execution's change log. The handler is only shown operations newer than it has seen. */
private long changeSequence;
@@ -52,6 +57,10 @@ public class DurableOperation {
copy.attempt = attempt;
copy.nextAttemptTimestamp = nextAttemptTimestamp;
copy.scheduledEndTimestamp = scheduledEndTimestamp;
+ copy.callbackId = callbackId;
+ copy.heartbeatTimeoutSeconds = heartbeatTimeoutSeconds;
+ copy.callbackDeadline = callbackDeadline;
+ copy.heartbeatDeadline = heartbeatDeadline;
copy.changeSequence = changeSequence;
return copy;
}
@@ -101,6 +110,20 @@ public class DurableOperation {
public Long getScheduledEndTimestamp() { return scheduledEndTimestamp; }
public void setScheduledEndTimestamp(Long scheduledEndTimestamp) { this.scheduledEndTimestamp = scheduledEndTimestamp; }
+ public String getCallbackId() { return callbackId; }
+ public void setCallbackId(String callbackId) { this.callbackId = callbackId; }
+
+ public Integer getHeartbeatTimeoutSeconds() { return heartbeatTimeoutSeconds; }
+ public void setHeartbeatTimeoutSeconds(Integer heartbeatTimeoutSeconds) {
+ this.heartbeatTimeoutSeconds = heartbeatTimeoutSeconds;
+ }
+
+ public Long getCallbackDeadline() { return callbackDeadline; }
+ public void setCallbackDeadline(Long callbackDeadline) { this.callbackDeadline = callbackDeadline; }
+
+ public Long getHeartbeatDeadline() { return heartbeatDeadline; }
+ public void setHeartbeatDeadline(Long heartbeatDeadline) { this.heartbeatDeadline = heartbeatDeadline; }
+
public long getChangeSequence() { return changeSequence; }
public void setChangeSequence(long changeSequence) { this.changeSequence = changeSequence; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java
index ca9b275f4e..e43c4f1860 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/durable/model/DurableOperationUpdate.java
@@ -14,5 +14,7 @@ public record DurableOperationUpdate(String id,
DurableErrorObject error,
Integer nextAttemptDelaySeconds,
Integer waitSeconds,
- Boolean replayChildren) {
+ Boolean replayChildren,
+ Integer callbackTimeoutSeconds,
+ Integer callbackHeartbeatTimeoutSeconds) {
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java
index d9f8926544..94388e26ea 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iam/IamActionRegistryTest.java
@@ -222,6 +222,10 @@ class IamActionRegistryTest {
assertEquals("lambda:GetDurableExecution", lambdaAction("GET", base));
assertEquals("lambda:ListDurableExecutionsByFunction",
lambdaAction("GET", "/2025-12-01/functions/f/durable-executions"));
+ String callback = "/2025-12-01/durable-execution-callbacks/QUJD%2BREVG%2FR0g%3D";
+ assertEquals("lambda:SendDurableExecutionCallbackSuccess", lambdaAction("POST", callback + "/succeed"));
+ assertEquals("lambda:SendDurableExecutionCallbackFailure", lambdaAction("POST", callback + "/fail"));
+ assertEquals("lambda:SendDurableExecutionCallbackHeartbeat", lambdaAction("POST", callback + "/heartbeat"));
}
private String lambdaAction(String method, String path) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java
index 42642f04d3..83888395b7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java
@@ -15,6 +15,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -605,6 +606,17 @@ class ResourceArnBuilderTest {
arn);
}
+ @Test
+ void lambdaDurableCallbackRoutesNameTheExecutionInTheCallbackId() {
+ String executionArn = "arn:aws:lambda:us-east-1:000000000000:function:my-function:1/durable-execution/run-1/0b1c";
+ String callbackId = Base64.getEncoder().encodeToString((executionArn + "|c1|nonce").getBytes(StandardCharsets.UTF_8));
+ when(uriInfo.getPath()).thenReturn("/2025-12-01/durable-execution-callbacks/" + callbackId + "/succeed");
+ assertEquals(executionArn, builder.build("lambda", ctx, "us-east-1", "000000000000"));
+
+ when(uriInfo.getPath()).thenReturn("/2025-12-01/durable-execution-callbacks/not-a-callback/heartbeat");
+ assertEquals("*", builder.build("lambda", ctx, "us-east-1", "000000000000"));
+ }
+
@Test
void lambdaDurableExecutionGetNamesTheExecution() {
when(uriInfo.getPath()).thenReturn("/2025-12-01/durable-executions/arn:aws:lambda:us-east-1:000000000000:"
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
index 65963c5b9d..400c80d89a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableCheckpointApplierTest.java
@@ -20,6 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
class DurableCheckpointApplierTest {
private static final long NOW = 1_700_000_000_000L;
+ private static final String EXECUTION_ARN =
+ "arn:aws:lambda:us-east-1:000000000000:function:fn:1/durable-execution/exec/exec-id";
@Test
void executionUpdateMustBeTheOnlyOneAndTheLast() {
@@ -31,10 +33,10 @@ class DurableCheckpointApplierTest {
void anExecutionCloseCarriesOnlyItsOwnResultKind() {
DurableErrorObject error = DurableErrorObject.of("m", "T");
assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,
- DurableOperationAction.SUCCEED, "\"p\"", error, null, null, null)),
+ DurableOperationAction.SUCCEED, "\"p\"", error, null, null, null, null, null)),
"Cannot provide an Error for SUCCEED action.");
assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,
- DurableOperationAction.FAIL, "\"p\"", error, null, null, null)),
+ DurableOperationAction.FAIL, "\"p\"", error, null, null, null, null, null)),
"Cannot provide a Payload for FAIL action.");
}
@@ -42,15 +44,16 @@ class DurableCheckpointApplierTest {
void anExecutionErrorHasTheSameSizeLimitAsAnOperationError() {
DurableErrorObject large = DurableErrorObject.of("x".repeat(DurableCheckpointApplier.MAX_ERROR_BYTES), "T");
assertRejected(List.of(new DurableOperationUpdate("result", null, null, DurableOperationType.EXECUTION, null,
- DurableOperationAction.FAIL, null, large, null, null, null)),
+ DurableOperationAction.FAIL, null, large, null, null, null, null, null)),
"Error object size must be less than 32768 bytes.");
}
@Test
void duplicateIdsAreRejectedUnlessAStartIsClosedInTheSameBatch() {
assertRejected(List.of(stepStart("s1", null), stepStart("s1", null)),
- "Cannot checkpoint multiple operations with the same ID.");
- assertRejected(List.of(waitStart("w1"), waitStart("w1")), "Cannot checkpoint multiple operations with the same ID.");
+ "Cannot update the same operation twice in a single request.");
+ assertRejected(List.of(waitStart("w1"), waitStart("w1")),
+ "Cannot update the same operation twice in a single request.");
DurableExecution execution = execution();
DurableCheckpointApplier.apply(execution, List.of(stepStart("s1", null), update("s1", null,
DurableOperationType.STEP, DurableOperationAction.SUCCEED, "1")), NOW);
@@ -74,7 +77,7 @@ class DurableCheckpointApplierTest {
assertEquals("Invalid current STEP state to start.", reject(execution, List.of(stepStart("s1", null))));
assertEquals("Cannot provide an Error for SUCCEED action.", reject(execution, List.of(
new DurableOperationUpdate("s1", null, null, DurableOperationType.STEP, null,
- DurableOperationAction.SUCCEED, null, DurableErrorObject.of("x", "y"), null, null, null))));
+ DurableOperationAction.SUCCEED, null, DurableErrorObject.of("x", "y"), null, null, null, null, null))));
assertEquals("Invalid StepOptions for the given action.", reject(execution, List.of(
update("s1", null, DurableOperationType.STEP, DurableOperationAction.RETRY, null))));
DurableCheckpointApplier.apply(execution, List.of(update("s1", null, DurableOperationType.STEP,
@@ -98,22 +101,68 @@ class DurableCheckpointApplierTest {
}
@Test
- void delaysOutsideTheirRangeAreValidationErrors() {
- AwsException rejected = assertThrows(AwsException.class, () -> DurableWire.parseUpdates(
+ void optionsOutsideTheirRangeAreValidationErrorsNamingTheUpdate() {
+ AwsException wait = assertThrows(AwsException.class, () -> DurableWire.parseUpdates(
List.of(Map.of("Id", "w1", "Type", "WAIT", "Action", "START", "WaitOptions", Map.of("WaitSeconds", 0)))));
- assertEquals("ValidationException", rejected.getErrorCode());
- assertEquals("1 validation error detected: Value '0' at 'waitOptions.waitSeconds' failed to satisfy "
- + "constraint: Member must have value greater than or equal to 1", rejected.getMessage());
+ assertEquals("ValidationException", wait.getErrorCode());
+ assertEquals("1 validation error detected: Value '0' at 'updates.1.member.waitOptions.waitSeconds' failed to "
+ + "satisfy constraint: Member must have value greater than or equal to 1", wait.getMessage());
+
+ AwsException callback = assertThrows(AwsException.class, () -> DurableWire.parseUpdates(List.of(
+ Map.of("Id", "c1", "Type", "CALLBACK", "Action", "START"),
+ Map.of("Id", "c2", "Type", "CALLBACK", "Action", "START",
+ "CallbackOptions", Map.of("HeartbeatTimeoutSeconds", 100_000_000)))));
+ assertEquals("1 validation error detected: Value '100000000' at "
+ + "'updates.2.member.callbackOptions.heartbeatTimeoutSeconds' failed to satisfy constraint: Member "
+ + "must have value less than or equal to 99999999", callback.getMessage());
}
@Test
- void callbacksAndChainedInvokesAreNotSupportedYet() {
- assertRejected(List.of(update("c1", null, DurableOperationType.CALLBACK, DurableOperationAction.START, null)),
- "CALLBACK operations are not supported yet");
+ void chainedInvokesAreNotSupportedYet() {
assertRejected(List.of(update("i1", null, DurableOperationType.CHAINED_INVOKE, DurableOperationAction.START,
null)), "CHAINED_INVOKE operations are not supported yet");
}
+ @Test
+ void aCallbackStartsWithAnIdThatNamesItsExecution() {
+ DurableExecution execution = execution();
+ DurableCheckpointApplier.apply(execution, List.of(callbackStart("c1", 30, 10)), NOW);
+
+ DurableOperation callback = execution.getOperations().get("c1");
+ assertEquals(DurableOperationStatus.STARTED, callback.getStatus());
+ assertEquals(EXECUTION_ARN, DurableTokens.callbackExecutionArn(callback.getCallbackId()).orElseThrow());
+ assertEquals(NOW + 30_000, callback.getCallbackDeadline());
+ assertEquals(NOW + 10_000, callback.getHeartbeatDeadline());
+ assertEquals(Map.of("CallbackStartedDetails", Map.of("CallbackId", callback.getCallbackId(), "Timeout", 30,
+ "HeartbeatTimeout", 10)), execution.getHistory().get(0).getDetails());
+
+ assertEquals("Cannot start a CALLBACK that already exist.", reject(execution, List.of(callbackStart("c1", 0, 0))));
+ assertEquals("Invalid action for the given operation type.", reject(execution, List.of(
+ update("c1", null, DurableOperationType.CALLBACK, DurableOperationAction.SUCCEED, "\"r\""))));
+ }
+
+ @Test
+ void aCallbackTimesOutOnTheDeadlineItMissesFirst() {
+ DurableExecution execution = execution();
+ DurableCheckpointApplier.apply(execution, List.of(callbackStart("slow", 5, 0), callbackStart("silent", 60, 3)),
+ NOW);
+
+ DurableCheckpointApplier.fireDueTimers(execution, NOW + 3_000);
+ DurableOperation silent = execution.getOperations().get("silent");
+ assertEquals(DurableOperationStatus.TIMED_OUT, silent.getStatus());
+ assertEquals("Callback.Heartbeat", silent.getError().getErrorType());
+ assertEquals("Callback timed out on heartbeat", silent.getError().getErrorMessage());
+ assertEquals(DurableOperationStatus.STARTED, execution.getOperations().get("slow").getStatus());
+
+ DurableCheckpointApplier.fireDueTimers(execution, NOW + 5_000);
+ DurableOperation slow = execution.getOperations().get("slow");
+ assertEquals("Callback.Timeout", slow.getError().getErrorType());
+ assertEquals("Callback timed out", slow.getError().getErrorMessage());
+ assertEquals(Map.of("CallbackTimedOutDetails", Map.of("Error", Map.of("Payload",
+ Map.of("ErrorType", "Callback.Timeout"), "Truncated", false))),
+ execution.getHistory().get(execution.getHistory().size() - 1).getDetails());
+ }
+
@Test
void aRejectedBatchLeavesTheExecutionUntouched() {
DurableExecution execution = execution();
@@ -140,6 +189,7 @@ class DurableCheckpointApplierTest {
private static DurableExecution execution() {
DurableExecution execution = new DurableExecution();
execution.setExecutionId("exec-id");
+ execution.setExecutionArn(EXECUTION_ARN);
execution.setName("exec");
execution.setMaxResultBytes(DurableExecutionService.ASYNC_PAYLOAD_LIMIT);
DurableOperation root = new DurableOperation();
@@ -158,7 +208,12 @@ class DurableCheckpointApplierTest {
private static DurableOperationUpdate waitStart(String id) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.WAIT, null, DurableOperationAction.START,
- null, null, null, 5, null);
+ null, null, null, 5, null, null, null);
+ }
+
+ private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds, int heartbeatSeconds) {
+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",
+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds);
}
private static DurableOperationUpdate executionSucceed() {
@@ -167,6 +222,7 @@ class DurableCheckpointApplierTest {
private static DurableOperationUpdate update(String id, String parentId, DurableOperationType type,
DurableOperationAction action, String payload) {
- return new DurableOperationUpdate(id, parentId, null, type, null, action, payload, null, null, null, null);
+ return new DurableOperationUpdate(id, parentId, null, type, null, action, payload, null, null, null, null, null,
+ null);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
index b967e1ff07..45642c4f6b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServicePersistenceTest.java
@@ -93,6 +93,43 @@ class DurableExecutionServicePersistenceTest {
execution.getHistory().stream().map(DurableHistoryEvent::getEventType).toList());
}
+ @Test
+ void aPendingCallbackKeepsItsIdAndDeadlineAcrossARestart() {
+ ScriptedInvoker before = new ScriptedInvoker();
+ PersistentStorageFactory firstStorage = new PersistentStorageFactory(directory);
+ DurableExecutionService first = newService(firstStorage, before);
+ before.script((service, event) -> {
+ service.checkpoint(arn(event), token(event), null, List.of(callbackStart("approve", 0),
+ callbackStart("expire", 5)));
+ return "{\"Status\":\"PENDING\"}";
+ });
+ String executionArn = first.start(new StartRequest(ACCOUNT, "us-east-1", "durable-fn", "1", "exec-1", "{}",
+ false)).getExecutionArn();
+ String callbackId = first.get(executionArn).getOperations().get("approve").getCallbackId();
+ firstStorage.flushAll();
+
+ ScriptedInvoker after = new ScriptedInvoker();
+ DurableExecutionService restarted = newService(new PersistentStorageFactory(directory), after);
+ after.script((service, event) -> "{\"Status\":\"PENDING\"}");
+ after.script((service, event) -> {
+ assertEquals("SUCCEEDED", operation(event, "approve").get("Status").asText());
+ assertEquals("STARTED", operation(event, "expire").get("Status").asText());
+ return "{\"Status\":\"PENDING\"}";
+ });
+ after.script((service, event) -> {
+ assertEquals("Callback.Timeout", operation(event, "expire").at("/CallbackDetails/Error/ErrorType").asText());
+ return "{\"Status\":\"SUCCEEDED\",\"Result\":\"\\\"done\\\"\"}";
+ });
+ restarted.recoverAfterRestart();
+
+ restarted.completeCallback(callbackId, ACCOUNT, "us-east-1", true, "\"yes\"", null);
+ clock.advance(Duration.ofSeconds(5));
+ restarted.sweep();
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, restarted.get(executionArn).getStatus());
+ assertEquals(3, after.events.size());
+ }
+
private DurableExecutionService newService(StorageFactory storage, ScriptedInvoker invoker) {
DurableExecutionService service = new DurableExecutionService(storage, MAPPER, clock, invoker, Runnable::run,
Duration.ZERO);
@@ -102,7 +139,12 @@ class DurableExecutionServicePersistenceTest {
private static DurableOperationUpdate update(String id, DurableOperationType type, DurableOperationAction action,
String payload, Integer waitSeconds) {
- return new DurableOperationUpdate(id, null, null, type, null, action, payload, null, null, waitSeconds, null);
+ return new DurableOperationUpdate(id, null, null, type, null, action, payload, null, null, waitSeconds, null, null, null);
+ }
+
+ private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds) {
+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",
+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, null);
}
private static String arn(JsonNode event) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
index 10ef6e45a7..6bdd5ced49 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/DurableExecutionServiceTest.java
@@ -19,6 +19,7 @@ import io.github.hectorvent.floci.services.lambda.durable.model.DurableOperation
import io.github.hectorvent.floci.testing.MutableClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.function.Executable;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -513,8 +514,130 @@ class DurableExecutionServiceTest {
assertEquals(2, invoker.events.size(), "no extra invocation after the retry ran");
}
+ @Test
+ void aCallbackCompletedFromOutsideReinvokesTheFunctionWithItsResult() {
+ invoker.script(event -> {
+ CheckpointResult started = checkpoint(event, token(event), List.of(callbackStart("c1", 0, 0)));
+ assertNotNull(started.newExecutionState().get(0).getCallbackId());
+ return pending();
+ });
+ invoker.script(event -> {
+ JsonNode callback = operation(event, "c1");
+ assertEquals("SUCCEEDED", callback.get("Status").asText());
+ assertEquals("\"approved\"", callback.at("/CallbackDetails/Result").asText());
+ assertEquals("c1", event.get("UpdatedOperationIds").get(0).asText());
+ return succeeded("\"done\"");
+ });
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ String callbackId = service.get(arn).getOperations().get("c1").getCallbackId();
+
+ service.completeCallback(callbackId, ACCOUNT, REGION, true, "\"approved\"", null);
+
+ DurableExecution execution = service.get(arn);
+ assertEquals(DurableExecutionStatus.SUCCEEDED, execution.getStatus());
+ assertEquals(List.of("ExecutionStarted", "CallbackStarted", "InvocationCompleted", "CallbackSucceeded",
+ "InvocationCompleted", "ExecutionSucceeded"), eventTypes(execution));
+ assertCallbackClosed(() -> service.completeCallback(callbackId, ACCOUNT, REGION, false, null, null));
+ assertCallbackClosed(() -> service.heartbeatCallback(callbackId, ACCOUNT, REGION));
+ }
+
+ @Test
+ void aFailedCallbackCarriesTheErrorItWasSent() {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(callbackStart("c1", 0, 0)));
+ return pending();
+ });
+ invoker.script(event -> {
+ JsonNode callback = operation(event, "c1");
+ assertEquals("FAILED", callback.get("Status").asText());
+ assertEquals("Rejected", callback.at("/CallbackDetails/Error/ErrorType").asText());
+ return succeeded("\"handled\"");
+ });
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+
+ service.completeCallback(service.get(arn).getOperations().get("c1").getCallbackId(), ACCOUNT, REGION, false,
+ null, DurableErrorObject.of("denied", "Rejected"));
+
+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(arn).getStatus());
+ }
+
+ @Test
+ void aFailureWithoutAnErrorIsRecordedButDoesNotInvokeTheFunction() {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(callbackStart("c1", 0, 0)));
+ return pending();
+ });
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ String callbackId = service.get(arn).getOperations().get("c1").getCallbackId();
+
+ service.completeCallback(callbackId, ACCOUNT, REGION, false, null, null);
+
+ DurableExecution execution = service.get(arn);
+ assertEquals(1, invoker.events.size());
+ assertEquals(DurableExecutionStatus.RUNNING, execution.getStatus());
+ assertEquals(DurableOperationStatus.FAILED, execution.getOperations().get("c1").getStatus());
+ assertEquals("CallbackFailed", eventTypes(execution).get(eventTypes(execution).size() - 1));
+ assertCallbackClosed(() -> service.completeCallback(callbackId, ACCOUNT, REGION, true, "\"x\"", null));
+ }
+
+ @Test
+ void heartbeatsKeepACallbackAliveUntilTheyStop() {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(callbackStart("c1", 0, 3)));
+ return pending();
+ });
+ invoker.script(event -> {
+ assertEquals("Callback.Heartbeat", operation(event, "c1").at("/CallbackDetails/Error/ErrorType").asText());
+ return succeeded("\"gave up\"");
+ });
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ String callbackId = service.get(arn).getOperations().get("c1").getCallbackId();
+
+ clock.advance(Duration.ofSeconds(2));
+ service.heartbeatCallback(callbackId, ACCOUNT, REGION);
+ clock.advance(Duration.ofSeconds(2));
+ service.sweep();
+ assertEquals(DurableOperationStatus.STARTED, service.get(arn).getOperations().get("c1").getStatus());
+
+ clock.advance(Duration.ofSeconds(1));
+ assertCallbackClosed(() -> service.heartbeatCallback(callbackId, ACCOUNT, REGION));
+ service.sweep();
+
+ assertEquals(DurableOperationStatus.TIMED_OUT, service.get(arn).getOperations().get("c1").getStatus());
+ assertEquals(DurableExecutionStatus.SUCCEEDED, service.get(arn).getStatus());
+ }
+
+ @Test
+ void callbackIdsThatMatchNoOpenCallbackAreRejected() {
+ invoker.script(event -> {
+ checkpoint(event, token(event), List.of(callbackStart("c1", 0, 0)));
+ return pending();
+ });
+ String arn = start("exec-1", "{}", false).getExecutionArn();
+ String callbackId = service.get(arn).getOperations().get("c1").getCallbackId();
+
+ for (String malformed : List.of("QUJD", "not-valid!")) {
+ AwsException rejected = assertThrows(AwsException.class,
+ () -> service.heartbeatCallback(malformed, ACCOUNT, REGION));
+ assertEquals("InvalidParameterValueException", rejected.getErrorCode());
+ assertEquals("Invalid callback id", rejected.getMessage());
+ }
+ assertCallbackClosed(() -> service.heartbeatCallback(callbackId, ACCOUNT, "eu-west-1"));
+ assertCallbackClosed(() -> service.heartbeatCallback(callbackId, "111111111111", REGION));
+
+ service.stop(arn, null);
+ assertCallbackClosed(() -> service.completeCallback(callbackId, ACCOUNT, REGION, true, "\"late\"", null));
+ }
+
// ──────────────────────────── helpers ────────────────────────────
+ private static void assertCallbackClosed(Executable call) {
+ AwsException rejected = assertThrows(AwsException.class, call);
+ assertEquals("CallbackTimeoutException", rejected.getErrorCode());
+ assertEquals("The callback is either timed out or already completed", rejected.getMessage());
+ assertEquals(400, rejected.getHttpStatus());
+ }
+
private static DurableExecutionService newService(InMemoryStorageFactory storage, ScriptedInvoker invoker,
MutableClock clock) {
return new DurableExecutionService(storage, MAPPER, clock, invoker, Runnable::run, Duration.ZERO);
@@ -553,22 +676,27 @@ class DurableExecutionServiceTest {
private static DurableOperationUpdate step(String id, DurableOperationAction action, String payload,
DurableErrorObject error) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.STEP, "Step", action, payload, error,
- null, null, null);
+ null, null, null, null, null);
}
private static DurableOperationUpdate stepRetry(String id, int delaySeconds, DurableErrorObject error) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.STEP, "Step",
- DurableOperationAction.RETRY, null, error, delaySeconds, null, null);
+ DurableOperationAction.RETRY, null, error, delaySeconds, null, null, null, null);
+ }
+
+ private static DurableOperationUpdate callbackStart(String id, int timeoutSeconds, int heartbeatSeconds) {
+ return new DurableOperationUpdate(id, null, null, DurableOperationType.CALLBACK, "Callback",
+ DurableOperationAction.START, null, null, null, null, null, timeoutSeconds, heartbeatSeconds);
}
private static DurableOperationUpdate waitStart(String id, int seconds) {
return new DurableOperationUpdate(id, null, null, DurableOperationType.WAIT, "Wait",
- DurableOperationAction.START, null, null, null, seconds, null);
+ DurableOperationAction.START, null, null, null, seconds, null, null, null);
}
private static DurableOperationUpdate executionSucceed(String payload) {
return new DurableOperationUpdate("execution-result", null, null, DurableOperationType.EXECUTION, null,
- DurableOperationAction.SUCCEED, payload, null, null, null, null);
+ DurableOperationAction.SUCCEED, payload, null, null, null, null, null, null);
}
private static DurableFunctionInvoker.DurableInvocationResult succeeded(String result) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java
index 4ab96caaea..cc0a3ab4fa 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/durable/LambdaDurableProtocolIntegrationTest.java
@@ -312,6 +312,135 @@ class LambdaDurableProtocolIntegrationTest {
.body("Events[0].EventType", equalTo("ExecutionStopped"));
}
+ @Test
+ void aCallbackCompletedOverTheApiResumesTheExecution() throws Exception {
+ Response accepted = given()
+ .header("X-Amz-Invocation-Type", "Event")
+ .header("X-Amz-Durable-Execution-Name", "callback-1")
+ .body("{}")
+ .post(LAMBDA + "/functions/" + FUNCTION + ":$LATEST/invocations");
+ String arn = accepted.getHeader("X-Amz-Durable-Execution-Arn");
+
+ Invocation first = invoker.awaitInvocation(Duration.ofSeconds(30));
+ JsonNode event = MAPPER.readTree(first.payload());
+ Response started = given()
+ .urlEncodingEnabled(false)
+ .contentType("application/json")
+ .body("""
+ {
+ "CheckpointToken": "%s",
+ "Updates": [{"Id": "c1", "Name": "approval", "Type": "CALLBACK", "SubType": "Callback",
+ "Action": "START",
+ "CallbackOptions": {"TimeoutSeconds": 300, "HeartbeatTimeoutSeconds": 120}}]
+ }
+ """.formatted(event.get("CheckpointToken").asText()))
+ .post(DURABLE + "/durable-executions/" + arn + "/checkpoint");
+ started.then()
+ .statusCode(200)
+ .body("NewExecutionState.Operations[0].Status", equalTo("STARTED"))
+ .body("NewExecutionState.Operations[0].CallbackDetails.CallbackId", notNullValue());
+ String callbackId = started.jsonPath().getString("NewExecutionState.Operations[0].CallbackDetails.CallbackId");
+ first.response().complete(handlerResponse("{\"Status\": \"PENDING\"}"));
+
+ // The SDKs percent-encode the base64 id. The CLI and curl send it raw.
+ String encodedId = callbackId.replace("+", "%2B").replace("/", "%2F").replace("=", "%3D");
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .post(DURABLE + "/durable-execution-callbacks/" + callbackId + "/heartbeat")
+ .then()
+ .statusCode(200)
+ .body(equalTo("{}"));
+
+ given()
+ .urlEncodingEnabled(false)
+ .contentType("application/octet-stream")
+ .body(new byte[1024 * 1024 + 1])
+ .when()
+ .post(DURABLE + "/durable-execution-callbacks/" + encodedId + "/succeed")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("1 validation error detected: Value at 'result' failed to satisfy constraint: "
+ + "Member must have length less than or equal to 1048576"));
+
+ for (String notAnObject : List.of(" ", "null", "{")) {
+ given()
+ .urlEncodingEnabled(false)
+ .contentType("application/json")
+ .body(notAnObject)
+ .when()
+ .post(DURABLE + "/durable-execution-callbacks/" + encodedId + "/fail")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("SerializationException"));
+ }
+
+ String oversizedError = "{\"ErrorType\":\"Big\",\"ErrorMessage\":\"" + "m".repeat(1024 * 1024) + "\"}";
+ given()
+ .urlEncodingEnabled(false)
+ .contentType("application/json")
+ .body(oversizedError)
+ .when()
+ .post(DURABLE + "/durable-execution-callbacks/" + encodedId + "/fail")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterValueException"))
+ .body("message", equalTo("Error object size must be less than or equal to 1048576 bytes."));
+
+ given()
+ .urlEncodingEnabled(false)
+ .contentType("application/octet-stream")
+ .body("{\"approved\":true}".getBytes(StandardCharsets.UTF_8))
+ .when()
+ .post(DURABLE + "/durable-execution-callbacks/" + encodedId + "/succeed")
+ .then()
+ .statusCode(200)
+ .body(equalTo("{}"));
+
+ Invocation second = invoker.awaitInvocation(Duration.ofSeconds(30));
+ JsonNode resumed = MAPPER.readTree(second.payload());
+ JsonNode callback = resumed.at("/InitialExecutionState/Operations/1");
+ assertEquals("SUCCEEDED", callback.get("Status").asText());
+ assertEquals(callbackId, callback.at("/CallbackDetails/CallbackId").asText());
+ assertEquals("{\"approved\":true}", callback.at("/CallbackDetails/Result").asText());
+ second.response().complete(handlerResponse("{\"Status\": \"SUCCEEDED\", \"Result\": \"\\\"ok\\\"\"}"));
+
+ given()
+ .urlEncodingEnabled(false)
+ .contentType("application/json")
+ .body("{\"ErrorMessage\": \"too late\"}")
+ .when()
+ .post(DURABLE + "/durable-execution-callbacks/" + encodedId + "/fail")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("CallbackTimeoutException"))
+ .body("message", equalTo("The callback is either timed out or already completed"));
+
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .post(DURABLE + "/durable-execution-callbacks/not-valid!/heartbeat")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterValueException"))
+ .body("message", equalTo("Invalid callback id"));
+
+ given()
+ .urlEncodingEnabled(false)
+ .when()
+ .get(DURABLE + "/durable-executions/" + arn + "/history?IncludeExecutionData=true")
+ .then()
+ .statusCode(200)
+ .body("Events.EventType", equalTo(List.of("ExecutionStarted", "CallbackStarted", "InvocationCompleted",
+ "CallbackSucceeded", "InvocationCompleted", "ExecutionSucceeded")))
+ .body("Events[1].Name", equalTo("approval"))
+ .body("Events[1].CallbackStartedDetails.CallbackId", equalTo(callbackId))
+ .body("Events[1].CallbackStartedDetails.Timeout", equalTo(300))
+ .body("Events[1].CallbackStartedDetails.HeartbeatTimeout", equalTo(120))
+ .body("Events[3].CallbackSucceededDetails.Result.Payload", equalTo("{\"approved\":true}"));
+ }
+
private static DurableInvocationResult handlerResponse(String json) {
return new DurableInvocationResult("req-" + System.nanoTime(), json.getBytes(StandardCharsets.UTF_8), null);
}