Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): enforce handler deadline at dispatch (#3177)

BingEdward438ae619fd1c
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
index aa6a5aab40..4ab412af05 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
@@ -13,6 +13,7 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.util.UUID;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -27,8 +28,8 @@ import java.util.concurrent.TimeoutException;
public class LambdaExecutorService {
private static final Logger LOG = Logger.getLogger(LambdaExecutorService.class);
- /** Grace period beyond the configured function timeout to allow the runtime to report back. */
- private static final int TIMEOUT_GRACE_SECONDS = 2;
+ /** Extra time for a newly started runtime to request its first invocation. */
+ private static final int RUNTIME_DISPATCH_GRACE_SECONDS = 2;
private final WarmPool warmPool;
private final ObjectMapper objectMapper;
@@ -99,26 +100,31 @@ public class LambdaExecutorService {
handle.getRuntimeApiServer().enqueue(invocation);
- InvokeResult result = invocation.getResultFuture()
- .get(fn.getTimeout() + TIMEOUT_GRACE_SECONDS, TimeUnit.SECONDS);
+ java.util.concurrent.CompletableFuture.anyOf(
+ invocation.getDispatchedFuture(), invocation.getResultFuture())
+ .get(fn.getTimeout() + RUNTIME_DISPATCH_GRACE_SECONDS, TimeUnit.SECONDS);
+ InvokeResult result = invocation.getResultFuture().get();
warmPool.release(handle);
return result;
- } catch (TimeoutException e) {
- LOG.warnv("Function {0} timed out after {1}s", fn.getFunctionName(), fn.getTimeout());
- warmPool.destroyHandle(handle);
- return new InvokeResult(200, "Unhandled",
- buildErrorPayload("Task timed out after " + fn.getTimeout() + " seconds", "Function.TimedOut"),
- null, requestId);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
warmPool.destroyHandle(handle);
return new InvokeResult(200, "Unhandled", buildErrorPayload("Invocation interrupted", "Interrupted"), null, requestId);
} catch (Exception e) {
- LOG.warnv("Invocation error for function {0}: {1}", fn.getFunctionName(), e.getMessage());
+ Throwable cause = e instanceof ExecutionException && e.getCause() != null ? e.getCause() : e;
+ if (cause instanceof TimeoutException) {
+ LOG.warnv("Function {0} timed out after {1}s", fn.getFunctionName(), fn.getTimeout());
+ warmPool.destroyHandle(handle);
+ return new InvokeResult(200, "Unhandled",
+ buildErrorPayload("Task timed out after " + fn.getTimeout() + " seconds", "Function.TimedOut"),
+ null, requestId);
+ }
+ LOG.warnv("Invocation error for function {0}: {1}", fn.getFunctionName(), cause.getMessage());
warmPool.destroyHandle(handle);
- return new InvokeResult(200, "Unhandled", buildErrorPayload(e.getMessage(), "InvocationError"), null, requestId);
+ return new InvokeResult(200, "Unhandled",
+ buildErrorPayload(cause.getMessage(), "InvocationError"), null, requestId);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/PendingInvocation.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/PendingInvocation.java
index a49c8d433b..708b209e68 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/model/PendingInvocation.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/PendingInvocation.java
@@ -1,27 +1,41 @@
package io.github.hectorvent.floci.services.lambda.model;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
public class PendingInvocation {
private final String requestId;
private final byte[] payload;
- private final long deadlineMs;
+ private final long timeoutMs;
+ private volatile long deadlineMs;
private final String functionArn;
private final CompletableFuture<InvokeResult> resultFuture;
+ private final CompletableFuture<Void> dispatchedFuture = new CompletableFuture<>();
public PendingInvocation(String requestId, byte[] payload, long deadlineMs,
String functionArn, CompletableFuture<InvokeResult> resultFuture) {
this.requestId = requestId;
this.payload = payload;
this.deadlineMs = deadlineMs;
+ this.timeoutMs = Math.max(0, deadlineMs - System.currentTimeMillis());
this.functionArn = functionArn;
this.resultFuture = resultFuture;
}
+ public void prepareForDispatch() {
+ deadlineMs = System.currentTimeMillis() + timeoutMs;
+ }
+
+ public void markDispatched() {
+ resultFuture.orTimeout(Math.max(0, deadlineMs - System.currentTimeMillis()), TimeUnit.MILLISECONDS);
+ dispatchedFuture.complete(null);
+ }
+
public String getRequestId() { return requestId; }
public byte[] getPayload() { return payload; }
public long getDeadlineMs() { return deadlineMs; }
public String getFunctionArn() { return functionArn; }
public CompletableFuture<InvokeResult> getResultFuture() { return resultFuture; }
+ public CompletableFuture<Void> getDispatchedFuture() { return dispatchedFuture; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
index da4f2d15d1..4d3a58b6f4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
@@ -868,6 +868,7 @@ public class RuntimeApiServer {
private void sendInvocation(RoutingContext ctx, PendingInvocation invocation) {
beforeSendInvocationWrite(invocation.getRequestId());
+ invocation.prepareForDispatch();
// Invariant: callers put the invocation in inFlight under the lock before
// dispatching, so no inFlight.put here.
byte[] payload = invocation.getPayload();
@@ -891,7 +892,10 @@ public class RuntimeApiServer {
}
// Gated on the successful write: onFailure below requeues the invocation, and a fan-out
// before the write would emit a second INVOKE for the same requestId on redelivery.
- write.onSuccess(v -> notifyExtensionsOfInvoke(invocation));
+ write.onSuccess(v -> {
+ invocation.markDispatched();
+ notifyExtensionsOfInvoke(invocation);
+ });
write.onFailure(err -> {
// Requeue for the next /next poller; mirrors enqueue()'s ctx-ended branch.
// Skip if quiesce() beat us — it already swept inFlight and settled the
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java
index 4483aa1a1d..6c1d90cfd8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java
@@ -94,6 +94,30 @@ class LambdaExecutorServiceTest {
assertNull(result.getFunctionError());
}
+ @Test
+ void resultAfterConfiguredDeadline_timesOutAndDestroysHandle() {
+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);
+ ContainerHandle handle = new ContainerHandle("cid-late", "test-fn", rtas, ContainerState.WARM);
+
+ when(warmPool.acquire(any())).thenReturn(handle);
+ doAnswer(inv -> {
+ PendingInvocation pi = inv.getArgument(0);
+ pi.prepareForDispatch();
+ pi.markDispatched();
+ CompletableFuture.delayedExecutor(1200, TimeUnit.MILLISECONDS)
+ .execute(() -> pi.getResultFuture().complete(
+ new InvokeResult(200, null, "{\"ok\":true}".getBytes(), null, "req-late")));
+ return pi.getResultFuture();
+ }).when(rtas).enqueue(any(PendingInvocation.class));
+
+ InvokeResult result = executor.invoke(fn, "{}".getBytes(), InvocationType.RequestResponse);
+
+ verify(warmPool).destroyHandle(handle);
+ verify(warmPool, never()).release(handle);
+ assertEquals("Unhandled", result.getFunctionError());
+ assertTrue(new String(result.getPayload()).contains("Function.TimedOut"));
+ }
+
@Test
void timeoutResponse_containsCorrectErrorPayload() {
fn.setTimeout(2);
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
index c9fdbb8d4b..7b8b15b0f4 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
@@ -109,6 +109,29 @@ class RuntimeApiServerTest {
assertTrue(response.body().contains("key"));
}
+ @Test
+ @Timeout(10)
+ void nextEndpoint_startsDeadlineWhenInvocationIsDispatched() throws Exception {
+ long queuedDeadline = System.currentTimeMillis() + 1_000;
+ PendingInvocation invocation = new PendingInvocation(
+ "req-deadline", "{}".getBytes(), queuedDeadline,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+
+ Thread.sleep(300);
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port
+ + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ long advertisedDeadline = Long.parseLong(response.headers()
+ .firstValue("Lambda-Runtime-Deadline-Ms").orElseThrow());
+ assertTrue(advertisedDeadline >= queuedDeadline + 200,
+ "queueing and cold-start time must not consume the handler timeout");
+ }
+
/**
* Regression: an Invoke with no body (e.g. {@code aws lambda invoke} without
* {@code --payload}) reaches the /next handler as a {@code byte[0]}, not
@@ -2067,4 +2090,4 @@ class RuntimeApiServerTest {
}
return server.isExtensionParked(extensionId);
}
-}
\ No newline at end of file
+}