Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): enforce handler deadline at dispatch (#3177)
BingEdward438ae619fd1c
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.javaindex aa6a5aab40..4ab412af05 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java@@ -13,6 +13,7 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.util.UUID;+import java.util.concurrent.ExecutionException;import java.util.concurrent.ExecutorService;import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.ThreadPoolExecutor;@@ -27,8 +28,8 @@ import java.util.concurrent.TimeoutException;public class LambdaExecutorService {private static final Logger LOG = Logger.getLogger(LambdaExecutorService.class);- /** Grace period beyond the configured function timeout to allow the runtime to report back. */- private static final int TIMEOUT_GRACE_SECONDS = 2;+ /** Extra time for a newly started runtime to request its first invocation. */+ private static final int RUNTIME_DISPATCH_GRACE_SECONDS = 2;private final WarmPool warmPool;private final ObjectMapper objectMapper;@@ -99,26 +100,31 @@ public class LambdaExecutorService {handle.getRuntimeApiServer().enqueue(invocation);- InvokeResult result = invocation.getResultFuture()- .get(fn.getTimeout() + TIMEOUT_GRACE_SECONDS, TimeUnit.SECONDS);+ java.util.concurrent.CompletableFuture.anyOf(+ invocation.getDispatchedFuture(), invocation.getResultFuture())+ .get(fn.getTimeout() + RUNTIME_DISPATCH_GRACE_SECONDS, TimeUnit.SECONDS);+ InvokeResult result = invocation.getResultFuture().get();warmPool.release(handle);return result;- } catch (TimeoutException e) {- LOG.warnv("Function {0} timed out after {1}s", fn.getFunctionName(), fn.getTimeout());- warmPool.destroyHandle(handle);- return new InvokeResult(200, "Unhandled",- buildErrorPayload("Task timed out after " + fn.getTimeout() + " seconds", "Function.TimedOut"),- null, requestId);} catch (InterruptedException e) {Thread.currentThread().interrupt();warmPool.destroyHandle(handle);return new InvokeResult(200, "Unhandled", buildErrorPayload("Invocation interrupted", "Interrupted"), null, requestId);} catch (Exception e) {- LOG.warnv("Invocation error for function {0}: {1}", fn.getFunctionName(), e.getMessage());+ Throwable cause = e instanceof ExecutionException && e.getCause() != null ? e.getCause() : e;+ if (cause instanceof TimeoutException) {+ LOG.warnv("Function {0} timed out after {1}s", fn.getFunctionName(), fn.getTimeout());+ warmPool.destroyHandle(handle);+ return new InvokeResult(200, "Unhandled",+ buildErrorPayload("Task timed out after " + fn.getTimeout() + " seconds", "Function.TimedOut"),+ null, requestId);+ }+ LOG.warnv("Invocation error for function {0}: {1}", fn.getFunctionName(), cause.getMessage());warmPool.destroyHandle(handle);- return new InvokeResult(200, "Unhandled", buildErrorPayload(e.getMessage(), "InvocationError"), null, requestId);+ return new InvokeResult(200, "Unhandled",+ buildErrorPayload(cause.getMessage(), "InvocationError"), null, requestId);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/PendingInvocation.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/PendingInvocation.javaindex a49c8d433b..708b209e68 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/model/PendingInvocation.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/PendingInvocation.java@@ -1,27 +1,41 @@package io.github.hectorvent.floci.services.lambda.model;import java.util.concurrent.CompletableFuture;+import java.util.concurrent.TimeUnit;public class PendingInvocation {private final String requestId;private final byte[] payload;- private final long deadlineMs;+ private final long timeoutMs;+ private volatile long deadlineMs;private final String functionArn;private final CompletableFuture<InvokeResult> resultFuture;+ private final CompletableFuture<Void> dispatchedFuture = new CompletableFuture<>();public PendingInvocation(String requestId, byte[] payload, long deadlineMs,String functionArn, CompletableFuture<InvokeResult> resultFuture) {this.requestId = requestId;this.payload = payload;this.deadlineMs = deadlineMs;+ this.timeoutMs = Math.max(0, deadlineMs - System.currentTimeMillis());this.functionArn = functionArn;this.resultFuture = resultFuture;}+ public void prepareForDispatch() {+ deadlineMs = System.currentTimeMillis() + timeoutMs;+ }++ public void markDispatched() {+ resultFuture.orTimeout(Math.max(0, deadlineMs - System.currentTimeMillis()), TimeUnit.MILLISECONDS);+ dispatchedFuture.complete(null);+ }+public String getRequestId() { return requestId; }public byte[] getPayload() { return payload; }public long getDeadlineMs() { return deadlineMs; }public String getFunctionArn() { return functionArn; }public CompletableFuture<InvokeResult> getResultFuture() { return resultFuture; }+ public CompletableFuture<Void> getDispatchedFuture() { return dispatchedFuture; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.javaindex da4f2d15d1..4d3a58b6f4 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java@@ -868,6 +868,7 @@ public class RuntimeApiServer {private void sendInvocation(RoutingContext ctx, PendingInvocation invocation) {beforeSendInvocationWrite(invocation.getRequestId());+ invocation.prepareForDispatch();// Invariant: callers put the invocation in inFlight under the lock before// dispatching, so no inFlight.put here.byte[] payload = invocation.getPayload();@@ -891,7 +892,10 @@ public class RuntimeApiServer {}// Gated on the successful write: onFailure below requeues the invocation, and a fan-out// before the write would emit a second INVOKE for the same requestId on redelivery.- write.onSuccess(v -> notifyExtensionsOfInvoke(invocation));+ write.onSuccess(v -> {+ invocation.markDispatched();+ notifyExtensionsOfInvoke(invocation);+ });write.onFailure(err -> {// Requeue for the next /next poller; mirrors enqueue()'s ctx-ended branch.// Skip if quiesce() beat us — it already swept inFlight and settled thediff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.javaindex 4483aa1a1d..6c1d90cfd8 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java@@ -94,6 +94,30 @@ class LambdaExecutorServiceTest {assertNull(result.getFunctionError());}+ @Test+ void resultAfterConfiguredDeadline_timesOutAndDestroysHandle() {+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ ContainerHandle handle = new ContainerHandle("cid-late", "test-fn", rtas, ContainerState.WARM);++ when(warmPool.acquire(any())).thenReturn(handle);+ doAnswer(inv -> {+ PendingInvocation pi = inv.getArgument(0);+ pi.prepareForDispatch();+ pi.markDispatched();+ CompletableFuture.delayedExecutor(1200, TimeUnit.MILLISECONDS)+ .execute(() -> pi.getResultFuture().complete(+ new InvokeResult(200, null, "{\"ok\":true}".getBytes(), null, "req-late")));+ return pi.getResultFuture();+ }).when(rtas).enqueue(any(PendingInvocation.class));++ InvokeResult result = executor.invoke(fn, "{}".getBytes(), InvocationType.RequestResponse);++ verify(warmPool).destroyHandle(handle);+ verify(warmPool, never()).release(handle);+ assertEquals("Unhandled", result.getFunctionError());+ assertTrue(new String(result.getPayload()).contains("Function.TimedOut"));+ }+@Testvoid timeoutResponse_containsCorrectErrorPayload() {fn.setTimeout(2);diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.javaindex c9fdbb8d4b..7b8b15b0f4 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java@@ -109,6 +109,29 @@ class RuntimeApiServerTest {assertTrue(response.body().contains("key"));}+ @Test+ @Timeout(10)+ void nextEndpoint_startsDeadlineWhenInvocationIsDispatched() throws Exception {+ long queuedDeadline = System.currentTimeMillis() + 1_000;+ PendingInvocation invocation = new PendingInvocation(+ "req-deadline", "{}".getBytes(), queuedDeadline,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);++ Thread.sleep(300);+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port+ + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());++ long advertisedDeadline = Long.parseLong(response.headers()+ .firstValue("Lambda-Runtime-Deadline-Ms").orElseThrow());+ assertTrue(advertisedDeadline >= queuedDeadline + 200,+ "queueing and cold-start time must not consume the handler timeout");+ }+/*** Regression: an Invoke with no body (e.g. {@code aws lambda invoke} without* {@code --payload}) reaches the /next handler as a {@code byte[0]}, not@@ -2067,4 +2090,4 @@ class RuntimeApiServerTest {}return server.isExtensionParked(extensionId);}-}\ No newline at end of file+}