Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): fix container lifecycle on timeout (rebase of #504) (#529)
TiMf837be005187
diff --git a/docker-compose.yml b/docker-compose.ymlindex 9244a9d256..aa0595555c 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -7,6 +7,7 @@ services:- "4566:4566"- "6379-6399:6379-6399"- "7001-7099:7001-7099"+ - "9200-9299:9200-9299"volumes:- /var/run/docker.sock:/var/run/docker.sock- ./scripts:/etc/floci/init/start.d@@ -26,5 +27,3 @@ services:networks:floci_default:name: floci_default--diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.javaindex 20d49634cf..d1966141dc 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java@@ -107,7 +107,7 @@ public class LambdaExecutorService {} catch (TimeoutException e) {LOG.warnv("Function {0} timed out after {1}s", fn.getFunctionName(), fn.getTimeout());- warmPool.drainFunction(fn.getFunctionName());+ warmPool.destroyHandle(handle);return new InvokeResult(200, "Unhandled",buildErrorPayload("Task timed out after " + fn.getTimeout() + " seconds", "Function.TimedOut"),null, requestId);diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.javaindex 6e204ecdb6..cae428680e 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java@@ -160,6 +160,17 @@ public class WarmPool {drainFunction(fn.getFunctionName());}+ /**+ * Stops and removes a single container that is no longer usable (e.g. after a timeout).+ * The container must have already been acquired (removed from the pool) so only a+ * stop is needed — no pool bookkeeping required.+ */+ public void destroyHandle(ContainerHandle handle) {+ LOG.debugv("Destroying timed-out container {0} for function {1}",+ handle.getContainerId(), handle.getFunctionName());+ stopQuietly(handle);+ }+/*** Stops and removes all warm containers for the given function.* Called on function delete or code update.diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex e0597486cf..587634c675 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -211,8 +211,8 @@ public class ContainerLauncher {LOG.infov("Stopping container {0}", handle.getContainerId());handle.setState(ContainerState.STOPPED);- lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());handle.getRuntimeApiServer().stop();+ lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());}private void copyDirToContainer(DockerClient dockerClient, String containerId,diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.javaindex a9157e1d00..58d7ccf72e 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java@@ -38,6 +38,7 @@ public class RuntimeApiServer {private final ConcurrentHashMap<String, PendingInvocation> inFlight = new ConcurrentHashMap<>();private volatile HttpServer httpServer;+ private volatile boolean stopped;RuntimeApiServer(Vertx vertx, int port) {this.vertx = vertx;@@ -57,7 +58,10 @@ public class RuntimeApiServer {// GET /runtime/invocation/next — long-poll for next invocationrouter.get(NEXT_PATH).blockingHandler(ctx -> {try {- PendingInvocation invocation = pendingQueue.poll(30, TimeUnit.SECONDS);+ PendingInvocation invocation = null;+ while (invocation == null && !stopped) {+ invocation = pendingQueue.poll(30, TimeUnit.SECONDS);+ }if (invocation == null) {ctx.response().setStatusCode(204).end();return;@@ -124,6 +128,7 @@ public class RuntimeApiServer {}public void stop() {+ stopped = true;if (httpServer != null) {httpServer.close();}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.javanew file mode 100644index 0000000000..57481947c6--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java@@ -0,0 +1,121 @@+package io.github.hectorvent.floci.services.lambda;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;+import io.github.hectorvent.floci.services.lambda.model.ContainerState;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;+import io.github.hectorvent.floci.services.lambda.model.PendingInvocation;+import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.extension.ExtendWith;+import org.mockito.Mock;+import org.mockito.junit.jupiter.MockitoExtension;+import org.mockito.junit.jupiter.MockitoSettings;+import org.mockito.quality.Strictness;++import java.util.concurrent.CompletableFuture;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++@ExtendWith(MockitoExtension.class)+@MockitoSettings(strictness = Strictness.LENIENT)+class LambdaExecutorServiceTest {++ @Mock WarmPool warmPool;+ @Mock LambdaConcurrencyLimiter concurrencyLimiter;++ private LambdaExecutorService executor;+ private LambdaFunction fn;++ @BeforeEach+ void setUp() {+ executor = new LambdaExecutorService(warmPool, new ObjectMapper(), concurrencyLimiter);++ fn = new LambdaFunction();+ fn.setFunctionName("test-fn");+ fn.setFunctionArn("arn:aws:lambda:us-east-1:000000000000:function:test-fn");+ fn.setTimeout(1);++ when(concurrencyLimiter.acquire(any())).thenReturn(() -> {});+ }++ @Test+ void timeoutInvocation_destroysHandle_doesNotRelease() {+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ ContainerHandle handle = new ContainerHandle("cid-1", "test-fn", rtas, ContainerState.WARM);++ when(warmPool.acquire(any())).thenReturn(handle);+ when(rtas.enqueue(any())).thenReturn(new CompletableFuture<>());++ InvokeResult result = executor.invoke(fn, "{}".getBytes(), InvocationType.RequestResponse);++ verify(warmPool).destroyHandle(handle);+ verify(warmPool, never()).release(handle);+ assertEquals(200, result.getStatusCode());+ assertEquals("Unhandled", result.getFunctionError());+ String payload = new String(result.getPayload());+ assertTrue(payload.contains("Function.TimedOut"), "payload should contain error type");+ assertTrue(payload.contains("1 seconds"), "payload should contain timeout value");+ }++ @Test+ void successfulInvocation_releasesHandle_doesNotDestroy() {+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ ContainerHandle handle = new ContainerHandle("cid-2", "test-fn", rtas, ContainerState.WARM);++ when(warmPool.acquire(any())).thenReturn(handle);+ InvokeResult expected = new InvokeResult(200, null, "{\"ok\":true}".getBytes(), null, "req-1");+ doAnswer(inv -> {+ PendingInvocation pi = inv.getArgument(0);+ pi.getResultFuture().complete(expected);+ return pi.getResultFuture();+ }).when(rtas).enqueue(any(PendingInvocation.class));++ InvokeResult result = executor.invoke(fn, "{}".getBytes(), InvocationType.RequestResponse);++ verify(warmPool).release(handle);+ verify(warmPool, never()).destroyHandle(handle);+ assertEquals(200, result.getStatusCode());+ assertNull(result.getFunctionError());+ }++ @Test+ void timeoutResponse_containsCorrectErrorPayload() {+ fn.setTimeout(2);+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ ContainerHandle handle = new ContainerHandle("cid-3", "test-fn", rtas, ContainerState.WARM);++ when(warmPool.acquire(any())).thenReturn(handle);+ when(rtas.enqueue(any())).thenReturn(new CompletableFuture<>());++ InvokeResult result = executor.invoke(fn, "{}".getBytes(), InvocationType.RequestResponse);++ assertNotNull(result.getPayload());+ String payload = new String(result.getPayload());+ assertTrue(payload.contains("\"errorType\":\"Function.TimedOut\""));+ assertTrue(payload.contains("Task timed out after 2 seconds"));+ assertNotNull(result.getRequestId());+ }++ @Test+ void dryRunInvocation_doesNotAcquireContainer() {+ InvokeResult result = executor.invoke(fn, "{}".getBytes(), InvocationType.DryRun);++ verify(warmPool, never()).acquire(any());+ verify(warmPool, never()).release(any());+ verify(warmPool, never()).destroyHandle(any());+ assertEquals(204, result.getStatusCode());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.javaindex bd74d930f4..67ca5601eb 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java@@ -1,7 +1,10 @@package io.github.hectorvent.floci.services.lambda;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;import io.github.hectorvent.floci.services.lambda.launcher.ContainerLauncher;+import io.github.hectorvent.floci.services.lambda.model.ContainerState;+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;import org.mockito.Mock;@@ -9,8 +12,14 @@ import org.mockito.junit.jupiter.MockitoExtension;import java.lang.reflect.Field;+import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertSame;+import static org.mockito.ArgumentMatchers.any;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@ExtendWith(MockitoExtension.class)@@ -56,4 +65,91 @@ class WarmPoolTest {pool.shutdown();}++ @Test+ void destroyHandleStopsContainerAndDoesNotReturnToPool() {+ WarmPool pool = buildPool();+ pool.init();++ ContainerHandle handle = new ContainerHandle("cid-123", "my-fn", null, ContainerState.BUSY);+ LambdaFunction fn = mock(LambdaFunction.class);+ when(fn.getFunctionName()).thenReturn("my-fn");+ when(containerLauncher.launch(any())).thenReturn(handle);++ ContainerHandle acquired = pool.acquire(fn);+ assertEquals(handle, acquired);++ pool.destroyHandle(acquired);+ verify(containerLauncher).stop(handle);++ // Pool must be empty — next acquire must cold-start+ ContainerHandle handle2 = new ContainerHandle("cid-456", "my-fn", null, ContainerState.WARM);+ when(containerLauncher.launch(any())).thenReturn(handle2);+ ContainerHandle secondAcquired = pool.acquire(fn);+ assertEquals(handle2, secondAcquired);++ pool.shutdown();+ }++ @Test+ void destroyHandle_doesNotAffectOtherContainersInPool() {+ WarmPool pool = buildPool();+ pool.init();++ LambdaFunction fn = mock(LambdaFunction.class);+ when(fn.getFunctionName()).thenReturn("multi-fn");++ ContainerHandle h1 = new ContainerHandle("cid-a", "multi-fn", null, ContainerState.WARM);+ ContainerHandle h2 = new ContainerHandle("cid-b", "multi-fn", null, ContainerState.WARM);++ when(containerLauncher.launch(any())).thenReturn(h1, h2);++ ContainerHandle acquired1 = pool.acquire(fn);+ pool.release(acquired1);++ ContainerHandle acquired2 = pool.acquire(fn);+ pool.release(acquired2);++ // Re-acquire both: h2 was released last so it's at the front of the deque+ ContainerHandle toDestroy = pool.acquire(fn);+ ContainerHandle survivor = pool.acquire(fn);++ pool.destroyHandle(toDestroy);+ verify(containerLauncher, times(1)).stop(toDestroy);+ verify(containerLauncher, never()).stop(survivor);++ // Survivor can be released back and re-acquired+ pool.release(survivor);+ ContainerHandle reacquired = pool.acquire(fn);+ assertSame(survivor, reacquired);++ pool.shutdown();+ }++ @Test+ void releaseAfterSuccessfulInvocation_returnsToPool() {+ WarmPool pool = buildPool();+ pool.init();++ LambdaFunction fn = mock(LambdaFunction.class);+ when(fn.getFunctionName()).thenReturn("reuse-fn");++ ContainerHandle handle = new ContainerHandle("cid-reuse", "reuse-fn", null, ContainerState.WARM);+ when(containerLauncher.launch(any())).thenReturn(handle);++ ContainerHandle first = pool.acquire(fn);+ assertEquals(ContainerState.BUSY, first.getState());++ pool.release(first);+ assertEquals(ContainerState.WARM, first.getState());++ // Second acquire should return the same handle from the pool (no cold start)+ ContainerHandle second = pool.acquire(fn);+ assertSame(handle, second);++ // containerLauncher.launch should only have been called once (cold start)+ verify(containerLauncher, times(1)).launch(any());++ pool.shutdown();+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.javanew file mode 100644index 0000000000..b7c362b8b4--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java@@ -0,0 +1,142 @@+package io.github.hectorvent.floci.services.lambda.runtime;++import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.services.lambda.model.PendingInvocation;+import io.vertx.core.Vertx;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;++import java.io.IOException;+import java.net.ServerSocket;+import java.net.URI;+import java.net.http.HttpClient;+import java.net.http.HttpRequest;+import java.net.http.HttpResponse;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.Executors;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++class RuntimeApiServerTest {++ private Vertx vertx;+ private RuntimeApiServer server;+ private int port;+ private HttpClient httpClient;+ private ScheduledExecutorService scheduler;++ @BeforeEach+ void setUp() throws Exception {+ vertx = Vertx.vertx();+ port = findFreePort();+ server = new RuntimeApiServer(vertx, port);+ server.start().get(5, TimeUnit.SECONDS);+ httpClient = HttpClient.newBuilder()+ .connectTimeout(java.time.Duration.ofSeconds(5))+ .build();+ scheduler = Executors.newSingleThreadScheduledExecutor();+ }++ @AfterEach+ void tearDown() {+ server.stop();+ scheduler.shutdownNow();+ vertx.close();+ }++ @Test+ @Timeout(15)+ void nextEndpoint_blocksUntilInvocationArrives() throws Exception {+ PendingInvocation invocation = new PendingInvocation(+ "req-1", "{\"key\":\"value\"}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());++ scheduler.schedule(() -> server.enqueue(invocation), 2, TimeUnit.SECONDS);++ HttpRequest request = HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET()+ .build();++ long start = System.currentTimeMillis();+ HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());+ long elapsed = System.currentTimeMillis() - start;++ assertEquals(200, response.statusCode());+ assertTrue(elapsed >= 1500, "should have blocked ~2s waiting for invocation");+ assertEquals("req-1", response.headers().firstValue("Lambda-Runtime-Aws-Request-Id").orElse(""));+ assertTrue(response.body().contains("key"));+ }++ @Test+ @Timeout(45)+ void nextEndpoint_doesNotReturn204AfterPollCycle() throws Exception {+ HttpRequest request = HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET()+ .build();++ CompletableFuture<HttpResponse<String>> asyncResponse =+ httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString());++ Thread.sleep(33_000);++ assertFalse(asyncResponse.isDone(),+ "GET /next must NOT return (old bug: 204 after 30s single poll)");++ PendingInvocation invocation = new PendingInvocation(+ "req-delayed", "{\"after_poll_cycle\":true}".getBytes(),+ System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);++ HttpResponse<String> response = asyncResponse.get(10, TimeUnit.SECONDS);+ assertEquals(200, response.statusCode());+ assertEquals("req-delayed",+ response.headers().firstValue("Lambda-Runtime-Aws-Request-Id").orElse(""));+ }++ @Test+ @Timeout(15)+ void stopCompletesInFlightWithContainerStopped() throws Exception {+ PendingInvocation invocation = new PendingInvocation(+ "req-stop", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());++ // Enqueue and have a GET request pick it up (moving it to inFlight)+ server.enqueue(invocation);++ HttpRequest request = HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET()+ .build();+ HttpResponse<String> getResponse = httpClient.send(request, HttpResponse.BodyHandlers.ofString());+ assertEquals(200, getResponse.statusCode());++ // Invocation is now in-flight (RIC got it but hasn't POSTed /response yet).+ // Stopping the server should complete the future with ContainerStopped.+ server.stop();++ InvokeResult result = invocation.getResultFuture().get(5, TimeUnit.SECONDS);+ assertNotNull(result);+ assertEquals("Unhandled", result.getFunctionError());+ String payload = new String(result.getPayload());+ assertTrue(payload.contains("ContainerStopped"));+ }++ private static int findFreePort() throws IOException {+ try (ServerSocket socket = new ServerSocket(0)) {+ return socket.getLocalPort();+ }+ }+}