Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): fix container lifecycle on timeout (rebase of #504) (#529)

TiMf837be005187
diff --git a/docker-compose.yml b/docker-compose.yml
index 9244a9d256..aa0595555c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,6 +7,7 @@ services:
- "4566:4566"
- "6379-6399:6379-6399"
- "7001-7099:7001-7099"
+ - "9200-9299:9200-9299"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./scripts:/etc/floci/init/start.d
@@ -26,5 +27,3 @@ services:
networks:
floci_default:
name: floci_default
-
-
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
index 20d49634cf..d1966141dc 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
@@ -107,7 +107,7 @@ public class LambdaExecutorService {
} catch (TimeoutException e) {
LOG.warnv("Function {0} timed out after {1}s", fn.getFunctionName(), fn.getTimeout());
- warmPool.drainFunction(fn.getFunctionName());
+ warmPool.destroyHandle(handle);
return new InvokeResult(200, "Unhandled",
buildErrorPayload("Task timed out after " + fn.getTimeout() + " seconds", "Function.TimedOut"),
null, requestId);
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
index 6e204ecdb6..cae428680e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
@@ -160,6 +160,17 @@ public class WarmPool {
drainFunction(fn.getFunctionName());
}
+ /**
+ * Stops and removes a single container that is no longer usable (e.g. after a timeout).
+ * The container must have already been acquired (removed from the pool) so only a
+ * stop is needed — no pool bookkeeping required.
+ */
+ public void destroyHandle(ContainerHandle handle) {
+ LOG.debugv("Destroying timed-out container {0} for function {1}",
+ handle.getContainerId(), handle.getFunctionName());
+ stopQuietly(handle);
+ }
+
/**
* Stops and removes all warm containers for the given function.
* Called on function delete or code update.
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
index e0597486cf..587634c675 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
@@ -211,8 +211,8 @@ public class ContainerLauncher {
LOG.infov("Stopping container {0}", handle.getContainerId());
handle.setState(ContainerState.STOPPED);
- lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());
handle.getRuntimeApiServer().stop();
+ lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());
}
private void copyDirToContainer(DockerClient dockerClient, String containerId,
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
index a9157e1d00..58d7ccf72e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
@@ -38,6 +38,7 @@ public class RuntimeApiServer {
private final ConcurrentHashMap<String, PendingInvocation> inFlight = new ConcurrentHashMap<>();
private volatile HttpServer httpServer;
+ private volatile boolean stopped;
RuntimeApiServer(Vertx vertx, int port) {
this.vertx = vertx;
@@ -57,7 +58,10 @@ public class RuntimeApiServer {
// GET /runtime/invocation/next — long-poll for next invocation
router.get(NEXT_PATH).blockingHandler(ctx -> {
try {
- PendingInvocation invocation = pendingQueue.poll(30, TimeUnit.SECONDS);
+ PendingInvocation invocation = null;
+ while (invocation == null && !stopped) {
+ invocation = pendingQueue.poll(30, TimeUnit.SECONDS);
+ }
if (invocation == null) {
ctx.response().setStatusCode(204).end();
return;
@@ -124,6 +128,7 @@ public class RuntimeApiServer {
}
public void stop() {
+ stopped = true;
if (httpServer != null) {
httpServer.close();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java
new file mode 100644
index 0000000000..57481947c6
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java
@@ -0,0 +1,121 @@
+package io.github.hectorvent.floci.services.lambda;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;
+import io.github.hectorvent.floci.services.lambda.model.ContainerState;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import io.github.hectorvent.floci.services.lambda.model.PendingInvocation;
+import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
+
+import java.util.concurrent.CompletableFuture;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+@MockitoSettings(strictness = Strictness.LENIENT)
+class LambdaExecutorServiceTest {
+
+ @Mock WarmPool warmPool;
+ @Mock LambdaConcurrencyLimiter concurrencyLimiter;
+
+ private LambdaExecutorService executor;
+ private LambdaFunction fn;
+
+ @BeforeEach
+ void setUp() {
+ executor = new LambdaExecutorService(warmPool, new ObjectMapper(), concurrencyLimiter);
+
+ fn = new LambdaFunction();
+ fn.setFunctionName("test-fn");
+ fn.setFunctionArn("arn:aws:lambda:us-east-1:000000000000:function:test-fn");
+ fn.setTimeout(1);
+
+ when(concurrencyLimiter.acquire(any())).thenReturn(() -> {});
+ }
+
+ @Test
+ void timeoutInvocation_destroysHandle_doesNotRelease() {
+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);
+ ContainerHandle handle = new ContainerHandle("cid-1", "test-fn", rtas, ContainerState.WARM);
+
+ when(warmPool.acquire(any())).thenReturn(handle);
+ when(rtas.enqueue(any())).thenReturn(new CompletableFuture<>());
+
+ InvokeResult result = executor.invoke(fn, "{}".getBytes(), InvocationType.RequestResponse);
+
+ verify(warmPool).destroyHandle(handle);
+ verify(warmPool, never()).release(handle);
+ assertEquals(200, result.getStatusCode());
+ assertEquals("Unhandled", result.getFunctionError());
+ String payload = new String(result.getPayload());
+ assertTrue(payload.contains("Function.TimedOut"), "payload should contain error type");
+ assertTrue(payload.contains("1 seconds"), "payload should contain timeout value");
+ }
+
+ @Test
+ void successfulInvocation_releasesHandle_doesNotDestroy() {
+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);
+ ContainerHandle handle = new ContainerHandle("cid-2", "test-fn", rtas, ContainerState.WARM);
+
+ when(warmPool.acquire(any())).thenReturn(handle);
+ InvokeResult expected = new InvokeResult(200, null, "{\"ok\":true}".getBytes(), null, "req-1");
+ doAnswer(inv -> {
+ PendingInvocation pi = inv.getArgument(0);
+ pi.getResultFuture().complete(expected);
+ return pi.getResultFuture();
+ }).when(rtas).enqueue(any(PendingInvocation.class));
+
+ InvokeResult result = executor.invoke(fn, "{}".getBytes(), InvocationType.RequestResponse);
+
+ verify(warmPool).release(handle);
+ verify(warmPool, never()).destroyHandle(handle);
+ assertEquals(200, result.getStatusCode());
+ assertNull(result.getFunctionError());
+ }
+
+ @Test
+ void timeoutResponse_containsCorrectErrorPayload() {
+ fn.setTimeout(2);
+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);
+ ContainerHandle handle = new ContainerHandle("cid-3", "test-fn", rtas, ContainerState.WARM);
+
+ when(warmPool.acquire(any())).thenReturn(handle);
+ when(rtas.enqueue(any())).thenReturn(new CompletableFuture<>());
+
+ InvokeResult result = executor.invoke(fn, "{}".getBytes(), InvocationType.RequestResponse);
+
+ assertNotNull(result.getPayload());
+ String payload = new String(result.getPayload());
+ assertTrue(payload.contains("\"errorType\":\"Function.TimedOut\""));
+ assertTrue(payload.contains("Task timed out after 2 seconds"));
+ assertNotNull(result.getRequestId());
+ }
+
+ @Test
+ void dryRunInvocation_doesNotAcquireContainer() {
+ InvokeResult result = executor.invoke(fn, "{}".getBytes(), InvocationType.DryRun);
+
+ verify(warmPool, never()).acquire(any());
+ verify(warmPool, never()).release(any());
+ verify(warmPool, never()).destroyHandle(any());
+ assertEquals(204, result.getStatusCode());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
index bd74d930f4..67ca5601eb 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
@@ -1,7 +1,10 @@
package io.github.hectorvent.floci.services.lambda;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;
import io.github.hectorvent.floci.services.lambda.launcher.ContainerLauncher;
+import io.github.hectorvent.floci.services.lambda.model.ContainerState;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
@@ -9,8 +12,14 @@ import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Field;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
@@ -56,4 +65,91 @@ class WarmPoolTest {
pool.shutdown();
}
+
+ @Test
+ void destroyHandleStopsContainerAndDoesNotReturnToPool() {
+ WarmPool pool = buildPool();
+ pool.init();
+
+ ContainerHandle handle = new ContainerHandle("cid-123", "my-fn", null, ContainerState.BUSY);
+ LambdaFunction fn = mock(LambdaFunction.class);
+ when(fn.getFunctionName()).thenReturn("my-fn");
+ when(containerLauncher.launch(any())).thenReturn(handle);
+
+ ContainerHandle acquired = pool.acquire(fn);
+ assertEquals(handle, acquired);
+
+ pool.destroyHandle(acquired);
+ verify(containerLauncher).stop(handle);
+
+ // Pool must be empty — next acquire must cold-start
+ ContainerHandle handle2 = new ContainerHandle("cid-456", "my-fn", null, ContainerState.WARM);
+ when(containerLauncher.launch(any())).thenReturn(handle2);
+ ContainerHandle secondAcquired = pool.acquire(fn);
+ assertEquals(handle2, secondAcquired);
+
+ pool.shutdown();
+ }
+
+ @Test
+ void destroyHandle_doesNotAffectOtherContainersInPool() {
+ WarmPool pool = buildPool();
+ pool.init();
+
+ LambdaFunction fn = mock(LambdaFunction.class);
+ when(fn.getFunctionName()).thenReturn("multi-fn");
+
+ ContainerHandle h1 = new ContainerHandle("cid-a", "multi-fn", null, ContainerState.WARM);
+ ContainerHandle h2 = new ContainerHandle("cid-b", "multi-fn", null, ContainerState.WARM);
+
+ when(containerLauncher.launch(any())).thenReturn(h1, h2);
+
+ ContainerHandle acquired1 = pool.acquire(fn);
+ pool.release(acquired1);
+
+ ContainerHandle acquired2 = pool.acquire(fn);
+ pool.release(acquired2);
+
+ // Re-acquire both: h2 was released last so it's at the front of the deque
+ ContainerHandle toDestroy = pool.acquire(fn);
+ ContainerHandle survivor = pool.acquire(fn);
+
+ pool.destroyHandle(toDestroy);
+ verify(containerLauncher, times(1)).stop(toDestroy);
+ verify(containerLauncher, never()).stop(survivor);
+
+ // Survivor can be released back and re-acquired
+ pool.release(survivor);
+ ContainerHandle reacquired = pool.acquire(fn);
+ assertSame(survivor, reacquired);
+
+ pool.shutdown();
+ }
+
+ @Test
+ void releaseAfterSuccessfulInvocation_returnsToPool() {
+ WarmPool pool = buildPool();
+ pool.init();
+
+ LambdaFunction fn = mock(LambdaFunction.class);
+ when(fn.getFunctionName()).thenReturn("reuse-fn");
+
+ ContainerHandle handle = new ContainerHandle("cid-reuse", "reuse-fn", null, ContainerState.WARM);
+ when(containerLauncher.launch(any())).thenReturn(handle);
+
+ ContainerHandle first = pool.acquire(fn);
+ assertEquals(ContainerState.BUSY, first.getState());
+
+ pool.release(first);
+ assertEquals(ContainerState.WARM, first.getState());
+
+ // Second acquire should return the same handle from the pool (no cold start)
+ ContainerHandle second = pool.acquire(fn);
+ assertSame(handle, second);
+
+ // containerLauncher.launch should only have been called once (cold start)
+ verify(containerLauncher, times(1)).launch(any());
+
+ pool.shutdown();
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
new file mode 100644
index 0000000000..b7c362b8b4
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
@@ -0,0 +1,142 @@
+package io.github.hectorvent.floci.services.lambda.runtime;
+
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.lambda.model.PendingInvocation;
+import io.vertx.core.Vertx;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class RuntimeApiServerTest {
+
+ private Vertx vertx;
+ private RuntimeApiServer server;
+ private int port;
+ private HttpClient httpClient;
+ private ScheduledExecutorService scheduler;
+
+ @BeforeEach
+ void setUp() throws Exception {
+ vertx = Vertx.vertx();
+ port = findFreePort();
+ server = new RuntimeApiServer(vertx, port);
+ server.start().get(5, TimeUnit.SECONDS);
+ httpClient = HttpClient.newBuilder()
+ .connectTimeout(java.time.Duration.ofSeconds(5))
+ .build();
+ scheduler = Executors.newSingleThreadScheduledExecutor();
+ }
+
+ @AfterEach
+ void tearDown() {
+ server.stop();
+ scheduler.shutdownNow();
+ vertx.close();
+ }
+
+ @Test
+ @Timeout(15)
+ void nextEndpoint_blocksUntilInvocationArrives() throws Exception {
+ PendingInvocation invocation = new PendingInvocation(
+ "req-1", "{\"key\":\"value\"}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+
+ scheduler.schedule(() -> server.enqueue(invocation), 2, TimeUnit.SECONDS);
+
+ HttpRequest request = HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET()
+ .build();
+
+ long start = System.currentTimeMillis();
+ HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+ long elapsed = System.currentTimeMillis() - start;
+
+ assertEquals(200, response.statusCode());
+ assertTrue(elapsed >= 1500, "should have blocked ~2s waiting for invocation");
+ assertEquals("req-1", response.headers().firstValue("Lambda-Runtime-Aws-Request-Id").orElse(""));
+ assertTrue(response.body().contains("key"));
+ }
+
+ @Test
+ @Timeout(45)
+ void nextEndpoint_doesNotReturn204AfterPollCycle() throws Exception {
+ HttpRequest request = HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET()
+ .build();
+
+ CompletableFuture<HttpResponse<String>> asyncResponse =
+ httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString());
+
+ Thread.sleep(33_000);
+
+ assertFalse(asyncResponse.isDone(),
+ "GET /next must NOT return (old bug: 204 after 30s single poll)");
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-delayed", "{\"after_poll_cycle\":true}".getBytes(),
+ System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+
+ HttpResponse<String> response = asyncResponse.get(10, TimeUnit.SECONDS);
+ assertEquals(200, response.statusCode());
+ assertEquals("req-delayed",
+ response.headers().firstValue("Lambda-Runtime-Aws-Request-Id").orElse(""));
+ }
+
+ @Test
+ @Timeout(15)
+ void stopCompletesInFlightWithContainerStopped() throws Exception {
+ PendingInvocation invocation = new PendingInvocation(
+ "req-stop", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+
+ // Enqueue and have a GET request pick it up (moving it to inFlight)
+ server.enqueue(invocation);
+
+ HttpRequest request = HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET()
+ .build();
+ HttpResponse<String> getResponse = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+ assertEquals(200, getResponse.statusCode());
+
+ // Invocation is now in-flight (RIC got it but hasn't POSTed /response yet).
+ // Stopping the server should complete the future with ContainerStopped.
+ server.stop();
+
+ InvokeResult result = invocation.getResultFuture().get(5, TimeUnit.SECONDS);
+ assertNotNull(result);
+ assertEquals("Unhandled", result.getFunctionError());
+ String payload = new String(result.getPayload());
+ assertTrue(payload.contains("ContainerStopped"));
+ }
+
+ private static int findFreePort() throws IOException {
+ try (ServerSocket socket = new ServerSocket(0)) {
+ return socket.getLocalPort();
+ }
+ }
+}