Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): drop dead pooled containers in WarmPool.acquire() before reuse (#610)
Ricardo Lima9bdcfaf2b12f
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 1585b566bf..8d18352bbf 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -245,6 +245,26 @@ public class ContainerLifecycleManager {}}+ /**+ * Returns whether the container is currently running. A missing container+ * is treated as not-running; any other Docker error is treated as running+ * so a transient daemon hiccup does not evict a healthy warm pool.+ *+ * @param containerId the container ID to inspect+ * @return true if the container exists and is reported as running+ */+ public boolean isContainerRunning(String containerId) {+ try {+ InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();+ return Boolean.TRUE.equals(inspect.getState().getRunning());+ } catch (NotFoundException e) {+ return false;+ } catch (Exception e) {+ LOG.warnv("Liveness check failed for container {0}: {1}", containerId, e.getMessage());+ return true;+ }+ }+/*** Resolves the endpoint (host and port) to connect to a specific container port.*diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.javaindex 69f7f03ef2..119bc6879f 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java@@ -103,8 +103,23 @@ public class WarmPool {if (!ephemeral) {ArrayDeque<ContainerHandle> queue = pool.computeIfAbsent(fn.getFunctionName(), k -> new ArrayDeque<>());- synchronized (queue) {- handle = queue.pollFirst();+ // Skip pooled handles whose container died out-of-band — otherwise the+ // caller would wait the full Lambda function timeout.+ while (true) {+ ContainerHandle candidate;+ synchronized (queue) {+ candidate = queue.pollFirst();+ }+ if (candidate == null) {+ break;+ }+ if (containerLauncher.isAlive(candidate)) {+ handle = candidate;+ break;+ }+ LOG.infov("Discarding dead pooled container {0} for function {1}",+ candidate.getContainerId(), fn.getFunctionName());+ stopQuietly(candidate);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex b451111965..c7f254b9fc 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -225,6 +225,16 @@ public class ContainerLauncher {lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());}+ /**+ * Probes whether the handle's underlying container is still running.+ *+ * @param handle the warm-pool handle to probe+ * @return true if the container is still running+ */+ public boolean isAlive(ContainerHandle handle) {+ return lifecycleManager.isContainerRunning(handle.getContainerId());+ }+private void copyDirToContainer(DockerClient dockerClient, String containerId,Path sourceDir, String remotePath, String functionName) {try (java.io.PipedOutputStream pos = new java.io.PipedOutputStream();diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.javaindex 67ca5601eb..dfa19e792e 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java@@ -15,6 +15,7 @@ import java.lang.reflect.Field;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertNotSame;import static org.mockito.ArgumentMatchers.any;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;@@ -103,6 +104,7 @@ class WarmPoolTest {ContainerHandle h2 = new ContainerHandle("cid-b", "multi-fn", null, ContainerState.WARM);when(containerLauncher.launch(any())).thenReturn(h1, h2);+ when(containerLauncher.isAlive(any())).thenReturn(true);ContainerHandle acquired1 = pool.acquire(fn);pool.release(acquired1);@@ -136,6 +138,7 @@ class WarmPoolTest {ContainerHandle handle = new ContainerHandle("cid-reuse", "reuse-fn", null, ContainerState.WARM);when(containerLauncher.launch(any())).thenReturn(handle);+ when(containerLauncher.isAlive(any())).thenReturn(true);ContainerHandle first = pool.acquire(fn);assertEquals(ContainerState.BUSY, first.getState());@@ -152,4 +155,70 @@ class WarmPoolTest {pool.shutdown();}++ @Test+ void acquire_discardsDeadPooledHandleAndColdStarts() {+ WarmPool pool = buildPool();+ pool.init();++ LambdaFunction fn = mock(LambdaFunction.class);+ when(fn.getFunctionName()).thenReturn("dead-fn");++ ContainerHandle dead = new ContainerHandle("cid-dead", "dead-fn", null, ContainerState.WARM);+ ContainerHandle fresh = new ContainerHandle("cid-fresh", "dead-fn", null, ContainerState.WARM);++ // Seed the pool with the dead handle by acquiring + releasing it once.+ // The seed acquire is a cold start (empty pool), so isAlive isn't called.+ when(containerLauncher.launch(any())).thenReturn(dead, fresh);+ ContainerHandle seeded = pool.acquire(fn);+ assertSame(dead, seeded);+ pool.release(seeded);++ // Now the container "dies" out-of-band (docker rm -f, OOM, etc.).+ when(containerLauncher.isAlive(dead)).thenReturn(false);++ ContainerHandle acquired = pool.acquire(fn);+ assertSame(fresh, acquired);+ assertNotSame(dead, acquired);+ verify(containerLauncher, times(1)).stop(dead);+ verify(containerLauncher, times(2)).launch(any());++ pool.shutdown();+ }++ @Test+ void acquire_skipsDeadHandleAndReusesNextAlive() {+ WarmPool pool = buildPool();+ pool.init();++ LambdaFunction fn = mock(LambdaFunction.class);+ when(fn.getFunctionName()).thenReturn("mixed-fn");++ ContainerHandle dead = new ContainerHandle("cid-dead", "mixed-fn", null, ContainerState.WARM);+ ContainerHandle alive = new ContainerHandle("cid-alive", "mixed-fn", null, ContainerState.WARM);++ // Seed deque with [dead, alive]: release(alive) first, then release(dead),+ // so dead ends up at the front (release uses addFirst). Both acquires+ // here are cold starts (empty pool) so no isAlive stub is needed yet.+ when(containerLauncher.launch(any())).thenReturn(alive, dead);+ ContainerHandle a1 = pool.acquire(fn);+ ContainerHandle a2 = pool.acquire(fn);+ assertSame(alive, a1);+ assertSame(dead, a2);+ pool.release(a1);+ pool.release(a2);++ // dead dies out-of-band, alive is still up.+ when(containerLauncher.isAlive(dead)).thenReturn(false);+ when(containerLauncher.isAlive(alive)).thenReturn(true);++ ContainerHandle acquired = pool.acquire(fn);+ assertSame(alive, acquired);+ verify(containerLauncher, times(1)).stop(dead);+ verify(containerLauncher, never()).stop(alive);+ // Only the original two cold starts; no extra launch was needed.+ verify(containerLauncher, times(2)).launch(any());++ pool.shutdown();+ }}