Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): drop dead pooled containers in WarmPool.acquire() before reuse (#610)

Ricardo Lima9bdcfaf2b12f
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 1585b566bf..8d18352bbf 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -245,6 +245,26 @@ public class ContainerLifecycleManager {
}
}
+ /**
+ * Returns whether the container is currently running. A missing container
+ * is treated as not-running; any other Docker error is treated as running
+ * so a transient daemon hiccup does not evict a healthy warm pool.
+ *
+ * @param containerId the container ID to inspect
+ * @return true if the container exists and is reported as running
+ */
+ public boolean isContainerRunning(String containerId) {
+ try {
+ InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();
+ return Boolean.TRUE.equals(inspect.getState().getRunning());
+ } catch (NotFoundException e) {
+ return false;
+ } catch (Exception e) {
+ LOG.warnv("Liveness check failed for container {0}: {1}", containerId, e.getMessage());
+ return true;
+ }
+ }
+
/**
* Resolves the endpoint (host and port) to connect to a specific container port.
*
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
index 69f7f03ef2..119bc6879f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
@@ -103,8 +103,23 @@ public class WarmPool {
if (!ephemeral) {
ArrayDeque<ContainerHandle> queue = pool.computeIfAbsent(fn.getFunctionName(), k -> new ArrayDeque<>());
- synchronized (queue) {
- handle = queue.pollFirst();
+ // Skip pooled handles whose container died out-of-band — otherwise the
+ // caller would wait the full Lambda function timeout.
+ while (true) {
+ ContainerHandle candidate;
+ synchronized (queue) {
+ candidate = queue.pollFirst();
+ }
+ if (candidate == null) {
+ break;
+ }
+ if (containerLauncher.isAlive(candidate)) {
+ handle = candidate;
+ break;
+ }
+ LOG.infov("Discarding dead pooled container {0} for function {1}",
+ candidate.getContainerId(), fn.getFunctionName());
+ stopQuietly(candidate);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
index b451111965..c7f254b9fc 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
@@ -225,6 +225,16 @@ public class ContainerLauncher {
lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());
}
+ /**
+ * Probes whether the handle's underlying container is still running.
+ *
+ * @param handle the warm-pool handle to probe
+ * @return true if the container is still running
+ */
+ public boolean isAlive(ContainerHandle handle) {
+ return lifecycleManager.isContainerRunning(handle.getContainerId());
+ }
+
private void copyDirToContainer(DockerClient dockerClient, String containerId,
Path sourceDir, String remotePath, String functionName) {
try (java.io.PipedOutputStream pos = new java.io.PipedOutputStream();
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
index 67ca5601eb..dfa19e792e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
@@ -15,6 +15,7 @@ import java.lang.reflect.Field;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -103,6 +104,7 @@ class WarmPoolTest {
ContainerHandle h2 = new ContainerHandle("cid-b", "multi-fn", null, ContainerState.WARM);
when(containerLauncher.launch(any())).thenReturn(h1, h2);
+ when(containerLauncher.isAlive(any())).thenReturn(true);
ContainerHandle acquired1 = pool.acquire(fn);
pool.release(acquired1);
@@ -136,6 +138,7 @@ class WarmPoolTest {
ContainerHandle handle = new ContainerHandle("cid-reuse", "reuse-fn", null, ContainerState.WARM);
when(containerLauncher.launch(any())).thenReturn(handle);
+ when(containerLauncher.isAlive(any())).thenReturn(true);
ContainerHandle first = pool.acquire(fn);
assertEquals(ContainerState.BUSY, first.getState());
@@ -152,4 +155,70 @@ class WarmPoolTest {
pool.shutdown();
}
+
+ @Test
+ void acquire_discardsDeadPooledHandleAndColdStarts() {
+ WarmPool pool = buildPool();
+ pool.init();
+
+ LambdaFunction fn = mock(LambdaFunction.class);
+ when(fn.getFunctionName()).thenReturn("dead-fn");
+
+ ContainerHandle dead = new ContainerHandle("cid-dead", "dead-fn", null, ContainerState.WARM);
+ ContainerHandle fresh = new ContainerHandle("cid-fresh", "dead-fn", null, ContainerState.WARM);
+
+ // Seed the pool with the dead handle by acquiring + releasing it once.
+ // The seed acquire is a cold start (empty pool), so isAlive isn't called.
+ when(containerLauncher.launch(any())).thenReturn(dead, fresh);
+ ContainerHandle seeded = pool.acquire(fn);
+ assertSame(dead, seeded);
+ pool.release(seeded);
+
+ // Now the container "dies" out-of-band (docker rm -f, OOM, etc.).
+ when(containerLauncher.isAlive(dead)).thenReturn(false);
+
+ ContainerHandle acquired = pool.acquire(fn);
+ assertSame(fresh, acquired);
+ assertNotSame(dead, acquired);
+ verify(containerLauncher, times(1)).stop(dead);
+ verify(containerLauncher, times(2)).launch(any());
+
+ pool.shutdown();
+ }
+
+ @Test
+ void acquire_skipsDeadHandleAndReusesNextAlive() {
+ WarmPool pool = buildPool();
+ pool.init();
+
+ LambdaFunction fn = mock(LambdaFunction.class);
+ when(fn.getFunctionName()).thenReturn("mixed-fn");
+
+ ContainerHandle dead = new ContainerHandle("cid-dead", "mixed-fn", null, ContainerState.WARM);
+ ContainerHandle alive = new ContainerHandle("cid-alive", "mixed-fn", null, ContainerState.WARM);
+
+ // Seed deque with [dead, alive]: release(alive) first, then release(dead),
+ // so dead ends up at the front (release uses addFirst). Both acquires
+ // here are cold starts (empty pool) so no isAlive stub is needed yet.
+ when(containerLauncher.launch(any())).thenReturn(alive, dead);
+ ContainerHandle a1 = pool.acquire(fn);
+ ContainerHandle a2 = pool.acquire(fn);
+ assertSame(alive, a1);
+ assertSame(dead, a2);
+ pool.release(a1);
+ pool.release(a2);
+
+ // dead dies out-of-band, alive is still up.
+ when(containerLauncher.isAlive(dead)).thenReturn(false);
+ when(containerLauncher.isAlive(alive)).thenReturn(true);
+
+ ContainerHandle acquired = pool.acquire(fn);
+ assertSame(alive, acquired);
+ verify(containerLauncher, times(1)).stop(dead);
+ verify(containerLauncher, never()).stop(alive);
+ // Only the original two cold starts; no extra launch was needed.
+ verify(containerLauncher, times(2)).launch(any());
+
+ pool.shutdown();
+ }
}