Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

perf(lambda): mount function code from a read-only volume instead of copying per cold start (#1673)

Alex Banna47abda8e9f1d
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 16078efda2..355eafd251 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -305,12 +305,14 @@ public class ContainerLifecycleManager {
}
/**
- * Returns whether the container is currently running. A missing container
- * is treated as not-running; any other Docker error is treated as running
- * so a transient daemon hiccup does not evict a healthy warm pool.
+ * Returns whether the container is currently running. A missing container is treated as
+ * not-running; any other Docker error (e.g. an inspect timeout under daemon overload) is also
+ * treated as not-running, so a hung/dead container is not reused from the warm pool — a false
+ * negative merely triggers a clean cold-start, which is far cheaper than blocking until the
+ * function timeout.
*
* @param containerId the container ID to inspect
- * @return true if the container exists and is reported as running
+ * @return true only if the container exists and is reported as running; false on any error
*/
public boolean isContainerRunning(String containerId) {
try {
@@ -319,8 +321,13 @@ public class ContainerLifecycleManager {
} catch (NotFoundException e) {
return false;
} catch (Exception e) {
- LOG.warnv("Liveness check failed for container {0}: {1}", containerId, e.getMessage());
- return true;
+ // Treat an inspect failure/timeout as NOT running. Under Docker-daemon overload,
+ // returning true here caused the warm pool to "reuse" dead/hung containers, so the
+ // invocation blocked until the function timeout (~20-30s) every time. A false
+ // negative merely triggers a clean cold-start, which is far cheaper than a hang.
+ LOG.warnv("Liveness check failed for container {0}; treating as not running: {1}",
+ containerId, e.getMessage());
+ return false;
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
index 991108fbc4..a06cae8d6f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
@@ -146,6 +146,15 @@ public class ContainerLauncher {
// Start Runtime API server first so container can connect on boot
RuntimeApiServer runtimeApiServer = runtimeApiServerFactory.create();
+ // Everything after the runtime-api server is allocated runs inside one try/catch: a failure
+ // ANYWHERE below — image/host resolve, the code-volume populate (ensureCodeVolume), the spec
+ // build, or the create/copy/start — must release that runtime-api port (and reap any
+ // half-built container). Otherwise a cold-start burst that trips the Docker daemon leaks one
+ // runtime-api port per failed attempt and eventually exhausts the pool, so launches keep
+ // failing even after the daemon recovers.
+ String containerId = null;
+ try {
+
// Resolve image
String image = "Image".equals(fn.getPackageType()) && fn.getImageUri() != null
? fn.getImageUri()
@@ -227,8 +236,26 @@ public class ContainerLauncher {
specBuilder.withEmbeddedDns();
+ // Whether /var/task is served from a shared read-only volume (large code) or copied
+ // directly into this container (small code). Decided once here so both the spec (below)
+ // and the create->start copy block agree.
+ boolean useCodeVolume = false;
if (fn.isHotReload()) {
specBuilder.withBind(fn.getHotReloadHostPath(), TASK_DIR);
+ } else if (fn.getCodeLocalPath() != null) {
+ useCodeVolume = shouldUseCodeVolume(Path.of(fn.getCodeLocalPath()));
+ if (useCodeVolume) {
+ // Large code: mount /var/task from a read-only volume that is populated ONCE per
+ // code version, instead of tar-copying its code (e.g. ~34k node_modules files) into
+ // every container. That copy cost ~95s per cold start on Docker Desktop (small-file
+ // overlay I/O) and, run many-at-once, saturated the daemon so even `docker create`
+ // ballooned to ~80s. A pre-populated volume mounts in ~0.2s and is shared read-only
+ // by all containers of the function — matching AWS, where /var/task is read-only.
+ String codeVolume = ensureCodeVolume(fn, image);
+ specBuilder.withNamedVolume(codeVolume, TASK_DIR, true);
+ }
+ // Small code takes the original per-container direct copy below (no volume): it's fast
+ // enough that a shared volume's helper round-trip would only add cold-start latency.
}
// For Image package type use ImageConfig.Command/EntryPoint/WorkingDirectory if set, otherwise fall back to Handler (Zip-style)
@@ -259,7 +286,7 @@ public class ContainerLauncher {
// Create container without starting — provided.* runtimes exec
// /var/runtime/bootstrap on start, so code must be copied first.
- String containerId = lifecycleManager.create(spec);
+ containerId = lifecycleManager.create(spec);
LOG.infov("Created container {0} for function {1}", containerId, fn.getFunctionName());
// Copy code into container via Docker API tar stream (works inside Docker too).
@@ -268,10 +295,15 @@ public class ContainerLauncher {
if (!fn.isHotReload() && fn.getCodeLocalPath() != null) {
Path codePath = Path.of(fn.getCodeLocalPath());
- // 1. Always copy all code to /var/task (TASK_DIR)
- copyDirToContainer(dockerClient, containerId, codePath, TASK_DIR, fn.getFunctionName());
+ // Large code (useCodeVolume): /var/task is supplied by the read-only named volume
+ // mounted on the spec above (populated once per code version) — no per-container copy.
+ // Small code: copy it directly into /var/task on this container (the original fast path,
+ // cheap enough that a shared volume's helper round-trip would only add cold-start latency).
+ if (!useCodeVolume) {
+ copyDirToContainer(dockerClient, containerId, codePath, TASK_DIR, fn.getFunctionName());
+ }
- // 2. For provided runtimes, also copy the 'bootstrap' file to /var/runtime (RUNTIME_DIR)
+ // For provided runtimes, also copy the 'bootstrap' file to /var/runtime (RUNTIME_DIR)
if (isProvidedRuntime(fn.getRuntime())) {
Path bootstrapPath = codePath.resolve("bootstrap");
if (Files.exists(bootstrapPath)) {
@@ -303,9 +335,12 @@ public class ContainerLauncher {
// 4. Copy Floci's CA cert so the container trusts Floci's HTTPS endpoint (TLS mode).
// Placed before start so NODE_EXTRA_CA_CERTS et al. resolve at runtime init.
- flociCaCert.ifPresent(certPath ->
- copyFileToContainer(dockerClient, containerId, certPath,
- FLOCI_CA_DIR, FLOCI_CA_FILE_NAME, fn.getFunctionName()));
+ // An if-block rather than ifPresent(...) because containerId is assigned along the
+ // code-volume path and so is not effectively final for a lambda capture.
+ if (flociCaCert.isPresent()) {
+ copyFileToContainer(dockerClient, containerId, flociCaCert.get(),
+ FLOCI_CA_DIR, FLOCI_CA_FILE_NAME, fn.getFunctionName());
+ }
// Now start the container with code in place
lifecycleManager.startCreated(containerId, spec);
@@ -318,6 +353,21 @@ public class ContainerLauncher {
handle.setLogStream(logHandle);
return handle;
+ } catch (RuntimeException e) {
+ // Launch failed somewhere after the runtime-api server was allocated — image/host
+ // resolve, the code-volume populate, the spec build, a create/copy/start under Docker
+ // load, or the log-stream attach. Free the runtime-api port (else a cold-start burst
+ // leaks one per attempt and exhausts the pool) and reap any half-built container (leaked
+ // "Created" containers bog the daemon and starve reuse). containerId is null when we
+ // failed before the container was created (e.g. an ensureCodeVolume populate failure).
+ LOG.errorv("Container launch failed for function {0}; cleaning up: {1}",
+ fn.getFunctionName(), e.getMessage());
+ if (containerId != null) {
+ try { lifecycleManager.stopAndRemove(containerId, null); } catch (Exception ignore) { /* best effort */ }
+ }
+ try { runtimeApiServerFactory.release(runtimeApiServer); } catch (Exception ignore) { /* best effort */ }
+ throw e;
+ }
}
public void stop(ContainerHandle handle) {
@@ -347,10 +397,170 @@ public class ContainerLauncher {
return lifecycleManager.isContainerRunning(handle.getContainerId());
}
+ // Cap concurrent code-volume POPULATES. Populating a large function's volume creates a helper
+ // container and streams ~90MB of node_modules into it; a burst of first-time populates run at
+ // once (e.g. a UI page-load firing 6-8 parallel API calls against never-before-seen functions)
+ // overwhelmed the Docker daemon, so copies hung/failed and left half-built "Created" containers.
+ // This gates ONLY the heavy populate — not every launch — so ordinary cold starts (volume mounts
+ // for already-populated large code, or the small-code direct copy) are never serialized.
+ private static final java.util.concurrent.Semaphore POPULATE_SEMAPHORE =
+ new java.util.concurrent.Semaphore(Math.max(2, Runtime.getRuntime().availableProcessors() / 2));
+
+ private static void acquirePopulatePermit(String functionName) {
+ try {
+ POPULATE_SEMAPHORE.acquire();
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while waiting to populate code volume for " + functionName, ie);
+ }
+ }
+
+ // Per-function-version code volumes: populated once, then mounted read-only into every
+ // container so cold starts skip the ~95s node_modules copy. Tracks which volumes are already
+ // populated and a per-volume lock so a concurrent cold-start storm populates only once. The
+ // lock entry is dropped after a successful populate so this map only ever holds in-flight
+ // populates (see ensureCodeVolume) rather than growing across redeploys.
+ private final java.util.Set<String> populatedCodeVolumes = java.util.concurrent.ConcurrentHashMap.newKeySet();
+ private final java.util.concurrent.ConcurrentHashMap<String, Object> codeVolumeLocks = new java.util.concurrent.ConcurrentHashMap<>();
+
+ /**
+ * Returns the name of a read-only Docker volume holding this function's {@code /var/task} code,
+ * populating it once (per code version) on first use. Populating streams the code into a
+ * throwaway helper container that has the volume mounted read-write; every real container then
+ * just mounts the volume read-only, turning a ~95s per-container copy into a ~0.2s mount.
+ */
+ private String ensureCodeVolume(LambdaFunction fn, String image) {
+ String volName = codeVolumeName(fn);
+ if (populatedCodeVolumes.contains(volName)) {
+ return volName;
+ }
+ Object lock = codeVolumeLocks.computeIfAbsent(volName, k -> new Object());
+ synchronized (lock) {
+ if (populatedCodeVolumes.contains(volName)) {
+ return volName;
+ }
+ long t0 = System.currentTimeMillis();
+ LOG.infov("Populating code volume {0} for function {1} (one-time per code version)",
+ volName, fn.getFunctionName());
+ populateCodeVolume(volName, fn, image);
+ populatedCodeVolumes.add(volName);
+ // Bound codeVolumeLocks: drop the lock now that the volume is populated, so the map only
+ // ever holds in-flight populates rather than growing across redeploys. Safe under the
+ // lock: a late thread that computeIfAbsent's a fresh lock object will still hit the
+ // populatedCodeVolumes.contains check inside its synchronized block and return without
+ // re-populating.
+ codeVolumeLocks.remove(volName);
+ LOG.infov("Populated code volume {0} in {1}ms; future cold starts mount it instead of copying",
+ volName, System.currentTimeMillis() - t0);
+ }
+ // We do NOT eagerly delete a function's previous code-version volume here: a concurrent
+ // in-flight launch may have already resolved that name and be about to mount it, so deleting
+ // it would give that container an empty /var/task. Stale volumes are labeled floci=true (see
+ // ContainerLifecycleManager.ensureVolume) and reclaimed by `docker volume prune --filter label=floci`.
+ return volName;
+ }
+
+ private void populateCodeVolume(String volName, LambdaFunction fn, String image) {
+ lifecycleManager.ensureVolume(volName);
+ String shortId = java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8);
+ // A minimal helper container (sleep) with the volume mounted read-write at /var/task; we
+ // tar-copy the code into it, then discard it — the data persists in the volume.
+ ContainerSpec helperSpec = containerBuilder.newContainer(image)
+ .withName("floci-codevol-" + fn.getFunctionName() + "-" + shortId)
+ .withEnv(java.util.List.of())
+ .withEntrypoint(java.util.List.of("sleep"))
+ .withCmd(java.util.List.of("3600"))
+ .withNamedVolume(volName, TASK_DIR, false)
+ .build();
+ // Gate the heavy work (helper create + ~90MB tar copy) so a burst of first-time populates
+ // doesn't thrash the Docker daemon. Only populates are serialized — plain cold starts aren't.
+ acquirePopulatePermit(fn.getFunctionName());
+ String helperId = null;
+ try {
+ helperId = lifecycleManager.create(helperSpec);
+ lifecycleManager.startCreated(helperId, helperSpec);
+ copyDirToContainer(lifecycleManager.getDockerClient(), helperId,
+ Path.of(fn.getCodeLocalPath()), TASK_DIR, fn.getFunctionName());
+ } finally {
+ if (helperId != null) {
+ try { lifecycleManager.stopAndRemove(helperId, null); } catch (Exception ignore) { /* best effort */ }
+ }
+ POPULATE_SEMAPHORE.release();
+ }
+ }
+
+ /**
+ * Total code size (bytes) at or above which a function's {@code /var/task} is served from a
+ * shared read-only volume rather than copied into every container. This is roughly the point
+ * where per-container small-file overlay copies get slow enough (many-file node_modules trees)
+ * that populating a volume once and mounting it read-only wins overall. Below it, the direct
+ * per-container copy is faster than the volume's populate-helper round-trip, so we keep it —
+ * which is what tiny handlers (e.g. WebSocket-route Lambdas) need to avoid cold-start latency.
+ * Non-final and package-private so tests can override it (restore it in a finally).
+ */
+ static long CODE_VOLUME_MIN_BYTES = 32L * 1024 * 1024;
+
+ /**
+ * Returns true iff the total size of files under {@code codeDir} meets or exceeds
+ * {@link #CODE_VOLUME_MIN_BYTES}. Walks the tree short-circuiting as soon as the running total
+ * crosses the threshold (so huge trees aren't fully summed). Any IO error returns false so the
+ * caller falls back to the direct per-container copy.
+ */
+ static boolean shouldUseCodeVolume(Path codeDir) {
+ final long threshold = CODE_VOLUME_MIN_BYTES;
+ final long[] total = {0L};
+ try (var stream = Files.walk(codeDir)) {
+ for (Path path : (Iterable<Path>) stream::iterator) {
+ if (Files.isRegularFile(path)) {
+ total[0] += Files.size(path);
+ if (total[0] >= threshold) {
+ return true;
+ }
+ }
+ }
+ } catch (IOException | RuntimeException e) {
+ LOG.debugv("Could not size code dir {0} ({1}); using direct copy", codeDir, e.getMessage());
+ return false;
+ }
+ return false;
+ }
+
+ /**
+ * Docker-volume-safe name keyed by function + code version, so a redeploy yields a new volume.
+ * Prefers the code SHA-256; falls back to last-modified when the SHA is unavailable.
+ */
+ static String codeVolumeName(LambdaFunction fn) {
+ String key = fn.getCodeSha256();
+ if (key == null || key.isBlank()) {
+ key = Long.toString(fn.getLastModified());
+ }
+ String h = key.replaceAll("[^a-zA-Z0-9]", "");
+ if (h.length() > 20) {
+ h = h.substring(0, 20);
+ }
+ if (h.isEmpty()) {
+ h = "0";
+ }
+ String fname = fn.getFunctionName().replaceAll("[^a-zA-Z0-9_.-]", "-");
+ return "floci-code-" + fname + "-" + h;
+ }
+
+ /**
+ * Buffer for the tar-streaming pipe. The default {@link java.io.PipedInputStream} buffer is
+ * only 1KB, which forces a writer/reader thread hand-off (wait/notify) every 1KB. Streaming a
+ * ~90MB node_modules through that ran at ~0.5MB/s (≈3 min per cold start) — pure synchronization
+ * thrash, not I/O. A large buffer lets the tar writer stream ahead so throughput is bound by the
+ * Docker daemon, not the pipe.
+ */
+ private static final int TAR_PIPE_BUFFER_BYTES = 16 * 1024 * 1024;
+
private void copyDirToContainer(DockerClient dockerClient, String containerId,
Path sourceDir, String remotePath, String functionName) {
+ // No per-copy gating here: the heavy /var/task populate for large code already holds a
+ // POPULATE_SEMAPHORE permit; small-code direct copies and layer copies are light enough
+ // to run unthrottled.
try (java.io.PipedOutputStream pos = new java.io.PipedOutputStream();
- java.io.PipedInputStream pis = new java.io.PipedInputStream(pos)) {
+ java.io.PipedInputStream pis = new java.io.PipedInputStream(pos, TAR_PIPE_BUFFER_BYTES)) {
new Thread(() -> {
try (pos) {
@@ -366,14 +576,16 @@ public class ContainerLauncher {
.exec();
LOG.debugv("Copied directory {0} into container {1} at {2}", sourceDir, containerId, remotePath);
} catch (Exception e) {
- LOG.warnv("Failed to copy directory {0} into container {1}: {2}", sourceDir, containerId, e.getMessage());
+ // Fail loudly so launch() cleans up the half-built container instead of leaking it.
+ throw new RuntimeException("Failed to copy directory " + sourceDir + " into container "
+ + containerId + " for function " + functionName + ": " + e.getMessage(), e);
}
}
private void copyFileToContainer(DockerClient dockerClient, String containerId,
Path sourceFile, String remotePath, String entryName, String functionName) {
try (java.io.PipedOutputStream pos = new java.io.PipedOutputStream();
- java.io.PipedInputStream pis = new java.io.PipedInputStream(pos)) {
+ java.io.PipedInputStream pis = new java.io.PipedInputStream(pos, TAR_PIPE_BUFFER_BYTES)) {
new Thread(() -> {
try (TarArchiveOutputStream tar = newTarStream(pos)) {
@@ -396,7 +608,8 @@ public class ContainerLauncher {
.exec();
LOG.debugv("Copied file {0} as {1} into container {2} at {3}", sourceFile, entryName, containerId, remotePath);
} catch (Exception e) {
- LOG.warnv("Failed to copy file {0} into container {1}: {2}", sourceFile, containerId, e.getMessage());
+ throw new RuntimeException("Failed to copy file " + sourceFile + " into container "
+ + containerId + " for function " + functionName + ": " + e.getMessage(), e);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
index 144d881981..451cc44b41 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
@@ -14,6 +14,8 @@ import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;
import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import com.github.dockerjava.api.model.Mount;
+import com.github.dockerjava.api.model.MountType;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -30,6 +32,9 @@ import java.util.Map;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
@@ -89,11 +94,13 @@ class ContainerLauncherTest {
when(runtimeApiServer.getPort()).thenReturn(9000);
when(dockerHostResolver.resolve()).thenReturn("127.0.0.1");
- when(lifecycleManager.create(any())).thenReturn("container-123");
+ // lenient: the failure-path test (populate fails before any container is created) never
+ // reaches these, but every success-path test does — they must not trip strict-stubs.
+ lenient().when(lifecycleManager.create(any())).thenReturn("container-123");
ContainerLifecycleManager.ContainerInfo info =
new ContainerLifecycleManager.ContainerInfo("container-123", Map.of());
- when(lifecycleManager.startCreated(eq("container-123"), any())).thenReturn(info);
- when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ lenient().when(lifecycleManager.startCreated(eq("container-123"), any())).thenReturn(info);
+ lenient().when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
// Stub the Docker copy chain so copyDirToContainer / copyFileToContainer
// don't throw when the mock DockerClient is used. Each invocation
@@ -123,8 +130,44 @@ class ContainerLauncherTest {
});
}
+ /**
+ * Captures every {@link ContainerSpec} passed to {@code lifecycleManager.create(...)} and
+ * returns the REAL Lambda container's spec.
+ *
+ * <p>Small code (below {@link ContainerLauncher#CODE_VOLUME_MIN_BYTES}, the case for these
+ * tempdir-backed tests) is copied directly into {@code /var/task} on the real container, so
+ * {@code create} is called exactly once. Large code is instead served from a read-only named
+ * volume populated by a throwaway helper container (also via {@code create}), so {@code create}
+ * is called twice: the helper first, then the real container. The real container is the one that
+ * mounts {@code /var/task} read-only from the volume, so we identify it by that mount when
+ * present, otherwise fall back to the last (only) {@code create}.
+ */
+ private ContainerSpec captureRealContainerSpec() {
+ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
+ verify(lifecycleManager, atLeastOnce()).create(specCaptor.capture());
+ List<ContainerSpec> specs = specCaptor.getAllValues();
+ // The real container (volume path) mounts /var/task read-only; the helper mounts it read-write.
+ return specs.stream()
+ .filter(s -> s.mounts() != null && s.mounts().stream()
+ .anyMatch(m -> "/var/task".equals(m.getTarget()) && Boolean.TRUE.equals(m.getReadOnly())))
+ .reduce((first, second) -> second) // last match, defensively
+ // Fall back to the last create() (the real container) for the direct-copy path.
+ .orElseGet(() -> specs.get(specs.size() - 1));
+ }
+
+ /** Returns the read-only {@code /var/task} volume mount on the spec, or null if absent. */
+ private static Mount varTaskVolumeMount(ContainerSpec spec) {
+ if (spec.mounts() == null) {
+ return null;
+ }
+ return spec.mounts().stream()
+ .filter(m -> m.getType() == MountType.VOLUME && "/var/task".equals(m.getTarget()))
+ .findFirst()
+ .orElse(null);
+ }
+
@Test
- void launchFunction_createsWithoutBindMounts() throws Exception {
+ void launchFunction_createsWithoutBindMountsOrVolume_forSmallCode() throws Exception {
Path codePath = Files.createDirectory(tempDir.resolve("code"));
LambdaFunction fn = new LambdaFunction();
@@ -135,11 +178,14 @@ class ContainerLauncherTest {
launcher.launch(fn);
- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
- verify(lifecycleManager).create(specCaptor.capture());
-
- ContainerSpec spec = specCaptor.getValue();
+ ContainerSpec spec = captureRealContainerSpec();
+ // Small code is copied directly into /var/task (the fast path), NOT bind-mounted...
assertTrue(spec.binds().isEmpty(), "Function should NOT have bind mounts");
+ // ...and NOT served from a named volume (that's reserved for large code).
+ assertNull(varTaskVolumeMount(spec), "small code should NOT mount /var/task from a volume");
+ // The code is tar-copied straight into /var/task on the real container.
+ assertTrue(capturedRemotePaths.contains("/var/task"),
+ "small code should be copied directly into /var/task");
}
@Test
@@ -154,13 +200,24 @@ class ContainerLauncherTest {
launcher.launch(fn);
- // Verify ordering: create → getDockerClient → Docker copy (to /var/task) → startCreated
+ // Small code takes the direct-copy path: the real container is created, its code is
+ // tar-copied straight into /var/task, and only then is it started. No populate helper.
+ // create(real) -> copy /var/task (real) -> start(real).
InOrder inOrder = inOrder(lifecycleManager, dockerClient);
inOrder.verify(lifecycleManager).create(any());
- inOrder.verify(lifecycleManager).getDockerClient();
inOrder.verify(dockerClient).copyArchiveToContainerCmd("container-123");
inOrder.verify(lifecycleManager).startCreated(eq("container-123"), any());
+ // The code is copied to /var/task on the real container (no helper populate for small code).
+ assertTrue(capturedRemotePaths.contains("/var/task"),
+ "small code should be tar-copied directly to /var/task");
+ assertEquals(1, capturedRemotePaths.stream().filter("/var/task"::equals).count(),
+ "/var/task should be copied exactly once (the direct per-container copy)");
+
+ // No populate helper is created/discarded for small code.
+ verify(lifecycleManager, times(1)).create(any());
+ verify(lifecycleManager, never()).stopAndRemove(any(), any());
+
// createAndStart must NOT be called — Lambda uses the split path
verify(lifecycleManager, never()).createAndStart(any());
}
@@ -177,10 +234,7 @@ class ContainerLauncherTest {
launcher.launch(fn);
- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
- verify(lifecycleManager).create(specCaptor.capture());
-
- List<String> env = specCaptor.getValue().env();
+ List<String> env = captureRealContainerSpec().env();
assertTrue(env.stream().anyMatch(e -> e.startsWith("AWS_ACCESS_KEY_ID=")),
"AWS_ACCESS_KEY_ID should be injected when awsConfigPath is absent");
assertTrue(env.stream().anyMatch(e -> e.startsWith("AWS_SECRET_ACCESS_KEY=")),
@@ -204,10 +258,7 @@ class ContainerLauncherTest {
launcher.launch(fn);
- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
- verify(lifecycleManager).create(specCaptor.capture());
-
- List<String> env = specCaptor.getValue().env();
+ List<String> env = captureRealContainerSpec().env();
String accessKey = env.stream().filter(e -> e.startsWith("AWS_ACCESS_KEY_ID=")).findFirst().orElse("");
String secretKey = env.stream().filter(e -> e.startsWith("AWS_SECRET_ACCESS_KEY=")).findFirst().orElse("");
String sessionToken = env.stream().filter(e -> e.startsWith("AWS_SESSION_TOKEN=")).findFirst().orElse("");
@@ -235,10 +286,7 @@ class ContainerLauncherTest {
launcher.launch(fn);
- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
- verify(lifecycleManager).create(specCaptor.capture());
-
- List<String> env = specCaptor.getValue().env();
+ List<String> env = captureRealContainerSpec().env();
assertTrue(env.contains("AWS_DEFAULT_REGION=eu-central-1"));
assertTrue(env.contains("AWS_REGION=eu-central-1"));
}
@@ -256,10 +304,7 @@ class ContainerLauncherTest {
launcher.launch(fn);
- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
- verify(lifecycleManager).create(specCaptor.capture());
-
- List<String> env = specCaptor.getValue().env();
+ List<String> env = captureRealContainerSpec().env();
assertTrue(env.contains("AWS_DEFAULT_REGION=eu-west-2"));
assertTrue(env.contains("AWS_REGION=eu-west-2"));
verify(logStreamer).attach(
@@ -281,10 +326,7 @@ class ContainerLauncherTest {
launcher.launch(fn);
- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
- verify(lifecycleManager).create(specCaptor.capture());
-
- List<String> env = specCaptor.getValue().env();
+ List<String> env = captureRealContainerSpec().env();
// Docker honours the last occurrence of a duplicate Env entry, so user
// overrides must appear after the Floci defaults.
int defaultKeyIdx = -1;
@@ -325,12 +367,11 @@ class ContainerLauncherTest {
launcher.launch(fn);
- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
- verify(lifecycleManager).create(specCaptor.capture());
+ ContainerSpec spec = captureRealContainerSpec();
verify(ecrRegistryManager).ensureStarted();
verify(ecrRegistryManager).getRepositoryUri("123456789012", "us-east-1", "backend-user:1");
assertEquals("123456789012.dkr.ecr.us-east-1.localhost:5100/backend-user:1",
- specCaptor.getValue().image());
+ spec.image());
}
@Test
@@ -345,12 +386,11 @@ class ContainerLauncherTest {
launcher.launch(fn);
- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
- verify(lifecycleManager).create(specCaptor.capture());
+ ContainerSpec spec = captureRealContainerSpec();
verify(ecrRegistryManager).ensureStarted();
verify(ecrRegistryManager).getRepositoryUri("123456789012", "us-east-1", "backend-user:1");
assertEquals("localhost:5100/123456789012/us-east-1/backend-user:1",
- specCaptor.getValue().image());
+ spec.image());
}
@Test
@@ -366,21 +406,28 @@ class ContainerLauncherTest {
launcher.launch(fn);
- // The critical invariant: create must happen before any Docker copy,
- // and start must happen after. This is the exact regression from #466.
+ // The critical invariant preserved from #466: the REAL container must be created before its
+ // code and bootstrap are copied, and started only after. Small code takes the direct-copy
+ // path, so everything happens on the one real container (no populate helper).
+ //
+ // Ordering:
+ // real: create -> copy /var/task + copy bootstrap to /var/runtime -> start
+ // (two copyArchiveToContainerCmd calls on the real container: code, then bootstrap).
InOrder inOrder = inOrder(lifecycleManager, dockerClient);
inOrder.verify(lifecycleManager).create(any());
- inOrder.verify(lifecycleManager).getDockerClient();
- // Two copies: code to /var/task + bootstrap to /var/runtime
- inOrder.verify(dockerClient, times(2)).copyArchiveToContainerCmd("container-123");
+ inOrder.verify(dockerClient, atLeastOnce()).copyArchiveToContainerCmd("container-123");
inOrder.verify(lifecycleManager).startCreated(eq("container-123"), any());
- // Verify both /var/task and /var/runtime were targeted
+ // Small code is copied directly to /var/task; bootstrap is copied to /var/runtime — both on
+ // the one real container.
assertTrue(capturedRemotePaths.contains("/var/task"),
- "code should be copied to /var/task");
+ "small code should be tar-copied directly to /var/task");
assertTrue(capturedRemotePaths.contains("/var/runtime"),
- "bootstrap should be copied to /var/runtime");
+ "bootstrap should be copied to /var/runtime on the real container");
+ // No populate helper for small code.
+ verify(lifecycleManager, times(1)).create(any());
+ verify(lifecycleManager, never()).stopAndRemove(any(), any());
verify(lifecycleManager, never()).createAndStart(any());
}
@@ -399,10 +446,7 @@ class ContainerLauncherTest {
launcher.launch(fn);
- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
- verify(lifecycleManager).create(specCaptor.capture());
-
- ContainerSpec spec = specCaptor.getValue();
+ ContainerSpec spec = captureRealContainerSpec();
// Should bind-mount to /opt/aws-config (read-only)
assertTrue(spec.binds().stream()
@@ -439,11 +483,84 @@ class ContainerLauncherTest {
launcher.launch(fn);
- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
- verify(lifecycleManager).create(specCaptor.capture());
-
- assertTrue(specCaptor.getValue().binds().stream()
+ assertTrue(captureRealContainerSpec().binds().stream()
.noneMatch(b -> b.getVolume().getPath().equals("/opt/aws-config")),
"no .aws bind mount when awsConfigPath is absent");
}
+
+ @Test
+ void launchFunction_usesReadOnlyVolume_forLargeCode() throws Exception {
+ Path codePath = Files.createDirectory(tempDir.resolve("large-code"));
+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("large-fn");
+ fn.setRuntime("nodejs20.x");
+ fn.setHandler("index.handler");
+ fn.setCodeLocalPath(codePath.toString());
+ fn.setCodeSha256("large-fn-sha-v1");
+
+ long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ try {
+ // Force the volume path without writing a 32 MiB file.
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024; // 4 KiB
+ launcher.launch(fn);
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;
+ }
+
+ ContainerSpec spec = captureRealContainerSpec();
+ // The real container mounts /var/task read-only from the code volume...
+ Mount codeMount = varTaskVolumeMount(spec);
+ assertNotNull(codeMount, "large code: /var/task should be a named-volume mount");
+ assertEquals(MountType.VOLUME, codeMount.getType(), "/var/task should be a volume mount");
+ assertTrue(Boolean.TRUE.equals(codeMount.getReadOnly()), "/var/task volume should be read-only");
+ assertTrue(spec.binds().isEmpty(), "large code: real container should have NO bind mounts");
+
+ // ...and /var/task is populated ONCE via a helper container (create -> start -> copy ->
+ // stopAndRemove), not copied onto the real container. The volume is populated exactly once.
+ assertEquals(1, capturedRemotePaths.stream().filter("/var/task"::equals).count(),
+ "/var/task should be copied exactly once (into the populate helper)");
+ // Two creates: the helper + the real container. The helper is discarded.
+ verify(lifecycleManager, times(2)).create(any());
+ verify(lifecycleManager, atLeastOnce()).ensureVolume(any());
+ verify(lifecycleManager, times(1)).stopAndRemove(any(), any()); // the helper only
+ // Old code-version volumes are NOT eagerly deleted (race fix); they are label-pruned instead.
+ verify(lifecycleManager, never()).removeVolume(any());
+ }
+
+ @Test
+ void launchFunction_releasesRuntimeApiServer_whenCodeVolumePopulateFails() throws Exception {
+ // Regression for the runtime-api port leak: the volume path allocates a runtime-api server
+ // up front (before the code-volume populate). If the populate then fails — the exact
+ // cold-start-burst scenario this PR targets, where the Docker daemon rejects the helper work
+ // under load — the launch must STILL release that port. Otherwise every failed attempt leaks
+ // one port and the pool eventually exhausts, so launches keep failing after the daemon recovers.
+ Path codePath = Files.createDirectory(tempDir.resolve("leak-code"));
+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("leak-fn");
+ fn.setRuntime("nodejs20.x");
+ fn.setHandler("index.handler");
+ fn.setCodeLocalPath(codePath.toString());
+ fn.setCodeSha256("leak-fn-sha-v1");
+
+ // The code-volume populate fails (daemon under load), before any container is created.
+ doThrow(new RuntimeException("docker daemon busy")).when(lifecycleManager).ensureVolume(any());
+
+ long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ try {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024; // force the volume path without a 32 MiB file
+ assertThrows(RuntimeException.class, () -> launcher.launch(fn));
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;
+ }
+
+ // The runtime-api port allocated before the populate is released on this failure path...
+ verify(runtimeApiServerFactory).release(runtimeApiServer);
+ // ...and we bailed before creating or starting any container (nothing to reap).
+ verify(lifecycleManager, never()).create(any());
+ verify(lifecycleManager, never()).startCreated(any(), any());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherVolumeNamingTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherVolumeNamingTest.java
new file mode 100644
index 0000000000..bb0c323447
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherVolumeNamingTest.java
@@ -0,0 +1,133 @@
+package io.github.hectorvent.floci.services.lambda.launcher;
+
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.regex.Pattern;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Unit tests for {@link ContainerLauncher#codeVolumeName(LambdaFunction)}, the naming logic for
+ * the per-function-version code volume. The name must be stable for a given code version (so all
+ * of a function's containers share one volume), distinct across code versions (so a redeploy gets
+ * a fresh volume), and always a legal Docker volume name.
+ *
+ * <p>Kept separate from {@link ContainerLauncherTest} because these are pure static-method tests
+ * that need none of that test's Mockito mocks; mixing them in would trip strict-stubbing.
+ */
+class ContainerLauncherVolumeNamingTest {
+
+ /** The pattern Docker requires a volume name to match. */
+ private static final Pattern DOCKER_VOLUME_NAME = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9_.-]*$");
+
+ private static LambdaFunction fnWithSha(String name, String sha) {
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName(name);
+ fn.setCodeSha256(sha);
+ return fn;
+ }
+
+ @Test
+ void isFlociCodeShapedAndDockerSafe() {
+ String name = ContainerLauncher.codeVolumeName(
+ fnWithSha("my-fn", "AbC123+/xyz=deadbeefcafebabe0123456789"));
+
+ assertTrue(name.startsWith("floci-code-my-fn-"),
+ "should be floci-code-<functionName>-<hash> shaped, was: " + name);
+ assertTrue(DOCKER_VOLUME_NAME.matcher(name).matches(),
+ "must be a docker-volume-safe name, was: " + name);
+ }
+
+ @Test
+ void sanitizesUnsafeFunctionNameChars() {
+ // Real function names are constrained, but be defensive: any unsafe char must not leak
+ // into the volume name (which would make `docker volume create` reject it).
+ String name = ContainerLauncher.codeVolumeName(fnWithSha("weird name/v2:1", "abc123"));
+ assertTrue(DOCKER_VOLUME_NAME.matcher(name).matches(),
+ "unsafe function-name chars must be sanitized, was: " + name);
+ }
+
+ @Test
+ void differsAcrossCodeVersions() {
+ String v1 = ContainerLauncher.codeVolumeName(fnWithSha("same-fn", "sha-version-one"));
+ String v2 = ContainerLauncher.codeVolumeName(fnWithSha("same-fn", "sha-version-two"));
+
+ assertNotEquals(v1, v2,
+ "different codeSha256 must yield different volume names so a redeploy gets a fresh volume");
+ }
+
+ @Test
+ void isStableForSameFunctionAndSha() {
+ String a = ContainerLauncher.codeVolumeName(fnWithSha("stable-fn", "stable-sha-abc"));
+ String b = ContainerLauncher.codeVolumeName(fnWithSha("stable-fn", "stable-sha-abc"));
+
+ assertEquals(a, b, "same function + same sha must produce a stable volume name");
+ }
+
+ @Test
+ void fallsBackToLastModifiedWhenShaMissing() {
+ LambdaFunction nullSha = new LambdaFunction();
+ nullSha.setFunctionName("no-sha-fn");
+ nullSha.setCodeSha256(null);
+ nullSha.setLastModified(1700000000000L);
+
+ LambdaFunction blankSha = new LambdaFunction();
+ blankSha.setFunctionName("no-sha-fn");
+ blankSha.setCodeSha256(" ");
+ blankSha.setLastModified(1700000000000L);
+
+ String nullName = ContainerLauncher.codeVolumeName(nullSha);
+ String blankName = ContainerLauncher.codeVolumeName(blankSha);
+
+ assertTrue(DOCKER_VOLUME_NAME.matcher(nullName).matches(),
+ "must be docker-safe even when falling back to lastModified, was: " + nullName);
+ assertTrue(nullName.contains("1700000000000"),
+ "null sha should key off lastModified, was: " + nullName);
+ assertEquals(nullName, blankName, "null and blank sha should both fall back to lastModified");
+
+ // A different lastModified (new deploy) must yield a different name.
+ LambdaFunction newer = new LambdaFunction();
+ newer.setFunctionName("no-sha-fn");
+ newer.setLastModified(1700000009999L);
+ assertNotEquals(nullName, ContainerLauncher.codeVolumeName(newer),
+ "a later lastModified must produce a different volume name");
+ }
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ void shouldUseCodeVolume_falseForSmallDir_trueWhenOverThreshold() throws Exception {
+ Path smallDir = Files.createDirectory(tempDir.resolve("small"));
+ Files.write(smallDir.resolve("a.txt"), new byte[1024]); // 1 KiB
+
+ Path bigDir = Files.createDirectory(tempDir.resolve("big"));
+ Files.write(bigDir.resolve("a.bin"), new byte[8 * 1024]); // 8 KiB
+
+ long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ try {
+ // Override the threshold small so we don't have to write huge files.
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024; // 4 KiB
+ assertFalse(ContainerLauncher.shouldUseCodeVolume(smallDir),
+ "1 KiB dir is below the 4 KiB threshold");
+ assertTrue(ContainerLauncher.shouldUseCodeVolume(bigDir),
+ "8 KiB dir exceeds the 4 KiB threshold");
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;
+ }
+ }
+
+ @Test
+ void shouldUseCodeVolume_falseWhenDirMissing() {
+ // IO errors (e.g. a non-existent dir) fall back to the direct copy.
+ assertFalse(ContainerLauncher.shouldUseCodeVolume(tempDir.resolve("does-not-exist")),
+ "a missing code dir should fall back to direct copy (false)");
+ }
+}