Light, fluffy, and always free - The AWS Local Emulator alternative
perf(lambda): mount function code from a read-only volume instead of copying per cold start (#1673)
Alex Banna47abda8e9f1d
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 16078efda2..355eafd251 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -305,12 +305,14 @@ public class ContainerLifecycleManager {}/**- * Returns whether the container is currently running. A missing container- * is treated as not-running; any other Docker error is treated as running- * so a transient daemon hiccup does not evict a healthy warm pool.+ * Returns whether the container is currently running. A missing container is treated as+ * not-running; any other Docker error (e.g. an inspect timeout under daemon overload) is also+ * treated as not-running, so a hung/dead container is not reused from the warm pool — a false+ * negative merely triggers a clean cold-start, which is far cheaper than blocking until the+ * function timeout.** @param containerId the container ID to inspect- * @return true if the container exists and is reported as running+ * @return true only if the container exists and is reported as running; false on any error*/public boolean isContainerRunning(String containerId) {try {@@ -319,8 +321,13 @@ public class ContainerLifecycleManager {} catch (NotFoundException e) {return false;} catch (Exception e) {- LOG.warnv("Liveness check failed for container {0}: {1}", containerId, e.getMessage());- return true;+ // Treat an inspect failure/timeout as NOT running. Under Docker-daemon overload,+ // returning true here caused the warm pool to "reuse" dead/hung containers, so the+ // invocation blocked until the function timeout (~20-30s) every time. A false+ // negative merely triggers a clean cold-start, which is far cheaper than a hang.+ LOG.warnv("Liveness check failed for container {0}; treating as not running: {1}",+ containerId, e.getMessage());+ return false;}}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex 991108fbc4..a06cae8d6f 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -146,6 +146,15 @@ public class ContainerLauncher {// Start Runtime API server first so container can connect on bootRuntimeApiServer runtimeApiServer = runtimeApiServerFactory.create();+ // Everything after the runtime-api server is allocated runs inside one try/catch: a failure+ // ANYWHERE below — image/host resolve, the code-volume populate (ensureCodeVolume), the spec+ // build, or the create/copy/start — must release that runtime-api port (and reap any+ // half-built container). Otherwise a cold-start burst that trips the Docker daemon leaks one+ // runtime-api port per failed attempt and eventually exhausts the pool, so launches keep+ // failing even after the daemon recovers.+ String containerId = null;+ try {+// Resolve imageString image = "Image".equals(fn.getPackageType()) && fn.getImageUri() != null? fn.getImageUri()@@ -227,8 +236,26 @@ public class ContainerLauncher {specBuilder.withEmbeddedDns();+ // Whether /var/task is served from a shared read-only volume (large code) or copied+ // directly into this container (small code). Decided once here so both the spec (below)+ // and the create->start copy block agree.+ boolean useCodeVolume = false;if (fn.isHotReload()) {specBuilder.withBind(fn.getHotReloadHostPath(), TASK_DIR);+ } else if (fn.getCodeLocalPath() != null) {+ useCodeVolume = shouldUseCodeVolume(Path.of(fn.getCodeLocalPath()));+ if (useCodeVolume) {+ // Large code: mount /var/task from a read-only volume that is populated ONCE per+ // code version, instead of tar-copying its code (e.g. ~34k node_modules files) into+ // every container. That copy cost ~95s per cold start on Docker Desktop (small-file+ // overlay I/O) and, run many-at-once, saturated the daemon so even `docker create`+ // ballooned to ~80s. A pre-populated volume mounts in ~0.2s and is shared read-only+ // by all containers of the function — matching AWS, where /var/task is read-only.+ String codeVolume = ensureCodeVolume(fn, image);+ specBuilder.withNamedVolume(codeVolume, TASK_DIR, true);+ }+ // Small code takes the original per-container direct copy below (no volume): it's fast+ // enough that a shared volume's helper round-trip would only add cold-start latency.}// For Image package type use ImageConfig.Command/EntryPoint/WorkingDirectory if set, otherwise fall back to Handler (Zip-style)@@ -259,7 +286,7 @@ public class ContainerLauncher {// Create container without starting — provided.* runtimes exec// /var/runtime/bootstrap on start, so code must be copied first.- String containerId = lifecycleManager.create(spec);+ containerId = lifecycleManager.create(spec);LOG.infov("Created container {0} for function {1}", containerId, fn.getFunctionName());// Copy code into container via Docker API tar stream (works inside Docker too).@@ -268,10 +295,15 @@ public class ContainerLauncher {if (!fn.isHotReload() && fn.getCodeLocalPath() != null) {Path codePath = Path.of(fn.getCodeLocalPath());- // 1. Always copy all code to /var/task (TASK_DIR)- copyDirToContainer(dockerClient, containerId, codePath, TASK_DIR, fn.getFunctionName());+ // Large code (useCodeVolume): /var/task is supplied by the read-only named volume+ // mounted on the spec above (populated once per code version) — no per-container copy.+ // Small code: copy it directly into /var/task on this container (the original fast path,+ // cheap enough that a shared volume's helper round-trip would only add cold-start latency).+ if (!useCodeVolume) {+ copyDirToContainer(dockerClient, containerId, codePath, TASK_DIR, fn.getFunctionName());+ }- // 2. For provided runtimes, also copy the 'bootstrap' file to /var/runtime (RUNTIME_DIR)+ // For provided runtimes, also copy the 'bootstrap' file to /var/runtime (RUNTIME_DIR)if (isProvidedRuntime(fn.getRuntime())) {Path bootstrapPath = codePath.resolve("bootstrap");if (Files.exists(bootstrapPath)) {@@ -303,9 +335,12 @@ public class ContainerLauncher {// 4. Copy Floci's CA cert so the container trusts Floci's HTTPS endpoint (TLS mode).// Placed before start so NODE_EXTRA_CA_CERTS et al. resolve at runtime init.- flociCaCert.ifPresent(certPath ->- copyFileToContainer(dockerClient, containerId, certPath,- FLOCI_CA_DIR, FLOCI_CA_FILE_NAME, fn.getFunctionName()));+ // An if-block rather than ifPresent(...) because containerId is assigned along the+ // code-volume path and so is not effectively final for a lambda capture.+ if (flociCaCert.isPresent()) {+ copyFileToContainer(dockerClient, containerId, flociCaCert.get(),+ FLOCI_CA_DIR, FLOCI_CA_FILE_NAME, fn.getFunctionName());+ }// Now start the container with code in placelifecycleManager.startCreated(containerId, spec);@@ -318,6 +353,21 @@ public class ContainerLauncher {handle.setLogStream(logHandle);return handle;+ } catch (RuntimeException e) {+ // Launch failed somewhere after the runtime-api server was allocated — image/host+ // resolve, the code-volume populate, the spec build, a create/copy/start under Docker+ // load, or the log-stream attach. Free the runtime-api port (else a cold-start burst+ // leaks one per attempt and exhausts the pool) and reap any half-built container (leaked+ // "Created" containers bog the daemon and starve reuse). containerId is null when we+ // failed before the container was created (e.g. an ensureCodeVolume populate failure).+ LOG.errorv("Container launch failed for function {0}; cleaning up: {1}",+ fn.getFunctionName(), e.getMessage());+ if (containerId != null) {+ try { lifecycleManager.stopAndRemove(containerId, null); } catch (Exception ignore) { /* best effort */ }+ }+ try { runtimeApiServerFactory.release(runtimeApiServer); } catch (Exception ignore) { /* best effort */ }+ throw e;+ }}public void stop(ContainerHandle handle) {@@ -347,10 +397,170 @@ public class ContainerLauncher {return lifecycleManager.isContainerRunning(handle.getContainerId());}+ // Cap concurrent code-volume POPULATES. Populating a large function's volume creates a helper+ // container and streams ~90MB of node_modules into it; a burst of first-time populates run at+ // once (e.g. a UI page-load firing 6-8 parallel API calls against never-before-seen functions)+ // overwhelmed the Docker daemon, so copies hung/failed and left half-built "Created" containers.+ // This gates ONLY the heavy populate — not every launch — so ordinary cold starts (volume mounts+ // for already-populated large code, or the small-code direct copy) are never serialized.+ private static final java.util.concurrent.Semaphore POPULATE_SEMAPHORE =+ new java.util.concurrent.Semaphore(Math.max(2, Runtime.getRuntime().availableProcessors() / 2));++ private static void acquirePopulatePermit(String functionName) {+ try {+ POPULATE_SEMAPHORE.acquire();+ } catch (InterruptedException ie) {+ Thread.currentThread().interrupt();+ throw new RuntimeException("Interrupted while waiting to populate code volume for " + functionName, ie);+ }+ }++ // Per-function-version code volumes: populated once, then mounted read-only into every+ // container so cold starts skip the ~95s node_modules copy. Tracks which volumes are already+ // populated and a per-volume lock so a concurrent cold-start storm populates only once. The+ // lock entry is dropped after a successful populate so this map only ever holds in-flight+ // populates (see ensureCodeVolume) rather than growing across redeploys.+ private final java.util.Set<String> populatedCodeVolumes = java.util.concurrent.ConcurrentHashMap.newKeySet();+ private final java.util.concurrent.ConcurrentHashMap<String, Object> codeVolumeLocks = new java.util.concurrent.ConcurrentHashMap<>();++ /**+ * Returns the name of a read-only Docker volume holding this function's {@code /var/task} code,+ * populating it once (per code version) on first use. Populating streams the code into a+ * throwaway helper container that has the volume mounted read-write; every real container then+ * just mounts the volume read-only, turning a ~95s per-container copy into a ~0.2s mount.+ */+ private String ensureCodeVolume(LambdaFunction fn, String image) {+ String volName = codeVolumeName(fn);+ if (populatedCodeVolumes.contains(volName)) {+ return volName;+ }+ Object lock = codeVolumeLocks.computeIfAbsent(volName, k -> new Object());+ synchronized (lock) {+ if (populatedCodeVolumes.contains(volName)) {+ return volName;+ }+ long t0 = System.currentTimeMillis();+ LOG.infov("Populating code volume {0} for function {1} (one-time per code version)",+ volName, fn.getFunctionName());+ populateCodeVolume(volName, fn, image);+ populatedCodeVolumes.add(volName);+ // Bound codeVolumeLocks: drop the lock now that the volume is populated, so the map only+ // ever holds in-flight populates rather than growing across redeploys. Safe under the+ // lock: a late thread that computeIfAbsent's a fresh lock object will still hit the+ // populatedCodeVolumes.contains check inside its synchronized block and return without+ // re-populating.+ codeVolumeLocks.remove(volName);+ LOG.infov("Populated code volume {0} in {1}ms; future cold starts mount it instead of copying",+ volName, System.currentTimeMillis() - t0);+ }+ // We do NOT eagerly delete a function's previous code-version volume here: a concurrent+ // in-flight launch may have already resolved that name and be about to mount it, so deleting+ // it would give that container an empty /var/task. Stale volumes are labeled floci=true (see+ // ContainerLifecycleManager.ensureVolume) and reclaimed by `docker volume prune --filter label=floci`.+ return volName;+ }++ private void populateCodeVolume(String volName, LambdaFunction fn, String image) {+ lifecycleManager.ensureVolume(volName);+ String shortId = java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8);+ // A minimal helper container (sleep) with the volume mounted read-write at /var/task; we+ // tar-copy the code into it, then discard it — the data persists in the volume.+ ContainerSpec helperSpec = containerBuilder.newContainer(image)+ .withName("floci-codevol-" + fn.getFunctionName() + "-" + shortId)+ .withEnv(java.util.List.of())+ .withEntrypoint(java.util.List.of("sleep"))+ .withCmd(java.util.List.of("3600"))+ .withNamedVolume(volName, TASK_DIR, false)+ .build();+ // Gate the heavy work (helper create + ~90MB tar copy) so a burst of first-time populates+ // doesn't thrash the Docker daemon. Only populates are serialized — plain cold starts aren't.+ acquirePopulatePermit(fn.getFunctionName());+ String helperId = null;+ try {+ helperId = lifecycleManager.create(helperSpec);+ lifecycleManager.startCreated(helperId, helperSpec);+ copyDirToContainer(lifecycleManager.getDockerClient(), helperId,+ Path.of(fn.getCodeLocalPath()), TASK_DIR, fn.getFunctionName());+ } finally {+ if (helperId != null) {+ try { lifecycleManager.stopAndRemove(helperId, null); } catch (Exception ignore) { /* best effort */ }+ }+ POPULATE_SEMAPHORE.release();+ }+ }++ /**+ * Total code size (bytes) at or above which a function's {@code /var/task} is served from a+ * shared read-only volume rather than copied into every container. This is roughly the point+ * where per-container small-file overlay copies get slow enough (many-file node_modules trees)+ * that populating a volume once and mounting it read-only wins overall. Below it, the direct+ * per-container copy is faster than the volume's populate-helper round-trip, so we keep it —+ * which is what tiny handlers (e.g. WebSocket-route Lambdas) need to avoid cold-start latency.+ * Non-final and package-private so tests can override it (restore it in a finally).+ */+ static long CODE_VOLUME_MIN_BYTES = 32L * 1024 * 1024;++ /**+ * Returns true iff the total size of files under {@code codeDir} meets or exceeds+ * {@link #CODE_VOLUME_MIN_BYTES}. Walks the tree short-circuiting as soon as the running total+ * crosses the threshold (so huge trees aren't fully summed). Any IO error returns false so the+ * caller falls back to the direct per-container copy.+ */+ static boolean shouldUseCodeVolume(Path codeDir) {+ final long threshold = CODE_VOLUME_MIN_BYTES;+ final long[] total = {0L};+ try (var stream = Files.walk(codeDir)) {+ for (Path path : (Iterable<Path>) stream::iterator) {+ if (Files.isRegularFile(path)) {+ total[0] += Files.size(path);+ if (total[0] >= threshold) {+ return true;+ }+ }+ }+ } catch (IOException | RuntimeException e) {+ LOG.debugv("Could not size code dir {0} ({1}); using direct copy", codeDir, e.getMessage());+ return false;+ }+ return false;+ }++ /**+ * Docker-volume-safe name keyed by function + code version, so a redeploy yields a new volume.+ * Prefers the code SHA-256; falls back to last-modified when the SHA is unavailable.+ */+ static String codeVolumeName(LambdaFunction fn) {+ String key = fn.getCodeSha256();+ if (key == null || key.isBlank()) {+ key = Long.toString(fn.getLastModified());+ }+ String h = key.replaceAll("[^a-zA-Z0-9]", "");+ if (h.length() > 20) {+ h = h.substring(0, 20);+ }+ if (h.isEmpty()) {+ h = "0";+ }+ String fname = fn.getFunctionName().replaceAll("[^a-zA-Z0-9_.-]", "-");+ return "floci-code-" + fname + "-" + h;+ }++ /**+ * Buffer for the tar-streaming pipe. The default {@link java.io.PipedInputStream} buffer is+ * only 1KB, which forces a writer/reader thread hand-off (wait/notify) every 1KB. Streaming a+ * ~90MB node_modules through that ran at ~0.5MB/s (≈3 min per cold start) — pure synchronization+ * thrash, not I/O. A large buffer lets the tar writer stream ahead so throughput is bound by the+ * Docker daemon, not the pipe.+ */+ private static final int TAR_PIPE_BUFFER_BYTES = 16 * 1024 * 1024;+private void copyDirToContainer(DockerClient dockerClient, String containerId,Path sourceDir, String remotePath, String functionName) {+ // No per-copy gating here: the heavy /var/task populate for large code already holds a+ // POPULATE_SEMAPHORE permit; small-code direct copies and layer copies are light enough+ // to run unthrottled.try (java.io.PipedOutputStream pos = new java.io.PipedOutputStream();- java.io.PipedInputStream pis = new java.io.PipedInputStream(pos)) {+ java.io.PipedInputStream pis = new java.io.PipedInputStream(pos, TAR_PIPE_BUFFER_BYTES)) {new Thread(() -> {try (pos) {@@ -366,14 +576,16 @@ public class ContainerLauncher {.exec();LOG.debugv("Copied directory {0} into container {1} at {2}", sourceDir, containerId, remotePath);} catch (Exception e) {- LOG.warnv("Failed to copy directory {0} into container {1}: {2}", sourceDir, containerId, e.getMessage());+ // Fail loudly so launch() cleans up the half-built container instead of leaking it.+ throw new RuntimeException("Failed to copy directory " + sourceDir + " into container "+ + containerId + " for function " + functionName + ": " + e.getMessage(), e);}}private void copyFileToContainer(DockerClient dockerClient, String containerId,Path sourceFile, String remotePath, String entryName, String functionName) {try (java.io.PipedOutputStream pos = new java.io.PipedOutputStream();- java.io.PipedInputStream pis = new java.io.PipedInputStream(pos)) {+ java.io.PipedInputStream pis = new java.io.PipedInputStream(pos, TAR_PIPE_BUFFER_BYTES)) {new Thread(() -> {try (TarArchiveOutputStream tar = newTarStream(pos)) {@@ -396,7 +608,8 @@ public class ContainerLauncher {.exec();LOG.debugv("Copied file {0} as {1} into container {2} at {3}", sourceFile, entryName, containerId, remotePath);} catch (Exception e) {- LOG.warnv("Failed to copy file {0} into container {1}: {2}", sourceFile, containerId, e.getMessage());+ throw new RuntimeException("Failed to copy file " + sourceFile + " into container "+ + containerId + " for function " + functionName + ": " + e.getMessage(), e);}}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.javaindex 144d881981..451cc44b41 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java@@ -14,6 +14,8 @@ import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;+import com.github.dockerjava.api.model.Mount;+import com.github.dockerjava.api.model.MountType;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;@@ -30,6 +32,9 @@ import java.util.Map;import java.util.Optional;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.eq;@@ -89,11 +94,13 @@ class ContainerLauncherTest {when(runtimeApiServer.getPort()).thenReturn(9000);when(dockerHostResolver.resolve()).thenReturn("127.0.0.1");- when(lifecycleManager.create(any())).thenReturn("container-123");+ // lenient: the failure-path test (populate fails before any container is created) never+ // reaches these, but every success-path test does — they must not trip strict-stubs.+ lenient().when(lifecycleManager.create(any())).thenReturn("container-123");ContainerLifecycleManager.ContainerInfo info =new ContainerLifecycleManager.ContainerInfo("container-123", Map.of());- when(lifecycleManager.startCreated(eq("container-123"), any())).thenReturn(info);- when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ lenient().when(lifecycleManager.startCreated(eq("container-123"), any())).thenReturn(info);+ lenient().when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);// Stub the Docker copy chain so copyDirToContainer / copyFileToContainer// don't throw when the mock DockerClient is used. Each invocation@@ -123,8 +130,44 @@ class ContainerLauncherTest {});}+ /**+ * Captures every {@link ContainerSpec} passed to {@code lifecycleManager.create(...)} and+ * returns the REAL Lambda container's spec.+ *+ * <p>Small code (below {@link ContainerLauncher#CODE_VOLUME_MIN_BYTES}, the case for these+ * tempdir-backed tests) is copied directly into {@code /var/task} on the real container, so+ * {@code create} is called exactly once. Large code is instead served from a read-only named+ * volume populated by a throwaway helper container (also via {@code create}), so {@code create}+ * is called twice: the helper first, then the real container. The real container is the one that+ * mounts {@code /var/task} read-only from the volume, so we identify it by that mount when+ * present, otherwise fall back to the last (only) {@code create}.+ */+ private ContainerSpec captureRealContainerSpec() {+ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);+ verify(lifecycleManager, atLeastOnce()).create(specCaptor.capture());+ List<ContainerSpec> specs = specCaptor.getAllValues();+ // The real container (volume path) mounts /var/task read-only; the helper mounts it read-write.+ return specs.stream()+ .filter(s -> s.mounts() != null && s.mounts().stream()+ .anyMatch(m -> "/var/task".equals(m.getTarget()) && Boolean.TRUE.equals(m.getReadOnly())))+ .reduce((first, second) -> second) // last match, defensively+ // Fall back to the last create() (the real container) for the direct-copy path.+ .orElseGet(() -> specs.get(specs.size() - 1));+ }++ /** Returns the read-only {@code /var/task} volume mount on the spec, or null if absent. */+ private static Mount varTaskVolumeMount(ContainerSpec spec) {+ if (spec.mounts() == null) {+ return null;+ }+ return spec.mounts().stream()+ .filter(m -> m.getType() == MountType.VOLUME && "/var/task".equals(m.getTarget()))+ .findFirst()+ .orElse(null);+ }+@Test- void launchFunction_createsWithoutBindMounts() throws Exception {+ void launchFunction_createsWithoutBindMountsOrVolume_forSmallCode() throws Exception {Path codePath = Files.createDirectory(tempDir.resolve("code"));LambdaFunction fn = new LambdaFunction();@@ -135,11 +178,14 @@ class ContainerLauncherTest {launcher.launch(fn);- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);- verify(lifecycleManager).create(specCaptor.capture());-- ContainerSpec spec = specCaptor.getValue();+ ContainerSpec spec = captureRealContainerSpec();+ // Small code is copied directly into /var/task (the fast path), NOT bind-mounted...assertTrue(spec.binds().isEmpty(), "Function should NOT have bind mounts");+ // ...and NOT served from a named volume (that's reserved for large code).+ assertNull(varTaskVolumeMount(spec), "small code should NOT mount /var/task from a volume");+ // The code is tar-copied straight into /var/task on the real container.+ assertTrue(capturedRemotePaths.contains("/var/task"),+ "small code should be copied directly into /var/task");}@Test@@ -154,13 +200,24 @@ class ContainerLauncherTest {launcher.launch(fn);- // Verify ordering: create → getDockerClient → Docker copy (to /var/task) → startCreated+ // Small code takes the direct-copy path: the real container is created, its code is+ // tar-copied straight into /var/task, and only then is it started. No populate helper.+ // create(real) -> copy /var/task (real) -> start(real).InOrder inOrder = inOrder(lifecycleManager, dockerClient);inOrder.verify(lifecycleManager).create(any());- inOrder.verify(lifecycleManager).getDockerClient();inOrder.verify(dockerClient).copyArchiveToContainerCmd("container-123");inOrder.verify(lifecycleManager).startCreated(eq("container-123"), any());+ // The code is copied to /var/task on the real container (no helper populate for small code).+ assertTrue(capturedRemotePaths.contains("/var/task"),+ "small code should be tar-copied directly to /var/task");+ assertEquals(1, capturedRemotePaths.stream().filter("/var/task"::equals).count(),+ "/var/task should be copied exactly once (the direct per-container copy)");++ // No populate helper is created/discarded for small code.+ verify(lifecycleManager, times(1)).create(any());+ verify(lifecycleManager, never()).stopAndRemove(any(), any());+// createAndStart must NOT be called — Lambda uses the split pathverify(lifecycleManager, never()).createAndStart(any());}@@ -177,10 +234,7 @@ class ContainerLauncherTest {launcher.launch(fn);- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);- verify(lifecycleManager).create(specCaptor.capture());-- List<String> env = specCaptor.getValue().env();+ List<String> env = captureRealContainerSpec().env();assertTrue(env.stream().anyMatch(e -> e.startsWith("AWS_ACCESS_KEY_ID=")),"AWS_ACCESS_KEY_ID should be injected when awsConfigPath is absent");assertTrue(env.stream().anyMatch(e -> e.startsWith("AWS_SECRET_ACCESS_KEY=")),@@ -204,10 +258,7 @@ class ContainerLauncherTest {launcher.launch(fn);- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);- verify(lifecycleManager).create(specCaptor.capture());-- List<String> env = specCaptor.getValue().env();+ List<String> env = captureRealContainerSpec().env();String accessKey = env.stream().filter(e -> e.startsWith("AWS_ACCESS_KEY_ID=")).findFirst().orElse("");String secretKey = env.stream().filter(e -> e.startsWith("AWS_SECRET_ACCESS_KEY=")).findFirst().orElse("");String sessionToken = env.stream().filter(e -> e.startsWith("AWS_SESSION_TOKEN=")).findFirst().orElse("");@@ -235,10 +286,7 @@ class ContainerLauncherTest {launcher.launch(fn);- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);- verify(lifecycleManager).create(specCaptor.capture());-- List<String> env = specCaptor.getValue().env();+ List<String> env = captureRealContainerSpec().env();assertTrue(env.contains("AWS_DEFAULT_REGION=eu-central-1"));assertTrue(env.contains("AWS_REGION=eu-central-1"));}@@ -256,10 +304,7 @@ class ContainerLauncherTest {launcher.launch(fn);- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);- verify(lifecycleManager).create(specCaptor.capture());-- List<String> env = specCaptor.getValue().env();+ List<String> env = captureRealContainerSpec().env();assertTrue(env.contains("AWS_DEFAULT_REGION=eu-west-2"));assertTrue(env.contains("AWS_REGION=eu-west-2"));verify(logStreamer).attach(@@ -281,10 +326,7 @@ class ContainerLauncherTest {launcher.launch(fn);- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);- verify(lifecycleManager).create(specCaptor.capture());-- List<String> env = specCaptor.getValue().env();+ List<String> env = captureRealContainerSpec().env();// Docker honours the last occurrence of a duplicate Env entry, so user// overrides must appear after the Floci defaults.int defaultKeyIdx = -1;@@ -325,12 +367,11 @@ class ContainerLauncherTest {launcher.launch(fn);- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);- verify(lifecycleManager).create(specCaptor.capture());+ ContainerSpec spec = captureRealContainerSpec();verify(ecrRegistryManager).ensureStarted();verify(ecrRegistryManager).getRepositoryUri("123456789012", "us-east-1", "backend-user:1");assertEquals("123456789012.dkr.ecr.us-east-1.localhost:5100/backend-user:1",- specCaptor.getValue().image());+ spec.image());}@Test@@ -345,12 +386,11 @@ class ContainerLauncherTest {launcher.launch(fn);- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);- verify(lifecycleManager).create(specCaptor.capture());+ ContainerSpec spec = captureRealContainerSpec();verify(ecrRegistryManager).ensureStarted();verify(ecrRegistryManager).getRepositoryUri("123456789012", "us-east-1", "backend-user:1");assertEquals("localhost:5100/123456789012/us-east-1/backend-user:1",- specCaptor.getValue().image());+ spec.image());}@Test@@ -366,21 +406,28 @@ class ContainerLauncherTest {launcher.launch(fn);- // The critical invariant: create must happen before any Docker copy,- // and start must happen after. This is the exact regression from #466.+ // The critical invariant preserved from #466: the REAL container must be created before its+ // code and bootstrap are copied, and started only after. Small code takes the direct-copy+ // path, so everything happens on the one real container (no populate helper).+ //+ // Ordering:+ // real: create -> copy /var/task + copy bootstrap to /var/runtime -> start+ // (two copyArchiveToContainerCmd calls on the real container: code, then bootstrap).InOrder inOrder = inOrder(lifecycleManager, dockerClient);inOrder.verify(lifecycleManager).create(any());- inOrder.verify(lifecycleManager).getDockerClient();- // Two copies: code to /var/task + bootstrap to /var/runtime- inOrder.verify(dockerClient, times(2)).copyArchiveToContainerCmd("container-123");+ inOrder.verify(dockerClient, atLeastOnce()).copyArchiveToContainerCmd("container-123");inOrder.verify(lifecycleManager).startCreated(eq("container-123"), any());- // Verify both /var/task and /var/runtime were targeted+ // Small code is copied directly to /var/task; bootstrap is copied to /var/runtime — both on+ // the one real container.assertTrue(capturedRemotePaths.contains("/var/task"),- "code should be copied to /var/task");+ "small code should be tar-copied directly to /var/task");assertTrue(capturedRemotePaths.contains("/var/runtime"),- "bootstrap should be copied to /var/runtime");+ "bootstrap should be copied to /var/runtime on the real container");+ // No populate helper for small code.+ verify(lifecycleManager, times(1)).create(any());+ verify(lifecycleManager, never()).stopAndRemove(any(), any());verify(lifecycleManager, never()).createAndStart(any());}@@ -399,10 +446,7 @@ class ContainerLauncherTest {launcher.launch(fn);- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);- verify(lifecycleManager).create(specCaptor.capture());-- ContainerSpec spec = specCaptor.getValue();+ ContainerSpec spec = captureRealContainerSpec();// Should bind-mount to /opt/aws-config (read-only)assertTrue(spec.binds().stream()@@ -439,11 +483,84 @@ class ContainerLauncherTest {launcher.launch(fn);- ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);- verify(lifecycleManager).create(specCaptor.capture());-- assertTrue(specCaptor.getValue().binds().stream()+ assertTrue(captureRealContainerSpec().binds().stream().noneMatch(b -> b.getVolume().getPath().equals("/opt/aws-config")),"no .aws bind mount when awsConfigPath is absent");}++ @Test+ void launchFunction_usesReadOnlyVolume_forLargeCode() throws Exception {+ Path codePath = Files.createDirectory(tempDir.resolve("large-code"));+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("large-fn");+ fn.setRuntime("nodejs20.x");+ fn.setHandler("index.handler");+ fn.setCodeLocalPath(codePath.toString());+ fn.setCodeSha256("large-fn-sha-v1");++ long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ try {+ // Force the volume path without writing a 32 MiB file.+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024; // 4 KiB+ launcher.launch(fn);+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;+ }++ ContainerSpec spec = captureRealContainerSpec();+ // The real container mounts /var/task read-only from the code volume...+ Mount codeMount = varTaskVolumeMount(spec);+ assertNotNull(codeMount, "large code: /var/task should be a named-volume mount");+ assertEquals(MountType.VOLUME, codeMount.getType(), "/var/task should be a volume mount");+ assertTrue(Boolean.TRUE.equals(codeMount.getReadOnly()), "/var/task volume should be read-only");+ assertTrue(spec.binds().isEmpty(), "large code: real container should have NO bind mounts");++ // ...and /var/task is populated ONCE via a helper container (create -> start -> copy ->+ // stopAndRemove), not copied onto the real container. The volume is populated exactly once.+ assertEquals(1, capturedRemotePaths.stream().filter("/var/task"::equals).count(),+ "/var/task should be copied exactly once (into the populate helper)");+ // Two creates: the helper + the real container. The helper is discarded.+ verify(lifecycleManager, times(2)).create(any());+ verify(lifecycleManager, atLeastOnce()).ensureVolume(any());+ verify(lifecycleManager, times(1)).stopAndRemove(any(), any()); // the helper only+ // Old code-version volumes are NOT eagerly deleted (race fix); they are label-pruned instead.+ verify(lifecycleManager, never()).removeVolume(any());+ }++ @Test+ void launchFunction_releasesRuntimeApiServer_whenCodeVolumePopulateFails() throws Exception {+ // Regression for the runtime-api port leak: the volume path allocates a runtime-api server+ // up front (before the code-volume populate). If the populate then fails — the exact+ // cold-start-burst scenario this PR targets, where the Docker daemon rejects the helper work+ // under load — the launch must STILL release that port. Otherwise every failed attempt leaks+ // one port and the pool eventually exhausts, so launches keep failing after the daemon recovers.+ Path codePath = Files.createDirectory(tempDir.resolve("leak-code"));+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("leak-fn");+ fn.setRuntime("nodejs20.x");+ fn.setHandler("index.handler");+ fn.setCodeLocalPath(codePath.toString());+ fn.setCodeSha256("leak-fn-sha-v1");++ // The code-volume populate fails (daemon under load), before any container is created.+ doThrow(new RuntimeException("docker daemon busy")).when(lifecycleManager).ensureVolume(any());++ long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ try {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024; // force the volume path without a 32 MiB file+ assertThrows(RuntimeException.class, () -> launcher.launch(fn));+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;+ }++ // The runtime-api port allocated before the populate is released on this failure path...+ verify(runtimeApiServerFactory).release(runtimeApiServer);+ // ...and we bailed before creating or starting any container (nothing to reap).+ verify(lifecycleManager, never()).create(any());+ verify(lifecycleManager, never()).startCreated(any(), any());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherVolumeNamingTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherVolumeNamingTest.javanew file mode 100644index 0000000000..bb0c323447--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherVolumeNamingTest.java@@ -0,0 +1,133 @@+package io.github.hectorvent.floci.services.lambda.launcher;++import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.nio.file.Files;+import java.nio.file.Path;+import java.util.regex.Pattern;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Unit tests for {@link ContainerLauncher#codeVolumeName(LambdaFunction)}, the naming logic for+ * the per-function-version code volume. The name must be stable for a given code version (so all+ * of a function's containers share one volume), distinct across code versions (so a redeploy gets+ * a fresh volume), and always a legal Docker volume name.+ *+ * <p>Kept separate from {@link ContainerLauncherTest} because these are pure static-method tests+ * that need none of that test's Mockito mocks; mixing them in would trip strict-stubbing.+ */+class ContainerLauncherVolumeNamingTest {++ /** The pattern Docker requires a volume name to match. */+ private static final Pattern DOCKER_VOLUME_NAME = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9_.-]*$");++ private static LambdaFunction fnWithSha(String name, String sha) {+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName(name);+ fn.setCodeSha256(sha);+ return fn;+ }++ @Test+ void isFlociCodeShapedAndDockerSafe() {+ String name = ContainerLauncher.codeVolumeName(+ fnWithSha("my-fn", "AbC123+/xyz=deadbeefcafebabe0123456789"));++ assertTrue(name.startsWith("floci-code-my-fn-"),+ "should be floci-code-<functionName>-<hash> shaped, was: " + name);+ assertTrue(DOCKER_VOLUME_NAME.matcher(name).matches(),+ "must be a docker-volume-safe name, was: " + name);+ }++ @Test+ void sanitizesUnsafeFunctionNameChars() {+ // Real function names are constrained, but be defensive: any unsafe char must not leak+ // into the volume name (which would make `docker volume create` reject it).+ String name = ContainerLauncher.codeVolumeName(fnWithSha("weird name/v2:1", "abc123"));+ assertTrue(DOCKER_VOLUME_NAME.matcher(name).matches(),+ "unsafe function-name chars must be sanitized, was: " + name);+ }++ @Test+ void differsAcrossCodeVersions() {+ String v1 = ContainerLauncher.codeVolumeName(fnWithSha("same-fn", "sha-version-one"));+ String v2 = ContainerLauncher.codeVolumeName(fnWithSha("same-fn", "sha-version-two"));++ assertNotEquals(v1, v2,+ "different codeSha256 must yield different volume names so a redeploy gets a fresh volume");+ }++ @Test+ void isStableForSameFunctionAndSha() {+ String a = ContainerLauncher.codeVolumeName(fnWithSha("stable-fn", "stable-sha-abc"));+ String b = ContainerLauncher.codeVolumeName(fnWithSha("stable-fn", "stable-sha-abc"));++ assertEquals(a, b, "same function + same sha must produce a stable volume name");+ }++ @Test+ void fallsBackToLastModifiedWhenShaMissing() {+ LambdaFunction nullSha = new LambdaFunction();+ nullSha.setFunctionName("no-sha-fn");+ nullSha.setCodeSha256(null);+ nullSha.setLastModified(1700000000000L);++ LambdaFunction blankSha = new LambdaFunction();+ blankSha.setFunctionName("no-sha-fn");+ blankSha.setCodeSha256(" ");+ blankSha.setLastModified(1700000000000L);++ String nullName = ContainerLauncher.codeVolumeName(nullSha);+ String blankName = ContainerLauncher.codeVolumeName(blankSha);++ assertTrue(DOCKER_VOLUME_NAME.matcher(nullName).matches(),+ "must be docker-safe even when falling back to lastModified, was: " + nullName);+ assertTrue(nullName.contains("1700000000000"),+ "null sha should key off lastModified, was: " + nullName);+ assertEquals(nullName, blankName, "null and blank sha should both fall back to lastModified");++ // A different lastModified (new deploy) must yield a different name.+ LambdaFunction newer = new LambdaFunction();+ newer.setFunctionName("no-sha-fn");+ newer.setLastModified(1700000009999L);+ assertNotEquals(nullName, ContainerLauncher.codeVolumeName(newer),+ "a later lastModified must produce a different volume name");+ }++ @TempDir+ Path tempDir;++ @Test+ void shouldUseCodeVolume_falseForSmallDir_trueWhenOverThreshold() throws Exception {+ Path smallDir = Files.createDirectory(tempDir.resolve("small"));+ Files.write(smallDir.resolve("a.txt"), new byte[1024]); // 1 KiB++ Path bigDir = Files.createDirectory(tempDir.resolve("big"));+ Files.write(bigDir.resolve("a.bin"), new byte[8 * 1024]); // 8 KiB++ long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ try {+ // Override the threshold small so we don't have to write huge files.+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024; // 4 KiB+ assertFalse(ContainerLauncher.shouldUseCodeVolume(smallDir),+ "1 KiB dir is below the 4 KiB threshold");+ assertTrue(ContainerLauncher.shouldUseCodeVolume(bigDir),+ "8 KiB dir exceeds the 4 KiB threshold");+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;+ }+ }++ @Test+ void shouldUseCodeVolume_falseWhenDirMissing() {+ // IO errors (e.g. a non-existent dir) fall back to the direct copy.+ assertFalse(ContainerLauncher.shouldUseCodeVolume(tempDir.resolve("does-not-exist")),+ "a missing code dir should fall back to direct copy (false)");+ }+}