Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): reconcile code volumes against real Docker state and clean up superseded ones (#2208)

Olamide Kosilea5daab6fb7da
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 42a35c8a89..3e1ad97f85 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -324,15 +324,23 @@ public class ContainerLifecycleManager {
/**
* Removes a named Docker volume, ignoring errors if it does not exist or is still in use.
+ * Returns whether the volume is confirmed gone: true if it was removed or was already absent,
+ * false if Docker refused (e.g. still in use by a container) or the attempt failed for some
+ * other reason (e.g. a transient daemon error). Callers that need to retry a removal should
+ * treat false as "unconfirmed, try again later" rather than "definitely still there" - a single
+ * boolean here can't always distinguish those two cases from the daemon's response alone.
*/
- public void removeVolume(String volumeName) {
+ public boolean removeVolume(String volumeName) {
try {
dockerClient.removeVolumeCmd(volumeName).exec();
LOG.debugv("Removed volume {0}", volumeName);
+ return true;
} catch (NotFoundException e) {
- // Already gone — nothing to do
+ // Already gone, which satisfies the caller's goal just as much as removing it would.
+ return true;
} catch (Exception e) {
LOG.warnv("Error removing volume {0}: {1}", volumeName, e.getMessage());
+ return false;
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
index e0fa962d62..e228aa671e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
@@ -18,6 +18,8 @@ import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;
import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.exception.NotFoundException;
+import jakarta.annotation.PostConstruct;
+import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
@@ -109,6 +111,17 @@ public class ContainerLauncher {
this.awsEnv = awsEnv;
}
+ @PostConstruct
+ void init() {
+ volumeCleanupScheduler.scheduleAtFixedRate(this::cleanupSupersededVolumes,
+ VOLUME_CLEANUP_GRACE_MS, VOLUME_CLEANUP_GRACE_MS, TimeUnit.MILLISECONDS);
+ }
+
+ @PreDestroy
+ void shutdown() {
+ volumeCleanupScheduler.shutdownNow();
+ }
+
/**
* Rewrites real-AWS-shaped ECR image URIs to point at Floci's loopback registry.
* Stored ImageUri is preserved (so describe-function returns the original);
@@ -159,6 +172,10 @@ public class ContainerLauncher {
// runtime-api port per failed attempt and eventually exhausts the pool, so launches keep
// failing even after the daemon recovers.
String containerId = null;
+ // Hoisted alongside containerId, for the same reason: the resolved volume name (if any)
+ // must be visible in the catch block below to release its in-flight reference on any
+ // failure path, not just the one where useCodeVolume's block itself throws.
+ String reservedCodeVolume = null;
try {
// Resolve image
@@ -250,8 +267,8 @@ public class ContainerLauncher {
// overlay I/O) and, run many-at-once, saturated the daemon so even `docker create`
// ballooned to ~80s. A pre-populated volume mounts in ~0.2s and is shared read-only
// by all containers of the function — matching AWS, where /var/task is read-only.
- String codeVolume = ensureCodeVolume(fn, image);
- specBuilder.withNamedVolume(codeVolume, TASK_DIR, true);
+ reservedCodeVolume = ensureCodeVolume(fn, image);
+ specBuilder.withNamedVolume(reservedCodeVolume, TASK_DIR, true);
}
// Small code takes the original per-container direct copy below (no volume): it's fast
// enough that a shared volume's helper round-trip would only add cold-start latency.
@@ -287,6 +304,11 @@ public class ContainerLauncher {
// /var/runtime/bootstrap on start, so code must be copied first.
containerId = lifecycleManager.create(spec);
LOG.infov("Created container {0} for function {1}", containerId, fn.getFunctionName());
+ // Docker now holds the real container-to-volume reference, which removeVolume's own in-use
+ // check protects from here on - release the in-flight marker that stood in for it before
+ // this point, and null it out so the catch block below doesn't release it a second time.
+ releaseCodeVolumeReference(reservedCodeVolume);
+ reservedCodeVolume = null;
// Copy code into container via Docker API tar stream (works inside Docker too).
// Hot-reload functions skip the tar-copy — the bind-mount already wires the host path.
@@ -387,6 +409,10 @@ public class ContainerLauncher {
if (containerId != null) {
try { lifecycleManager.stopAndRemove(containerId, null); } catch (Exception ignore) { /* best effort */ }
}
+ // No-op if create() already succeeded and released this above (reservedCodeVolume is
+ // null by then); otherwise the failure happened before Docker ever saw the volume, so
+ // its in-flight reference must be released here or cleanup would wait on it forever.
+ releaseCodeVolumeReference(reservedCodeVolume);
try { runtimeApiServerFactory.release(runtimeApiServer); } catch (Exception ignore) { /* best effort */ }
throw e;
}
@@ -439,12 +465,75 @@ public class ContainerLauncher {
// Per-function-version code volumes: populated once, then mounted read-only into every
// container so cold starts skip the ~95s node_modules copy. Tracks which volumes are already
- // populated and a per-volume lock so a concurrent cold-start storm populates only once. The
- // lock entry is dropped after a successful populate so this map only ever holds in-flight
- // populates (see ensureCodeVolume) rather than growing across redeploys.
+ // populated and a per-volume lock so a concurrent cold-start storm populates only once - and so
+ // ensureCodeVolume's bookkeeping and cleanupSupersededVolumes' claim-and-delete for the same
+ // volume name can never interleave (see both call sites).
+ //
+ // This in-memory bookkeeping is a cache of Docker's actual state, not a source of truth: a
+ // volume manually removed mid-session (docker volume rm) or reclaimed by an out-of-band
+ // `docker volume prune` would otherwise still read as "populated" here even though Docker has
+ // nothing under that name, silently mounting an empty /var/task into the next container.
+ // ensureCodeVolume re-checks lifecycleManager.volumeExists() rather than trusting this alone.
private final java.util.Set<String> populatedCodeVolumes = java.util.concurrent.ConcurrentHashMap.newKeySet();
+ // Deliberately never pruned: removing an entry while a caller elsewhere still held a reference
+ // to its lock object let a third caller's computeIfAbsent create a replacement lock for the same
+ // volume name, so the waiter (once granted the old lock) and the new caller (holding the new
+ // one) could run their supposedly-exclusive sections concurrently - the exact race this map
+ // exists to prevent. Bounded in practice by the number of distinct function+code-version
+ // combinations ever launched, each entry a single Object; negligible next to the disk space the
+ // rest of this fix reclaims.
private final java.util.concurrent.ConcurrentHashMap<String, Object> codeVolumeLocks = new java.util.concurrent.ConcurrentHashMap<>();
+ /** Each function's current code volume, so a redeploy can queue the superseded one for cleanup. */
+ private final java.util.concurrent.ConcurrentHashMap<String, String> functionCurrentVolume =
+ new java.util.concurrent.ConcurrentHashMap<>();
+
+ /**
+ * Serializes the functionCurrentVolume/volumesPendingCleanup transition below, keyed by function
+ * name rather than volume name. Two code versions of the same function resolve to two different
+ * volume names, so they hold two different codeVolumeLocks entries and can run that per-volume
+ * critical section concurrently - which is fine for the populate work, but not for this
+ * transition: without a shared lock, a rapid back-to-back redeploy (v2 then v3) can have v2's
+ * functionCurrentVolume.put land after v3's, leaving v3 - the actually-current volume - recorded
+ * as superseded and queued for cleanup while stale v2 is left marked current. Never pruned, for
+ * the same lock-identity reason as codeVolumeLocks, and bounded the same way (one entry per
+ * distinct function name ever launched).
+ */
+ private final java.util.concurrent.ConcurrentHashMap<String, Object> functionVolumeTransitionLocks =
+ new java.util.concurrent.ConcurrentHashMap<>();
+
+ /**
+ * Superseded code volumes queued for cleanup, mapped to the time they were superseded (millis
+ * since epoch). Deletion is deferred rather than immediate: Docker auto-creates an empty named
+ * volume for a container mount that doesn't reference an existing one rather than failing, so a
+ * launch that resolved the old volume name just before a redeploy, but hasn't created its
+ * container yet, could otherwise be handed an empty /var/task by an immediate delete. Waiting
+ * out a grace period comfortably longer than a single container launch takes crosses that
+ * window safely; removeVolume() also no-ops if the volume is still genuinely in use by then.
+ */
+ private final java.util.concurrent.ConcurrentHashMap<String, Long> volumesPendingCleanup =
+ new java.util.concurrent.ConcurrentHashMap<>();
+
+ /**
+ * Counts launches that hold a resolved volume name but haven't yet handed it to Docker via
+ * {@code create()} - i.e. no real container-to-volume reference exists yet for Docker's own
+ * in-use check to protect. The grace period in {@link #volumesPendingCleanup} assumes a launch
+ * completes create() well within it, but create() itself has no proven upper bound (observed
+ * up to ~80s under daemon load, longer than the default 60s grace period), so a slow launch can
+ * still be mid-flight when a sweep would otherwise delete its volume. cleanupSupersededVolumes
+ * re-queues rather than deletes while a volume's count here is nonzero, regardless of elapsed
+ * time. Entries are removed once their count returns to zero, so this stays bounded like the
+ * other per-volume maps.
+ */
+ private final java.util.concurrent.ConcurrentHashMap<String, java.util.concurrent.atomic.AtomicInteger> volumesInFlight =
+ new java.util.concurrent.ConcurrentHashMap<>();
+ /** Non-final and package-private, like {@link #CODE_VOLUME_MIN_BYTES}, so tests can shrink it
+ * instead of waiting out a real 60s window (restore it in a finally). */
+ static long VOLUME_CLEANUP_GRACE_MS = 60_000L;
+ private final java.util.concurrent.ScheduledExecutorService volumeCleanupScheduler =
+ java.util.concurrent.Executors.newSingleThreadScheduledExecutor(
+ r -> { Thread t = new Thread(r, "floci-code-volume-cleanup"); t.setDaemon(true); return t; });
+
/**
* Returns the name of a read-only Docker volume holding this function's {@code /var/task} code,
* populating it once (per code version) on first use. Populating streams the code into a
@@ -453,35 +542,145 @@ public class ContainerLauncher {
*/
private String ensureCodeVolume(LambdaFunction fn, String image) {
String volName = codeVolumeName(fn);
- if (populatedCodeVolumes.contains(volName)) {
- return volName;
- }
+ // Held for the whole resolve-and-reconcile, not just the populate branch: this is the same
+ // lock cleanupSupersededVolumes acquires before claiming a volume for deletion, so a launch
+ // that resolves a volume can never race a sweep that's about to delete that exact volume out
+ // from under it - whichever gets the lock first, the other reliably observes the result
+ // (rescued-from-cleanup, or actually-gone-and-needs-repopulating) rather than acting on
+ // stale state. Population itself is the only expensive part, and it happens at most once
+ // per code version regardless, so holding this for cheap map bookkeeping too costs nothing
+ // meaningful. codeVolumeLocks is never pruned (see the comment on that field): removing an
+ // entry while a waiter still held a reference to its lock object let a third caller create a
+ // replacement lock for the same name and run concurrently with the waiter once released,
+ // defeating this exclusion entirely.
Object lock = codeVolumeLocks.computeIfAbsent(volName, k -> new Object());
synchronized (lock) {
- if (populatedCodeVolumes.contains(volName)) {
- return volName;
+ if (!(populatedCodeVolumes.contains(volName) && lifecycleManager.volumeExists(volName))) {
+ // Either never populated, or the bookkeeping says populated but Docker disagrees
+ // (removed out of band). Repopulate rather than trust a stale flag.
+ long t0 = System.currentTimeMillis();
+ LOG.infov("Populating code volume {0} for function {1} (one-time per code version)",
+ volName, fn.getFunctionName());
+ populateCodeVolume(volName, fn, image);
+ populatedCodeVolumes.add(volName);
+ LOG.infov("Populated code volume {0} in {1}ms; future cold starts mount it instead of copying",
+ volName, System.currentTimeMillis() - t0);
+ }
+ // Reconcile bookkeeping for the resolved volume even when the check above already found
+ // it populated: a redeploy rolled back to an older code version resolves a volume name
+ // that's already populated but may have been queued as superseded by the deploy this
+ // just rolled back. Without pulling it back out of volumesPendingCleanup and marking it
+ // current again, the next sweep would delete the volume this function is actively using.
+ //
+ // Under the per-function lock too: this specific read-modify-write (put, then queue
+ // whatever was previously current) must be atomic across different code versions of the
+ // same function, or two concurrent resolves can interleave their put()/queue steps and
+ // leave the actually-current volume queued for cleanup instead of the stale one - see the
+ // comment on functionVolumeTransitionLocks.
+ Object functionLock = functionVolumeTransitionLocks.computeIfAbsent(
+ fn.getFunctionName(), k -> new Object());
+ synchronized (functionLock) {
+ volumesPendingCleanup.remove(volName);
+ String previous = functionCurrentVolume.put(fn.getFunctionName(), volName);
+ if (previous != null && !previous.equals(volName)) {
+ volumesPendingCleanup.put(previous, System.currentTimeMillis());
+ }
}
- long t0 = System.currentTimeMillis();
- LOG.infov("Populating code volume {0} for function {1} (one-time per code version)",
- volName, fn.getFunctionName());
- populateCodeVolume(volName, fn, image);
- populatedCodeVolumes.add(volName);
- // Bound codeVolumeLocks: drop the lock now that the volume is populated, so the map only
- // ever holds in-flight populates rather than growing across redeploys. Safe under the
- // lock: a late thread that computeIfAbsent's a fresh lock object will still hit the
- // populatedCodeVolumes.contains check inside its synchronized block and return without
- // re-populating.
- codeVolumeLocks.remove(volName);
- LOG.infov("Populated code volume {0} in {1}ms; future cold starts mount it instead of copying",
- volName, System.currentTimeMillis() - t0);
- }
- // We do NOT eagerly delete a function's previous code-version volume here: a concurrent
- // in-flight launch may have already resolved that name and be about to mount it, so deleting
- // it would give that container an empty /var/task. Stale volumes are labeled floci=true (see
- // ContainerLifecycleManager.ensureVolume) and reclaimed by `docker volume prune --filter label=floci`.
+ // Mark this volume as having an unconfirmed reference before releasing the lock: the
+ // caller now holds this name but hasn't handed it to Docker yet, so a sweep that acquires
+ // this same lock next must see the increment and skip deleting it. The caller releases
+ // this via releaseCodeVolumeReference once create() succeeds (or the launch fails).
+ volumesInFlight.computeIfAbsent(volName, k -> new java.util.concurrent.atomic.AtomicInteger())
+ .incrementAndGet();
+ }
return volName;
}
+ /**
+ * Releases the in-flight reference {@link #ensureCodeVolume} placed on {@code volName}. Safe to
+ * call with {@code null} (nothing was reserved) and idempotent bookkeeping-wise: the counter
+ * only ever reflects reservations actually made, since callers null out their local reference
+ * after releasing it (see {@link #launch}).
+ */
+ private void releaseCodeVolumeReference(String volName) {
+ if (volName == null) {
+ return;
+ }
+ volumesInFlight.computeIfPresent(volName, (k, count) -> count.decrementAndGet() > 0 ? count : null);
+ }
+
+ /**
+ * Removes superseded code volumes whose grace period has elapsed. Scheduled at the same
+ * interval as the grace period itself, so a volume is deleted within roughly one to two
+ * intervals of becoming superseded. Not a hard deadline, since this is best-effort cleanup,
+ * not a correctness requirement (a volume that outlives a few extra sweeps is still reclaimed by
+ * `docker volume prune --filter label=floci` same as before this existed).
+ */
+ void cleanupSupersededVolumes() {
+ long cutoff = System.currentTimeMillis() - VOLUME_CLEANUP_GRACE_MS;
+ for (String volName : new java.util.ArrayList<>(volumesPendingCleanup.keySet())) {
+ Long queuedAt = volumesPendingCleanup.get(volName);
+ if (queuedAt == null || queuedAt > cutoff) {
+ continue;
+ }
+ // Same lock ensureCodeVolume holds for this volume name: claiming and deleting it here
+ // must not interleave with a concurrent launch resolving (and possibly rescuing) it.
+ Object lock = codeVolumeLocks.computeIfAbsent(volName, k -> new Object());
+ synchronized (lock) {
+ // Re-check under the lock: a concurrent ensureCodeVolume may have rescued this
+ // volume (removed it from volumesPendingCleanup) or requeued it with a fresher
+ // timestamp between the check above and acquiring this lock.
+ Long stillQueuedAt = volumesPendingCleanup.get(volName);
+ if (stillQueuedAt == null || stillQueuedAt > cutoff) {
+ continue;
+ }
+ java.util.concurrent.atomic.AtomicInteger inFlight = volumesInFlight.get(volName);
+ if (inFlight != null && inFlight.get() > 0) {
+ // A launch has resolved this volume but hasn't handed it to Docker yet, so no
+ // real container-to-volume reference exists for removeVolume's own in-use check
+ // to catch. The grace period alone isn't a safe upper bound on that launch's
+ // duration (create() has been observed taking ~80s under daemon load), so requeue
+ // for a later sweep instead of trusting elapsed time here.
+ volumesPendingCleanup.put(volName, System.currentTimeMillis());
+ continue;
+ }
+ volumesPendingCleanup.remove(volName, stillQueuedAt);
+ if (lifecycleManager.removeVolume(volName)) {
+ populatedCodeVolumes.remove(volName);
+ LOG.debugv("Removed superseded code volume {0}", volName);
+ } else {
+ // Not confirmed gone: still in use (e.g. a slow-draining in-flight container
+ // outlived the grace period), or the removal attempt itself failed for some
+ // other reason (e.g. a transient daemon error). Either way, without this the
+ // entry would be lost with no later sweep ever retrying it. Leave
+ // populatedCodeVolumes alone too, since for all we know it's still there and
+ // still valid.
+ volumesPendingCleanup.put(volName, System.currentTimeMillis());
+ }
+ }
+ }
+ }
+
+ /** Test-only: whether a per-volume lock object has ever been created for this volume name. */
+ boolean hasCodeVolumeLock(String volName) {
+ return codeVolumeLocks.containsKey(volName);
+ }
+
+ /**
+ * Test-only: the same lock object ensureCodeVolume synchronizes on for this function's
+ * current-volume transition, so a test can hold it directly to prove a concurrent launch blocks
+ * on it rather than racing past it.
+ */
+ Object functionVolumeTransitionLockFor(String functionName) {
+ return functionVolumeTransitionLocks.computeIfAbsent(functionName, k -> new Object());
+ }
+
+ /** Test-only: the current in-flight reference count for this volume name (0 if none). */
+ int inFlightCount(String volName) {
+ java.util.concurrent.atomic.AtomicInteger count = volumesInFlight.get(volName);
+ return count == null ? 0 : count.get();
+ }
+
private void populateCodeVolume(String volName, LambdaFunction fn, String image) {
lifecycleManager.ensureVolume(volName);
String shortId = java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8);
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
index 4de92c240c..5e89c941f2 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
@@ -48,6 +48,7 @@ import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -592,7 +593,9 @@ class ContainerLauncherTest {
verify(lifecycleManager, times(2)).create(any());
verify(lifecycleManager, atLeastOnce()).ensureVolume(any());
verify(lifecycleManager, times(1)).stopAndRemove(any(), any()); // the helper only
- // Old code-version volumes are NOT eagerly deleted (race fix); they are label-pruned instead.
+ // A superseded code-version volume is never deleted synchronously within a single launch
+ // (see the cleanupSupersededVolumes tests below for the deferred sweep; this fn has no
+ // prior volume to supersede, since it's the fn's first deploy here).
verify(lifecycleManager, never()).removeVolume(any());
}
@@ -631,6 +634,461 @@ class ContainerLauncherTest {
verify(lifecycleManager, never()).startCreated(any(), any());
}
+ @Test
+ void launchFunction_reprovisionsCodeVolume_whenDockerHasNoRecordOfIt() throws Exception {
+ // Regression for #2164: the "populated" bookkeeping is only ever an in-memory cache of
+ // Docker's state, so a volume removed out of band (manual `docker volume rm`, or a stale
+ // in-memory flag left over from a process restart racing a prune) must not be trusted.
+ Path codePath = Files.createDirectory(tempDir.resolve("revalidate-code"));
+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("revalidate-fn");
+ fn.setRuntime("nodejs20.x");
+ fn.setHandler("index.handler");
+ fn.setCodeLocalPath(codePath.toString());
+ fn.setCodeSha256("revalidate-fn-sha-v1");
+
+ // Docker never reports this volume as existing, simulating it being gone every time
+ // ensureCodeVolume checks, regardless of what the in-memory flag says.
+ when(lifecycleManager.volumeExists(anyString())).thenReturn(false);
+
+ long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ try {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+ launcher.launch(fn);
+ launcher.launch(fn);
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;
+ }
+
+ // Populated twice, once per launch, because the bookkeeping alone is never trusted.
+ assertEquals(2, capturedRemotePaths.stream().filter("/var/task"::equals).count(),
+ "each launch should repopulate since Docker never confirms the volume exists");
+ }
+
+ @Test
+ void cleanupSupersededVolumes_removesPreviousVersionVolume_onceGracePeriodElapses() throws Exception {
+ Path codePath = Files.createDirectory(tempDir.resolve("cleanup-code"));
+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB
+ // No volumeExists stub needed: v1 and v2 are each a first-ever population of their own
+ // distinct volume name, so populatedCodeVolumes.contains(volName) is false both times
+ // ensureCodeVolume checks it, short-circuiting the volumeExists call away entirely.
+
+ LambdaFunction v1 = new LambdaFunction();
+ v1.setFunctionName("cleanup-fn");
+ v1.setRuntime("nodejs20.x");
+ v1.setHandler("index.handler");
+ v1.setCodeLocalPath(codePath.toString());
+ v1.setCodeSha256("cleanup-fn-sha-v1");
+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);
+
+ LambdaFunction v2 = new LambdaFunction();
+ v2.setFunctionName("cleanup-fn");
+ v2.setRuntime("nodejs20.x");
+ v2.setHandler("index.handler");
+ v2.setCodeLocalPath(codePath.toString());
+ v2.setCodeSha256("cleanup-fn-sha-v2");
+ String volumeV2 = ContainerLauncher.codeVolumeName(v2);
+
+ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+ launcher.launch(v1);
+ launcher.launch(v2);
+
+ // The v1 volume is superseded but not yet cleaned up: the grace period hasn't elapsed.
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, never()).removeVolume(volumeV1);
+
+ // Once the grace period has (trivially) elapsed, the sweep removes exactly the
+ // superseded v1 volume, not the current v2 one.
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, times(1)).removeVolume(volumeV1);
+ verify(lifecycleManager, never()).removeVolume(volumeV2);
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ }
+
+ @Test
+ void rollingBackToAPreviousCodeVersion_rescuesItsVolumeFromCleanup() throws Exception {
+ // Regression: v1 -> v2 -> back to v1 (e.g. a CloudFormation rollback) resolves v1's volume
+ // name again, which is already populated - a fast-path hit in ensureCodeVolume. Without
+ // reconciling functionCurrentVolume/volumesPendingCleanup on that path too, v1 would stay
+ // queued as superseded from the v1 -> v2 step and the next sweep would delete the volume
+ // this function is actively using again.
+ Path codePath = Files.createDirectory(tempDir.resolve("rollback-code"));
+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB
+
+ LambdaFunction v1 = new LambdaFunction();
+ v1.setFunctionName("rollback-fn");
+ v1.setRuntime("nodejs20.x");
+ v1.setHandler("index.handler");
+ v1.setCodeLocalPath(codePath.toString());
+ v1.setCodeSha256("rollback-fn-sha-v1");
+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);
+
+ LambdaFunction v2 = new LambdaFunction();
+ v2.setFunctionName("rollback-fn");
+ v2.setRuntime("nodejs20.x");
+ v2.setHandler("index.handler");
+ v2.setCodeLocalPath(codePath.toString());
+ v2.setCodeSha256("rollback-fn-sha-v2");
+ String volumeV2 = ContainerLauncher.codeVolumeName(v2);
+
+ // Needed for the rollback launch below: v1's volume is already populated by then, so its
+ // fast path actually evaluates volumeExists instead of short-circuiting past it.
+ when(lifecycleManager.volumeExists(volumeV1)).thenReturn(true);
+
+ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+ launcher.launch(v1);
+ launcher.launch(v2);
+ launcher.launch(v1); // roll back
+
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, never()).removeVolume(volumeV1);
+ verify(lifecycleManager, times(1)).removeVolume(volumeV2);
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ }
+
+ @Test
+ void cleanupSupersededVolumes_retriesOnALaterSweep_whenTheVolumeIsStillInUse() throws Exception {
+ // Regression: removeVolume() silently no-ops when Docker refuses because the volume is
+ // still in use (e.g. a slow-draining in-flight container outliving the grace period). The
+ // pending-cleanup entry must not be discarded in that case, or the volume is orphaned until
+ // someone manually runs `docker volume prune`- the exact problem this fix exists to avoid.
+ Path codePath = Files.createDirectory(tempDir.resolve("retry-code"));
+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB
+
+ LambdaFunction v1 = new LambdaFunction();
+ v1.setFunctionName("retry-fn");
+ v1.setRuntime("nodejs20.x");
+ v1.setHandler("index.handler");
+ v1.setCodeLocalPath(codePath.toString());
+ v1.setCodeSha256("retry-fn-sha-v1");
+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);
+
+ LambdaFunction v2 = new LambdaFunction();
+ v2.setFunctionName("retry-fn");
+ v2.setRuntime("nodejs20.x");
+ v2.setHandler("index.handler");
+ v2.setCodeLocalPath(codePath.toString());
+ v2.setCodeSha256("retry-fn-sha-v2");
+
+ // v1's volume persists (still in use) no matter how many times removal is attempted.
+ when(lifecycleManager.removeVolume(volumeV1)).thenReturn(false);
+
+ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+ launcher.launch(v1);
+ launcher.launch(v2);
+
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, times(1)).removeVolume(volumeV1);
+
+ // A later sweep must still retry it, not have silently dropped it after the first
+ // no-op'd attempt.
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, times(2)).removeVolume(volumeV1);
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ }
+
+ @Test
+ void cleanupAndAConcurrentRollback_areMutuallyExclusiveForTheSameVolume() throws Exception {
+ // Regression: without a shared per-volume lock, a sweep that has already claimed a
+ // superseded volume (removed its pending-cleanup entry) could race a concurrent rollback
+ // that resolves the same volume name, marks it current, and hands it to a new container -
+ // while the sweep proceeds to actually delete it underneath that launch. Proving this
+ // requires real concurrency: this test forces cleanupSupersededVolumes to block mid-deletion
+ // (still holding the volume's lock) and asserts a concurrent rollback blocks too, rather than
+ // racing past it.
+ Path codePath = Files.createDirectory(tempDir.resolve("race-code"));
+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB
+
+ LambdaFunction v1 = new LambdaFunction();
+ v1.setFunctionName("race-fn");
+ v1.setRuntime("nodejs20.x");
+ v1.setHandler("index.handler");
+ v1.setCodeLocalPath(codePath.toString());
+ v1.setCodeSha256("race-fn-sha-v1");
+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);
+
+ LambdaFunction v2 = new LambdaFunction();
+ v2.setFunctionName("race-fn");
+ v2.setRuntime("nodejs20.x");
+ v2.setHandler("index.handler");
+ v2.setCodeLocalPath(codePath.toString());
+ v2.setCodeSha256("race-fn-sha-v2");
+
+ when(lifecycleManager.volumeExists(volumeV1)).thenReturn(true);
+
+ java.util.concurrent.CountDownLatch cleanupHoldingLock = new java.util.concurrent.CountDownLatch(1);
+ java.util.concurrent.CountDownLatch releaseCleanup = new java.util.concurrent.CountDownLatch(1);
+ doAnswer(inv -> {
+ cleanupHoldingLock.countDown();
+ // Blocks here while still holding volumeV1's per-volume lock, simulating a slow delete.
+ assertTrue(releaseCleanup.await(5, java.util.concurrent.TimeUnit.SECONDS),
+ "test did not release cleanup in time");
+ return null;
+ }).when(lifecycleManager).removeVolume(volumeV1);
+
+ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+ launcher.launch(v1);
+ launcher.launch(v2);
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+
+ Thread cleanupThread = new Thread(launcher::cleanupSupersededVolumes);
+ cleanupThread.start();
+ assertTrue(cleanupHoldingLock.await(5, java.util.concurrent.TimeUnit.SECONDS),
+ "cleanup never reached removeVolume");
+
+ java.util.concurrent.atomic.AtomicBoolean rollbackReturned =
+ new java.util.concurrent.atomic.AtomicBoolean(false);
+ Thread rollbackThread = new Thread(() -> {
+ launcher.launch(v1);
+ rollbackReturned.set(true);
+ });
+ rollbackThread.start();
+
+ // Give the rollback every chance to (wrongly) proceed if the lock weren't shared.
+ Thread.sleep(200);
+ assertFalse(rollbackReturned.get(),
+ "rollback must block while cleanup holds the volume's lock, not race past it");
+
+ releaseCleanup.countDown();
+ cleanupThread.join(5000);
+ rollbackThread.join(5000);
+ assertTrue(rollbackReturned.get(), "rollback should complete once cleanup releases the lock");
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ }
+
+ @Test
+ void cleanupSkipsAVolumeStillInFlight_evenPastItsGracePeriod() throws Exception {
+ // Regression: a launch that has resolved its code volume (ensureCodeVolume returned, its
+ // per-volume lock released) but hasn't yet reached lifecycleManager.create() has no real
+ // Docker container-to-volume reference for removeVolume's own in-use check to protect - and
+ // create() itself has no proven upper bound (observed ~80s under daemon load, longer than
+ // the default 60s grace period). This forces a second launch of v1 to block right before
+ // create() while a redeploy to v2 supersedes its volume and the grace period is made to
+ // elapse instantly, then asserts cleanup does NOT delete v1's volume while that launch is
+ // still in flight - only once it completes and releases its reference.
+ Path codePath = Files.createDirectory(tempDir.resolve("inflight-code"));
+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB
+
+ LambdaFunction v1 = new LambdaFunction();
+ v1.setFunctionName("inflight-fn");
+ v1.setRuntime("nodejs20.x");
+ v1.setHandler("index.handler");
+ v1.setCodeLocalPath(codePath.toString());
+ v1.setCodeSha256("inflight-fn-sha-v1");
+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);
+
+ LambdaFunction v2 = new LambdaFunction();
+ v2.setFunctionName("inflight-fn");
+ v2.setRuntime("nodejs20.x");
+ v2.setHandler("index.handler");
+ v2.setCodeLocalPath(codePath.toString());
+ v2.setCodeSha256("inflight-fn-sha-v2");
+
+ // Needed for the delayed re-launch below: v1's volume is already populated by then, so its
+ // fast path actually evaluates volumeExists instead of short-circuiting past it.
+ when(lifecycleManager.volumeExists(volumeV1)).thenReturn(true);
+
+ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+
+ // Pre-populate v1's volume with an ordinary launch, before installing the create()
+ // blocking stub below - otherwise that stub would catch populateCodeVolume's own
+ // helper-container create() call instead of the real container's.
+ launcher.launch(v1);
+
+ java.util.concurrent.CountDownLatch launchReachedCreate = new java.util.concurrent.CountDownLatch(1);
+ java.util.concurrent.CountDownLatch releaseLaunch = new java.util.concurrent.CountDownLatch(1);
+ java.util.concurrent.atomic.AtomicBoolean blockedOnce = new java.util.concurrent.atomic.AtomicBoolean(false);
+ doAnswer(inv -> {
+ if (blockedOnce.compareAndSet(false, true)) {
+ // Only the delayed re-launch of v1 (the first caller after this stub is
+ // installed) blocks here; v2's later launch below must not, or the test would
+ // deadlock itself waiting on its own main thread to release the latch.
+ launchReachedCreate.countDown();
+ assertTrue(releaseLaunch.await(5, java.util.concurrent.TimeUnit.SECONDS),
+ "test did not release the delayed launch in time");
+ }
+ return "container-123";
+ }).when(lifecycleManager).create(any());
+
+ Thread launchThread = new Thread(() -> launcher.launch(v1));
+ launchThread.start();
+ assertTrue(launchReachedCreate.await(5, java.util.concurrent.TimeUnit.SECONDS),
+ "v1's delayed re-launch never reached create()");
+
+ assertEquals(1, launcher.inFlightCount(volumeV1),
+ "ensureCodeVolume must mark the volume in-flight before create() confirms it");
+
+ // A redeploy resolves v2 and supersedes v1's volume while v1's own launch is still stuck.
+ launcher.launch(v2);
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1; // grace period elapses instantly
+
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, never()).removeVolume(volumeV1);
+
+ // Let the delayed launch finish; it releases its in-flight reference on success.
+ releaseLaunch.countDown();
+ launchThread.join(5000);
+ assertEquals(0, launcher.inFlightCount(volumeV1),
+ "in-flight count must be released once create() succeeds");
+
+ // Nothing is in flight anymore, so a later sweep is free to actually delete it.
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, times(1)).removeVolume(volumeV1);
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ }
+
+ @Test
+ void concurrentRedeploysOfTheSameFunction_serializeTheCurrentVolumeTransition() throws Exception {
+ // Regression: functionCurrentVolume/volumesPendingCleanup reconciliation ran under the
+ // per-volume lock only, but two code versions of the same function resolve to two different
+ // volume names and so hold two different per-volume locks - meaning that reconciliation
+ // could interleave across a rapid back-to-back redeploy (v2 then v3), leaving the actually-
+ // current v3 volume mistakenly queued for cleanup while stale v2 stayed recorded as current.
+ // The three statements involved (remove pending-cleanup entry, swap in the new current
+ // volume, queue whatever was displaced) are plain map operations with no interception point
+ // inside them, so forcing that exact interleaving isn't reliably doable in a test. This
+ // instead verifies the fix's actual mechanism directly - a concurrent launch for a second
+ // code version of the same function must block on the same per-function lock object while
+ // another is still transitioning - the same style cleanupAndAConcurrentRollback... already
+ // uses to prove the per-volume lock.
+ Path codePath = Files.createDirectory(tempDir.resolve("xfn-code"));
+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB
+
+ LambdaFunction v1 = new LambdaFunction();
+ v1.setFunctionName("xfn");
+ v1.setRuntime("nodejs20.x");
+ v1.setHandler("index.handler");
+ v1.setCodeLocalPath(codePath.toString());
+ v1.setCodeSha256("xfn-sha-v1");
+
+ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ try {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+
+ Object functionLock = launcher.functionVolumeTransitionLockFor("xfn");
+ java.util.concurrent.CountDownLatch testHoldingLock = new java.util.concurrent.CountDownLatch(1);
+ java.util.concurrent.CountDownLatch releaseLock = new java.util.concurrent.CountDownLatch(1);
+ Thread holderThread = new Thread(() -> {
+ synchronized (functionLock) {
+ testHoldingLock.countDown();
+ try {
+ assertTrue(releaseLock.await(5, java.util.concurrent.TimeUnit.SECONDS),
+ "test did not release the function lock in time");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ });
+ holderThread.start();
+ assertTrue(testHoldingLock.await(5, java.util.concurrent.TimeUnit.SECONDS),
+ "test thread never acquired the function lock");
+
+ AtomicBoolean launchReturned = new AtomicBoolean(false);
+ Thread launchThread = new Thread(() -> {
+ launcher.launch(v1);
+ launchReturned.set(true);
+ });
+ launchThread.start();
+
+ // Give the launch every chance to (wrongly) proceed if the transition weren't guarded
+ // by this same lock.
+ Thread.sleep(200);
+ assertFalse(launchReturned.get(),
+ "launch must block on the function lock while another holder has it, not race past it");
+
+ releaseLock.countDown();
+ holderThread.join(5000);
+ launchThread.join(5000);
+ assertTrue(launchReturned.get(), "launch should complete once the function lock is released");
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
+ }
+ }
+
+ @Test
+ void codeVolumeLocks_isNeverPruned_evenAfterCleanupDeletesTheVolume() throws Exception {
+ // Regression: pruning a volume's lock entry while a waiter still held a reference to the
+ // removed entry's lock object let a third caller's computeIfAbsent create a *different* lock
+ // object for the same volume name, so the waiter (once granted the old lock) and that third
+ // caller (holding the new one) could run their supposedly mutually-exclusive sections
+ // concurrently - defeating the whole point. The actual JVM interleaving needed to trigger
+ // that is timing-dependent and not reliably forceable in a test, so this instead verifies the
+ // fix's real invariant directly: the lock entry must survive a full populate-then-delete
+ // cycle unchanged, which is what actually guarantees computeIfAbsent always returns the same
+ // object for a given volume name for the life of the process.
+ Path codePath = Files.createDirectory(tempDir.resolve("lock-persist-code"));
+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB
+
+ LambdaFunction v1 = new LambdaFunction();
+ v1.setFunctionName("lock-persist-fn");
+ v1.setRuntime("nodejs20.x");
+ v1.setHandler("index.handler");
+ v1.setCodeLocalPath(codePath.toString());
+ v1.setCodeSha256("lock-persist-fn-sha-v1");
+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);
+
+ LambdaFunction v2 = new LambdaFunction();
+ v2.setFunctionName("lock-persist-fn");
+ v2.setRuntime("nodejs20.x");
+ v2.setHandler("index.handler");
+ v2.setCodeLocalPath(codePath.toString());
+ v2.setCodeSha256("lock-persist-fn-sha-v2");
+
+ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+ launcher.launch(v1);
+ assertTrue(launcher.hasCodeVolumeLock(volumeV1), "lock must exist right after populate");
+
+ launcher.launch(v2); // supersedes v1's volume, queues it for cleanup
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+ launcher.cleanupSupersededVolumes(); // deletes v1's volume for real
+
+ assertTrue(launcher.hasCodeVolumeLock(volumeV1),
+ "lock must still exist even after the volume itself was deleted - "
+ + "pruning it here is exactly the bug this test guards against");
+ } finally {
+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ }
+
/** Builds a tar archive matching what {@code docker cp}/{@code copyArchiveFromContainerCmd}
* returns for a directory: the directory itself as a leading entry, then each name as a direct
* child, executable. */