Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): reconcile code volumes against real Docker state and clean up superseded ones (#2208)
Olamide Kosilea5daab6fb7da
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 42a35c8a89..3e1ad97f85 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -324,15 +324,23 @@ public class ContainerLifecycleManager {/*** Removes a named Docker volume, ignoring errors if it does not exist or is still in use.+ * Returns whether the volume is confirmed gone: true if it was removed or was already absent,+ * false if Docker refused (e.g. still in use by a container) or the attempt failed for some+ * other reason (e.g. a transient daemon error). Callers that need to retry a removal should+ * treat false as "unconfirmed, try again later" rather than "definitely still there" - a single+ * boolean here can't always distinguish those two cases from the daemon's response alone.*/- public void removeVolume(String volumeName) {+ public boolean removeVolume(String volumeName) {try {dockerClient.removeVolumeCmd(volumeName).exec();LOG.debugv("Removed volume {0}", volumeName);+ return true;} catch (NotFoundException e) {- // Already gone — nothing to do+ // Already gone, which satisfies the caller's goal just as much as removing it would.+ return true;} catch (Exception e) {LOG.warnv("Error removing volume {0}: {1}", volumeName, e.getMessage());+ return false;}}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex e0fa962d62..e228aa671e 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -18,6 +18,8 @@ import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.exception.NotFoundException;+import jakarta.annotation.PostConstruct;+import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -109,6 +111,17 @@ public class ContainerLauncher {this.awsEnv = awsEnv;}+ @PostConstruct+ void init() {+ volumeCleanupScheduler.scheduleAtFixedRate(this::cleanupSupersededVolumes,+ VOLUME_CLEANUP_GRACE_MS, VOLUME_CLEANUP_GRACE_MS, TimeUnit.MILLISECONDS);+ }++ @PreDestroy+ void shutdown() {+ volumeCleanupScheduler.shutdownNow();+ }+/*** Rewrites real-AWS-shaped ECR image URIs to point at Floci's loopback registry.* Stored ImageUri is preserved (so describe-function returns the original);@@ -159,6 +172,10 @@ public class ContainerLauncher {// runtime-api port per failed attempt and eventually exhausts the pool, so launches keep// failing even after the daemon recovers.String containerId = null;+ // Hoisted alongside containerId, for the same reason: the resolved volume name (if any)+ // must be visible in the catch block below to release its in-flight reference on any+ // failure path, not just the one where useCodeVolume's block itself throws.+ String reservedCodeVolume = null;try {// Resolve image@@ -250,8 +267,8 @@ public class ContainerLauncher {// overlay I/O) and, run many-at-once, saturated the daemon so even `docker create`// ballooned to ~80s. A pre-populated volume mounts in ~0.2s and is shared read-only// by all containers of the function — matching AWS, where /var/task is read-only.- String codeVolume = ensureCodeVolume(fn, image);- specBuilder.withNamedVolume(codeVolume, TASK_DIR, true);+ reservedCodeVolume = ensureCodeVolume(fn, image);+ specBuilder.withNamedVolume(reservedCodeVolume, TASK_DIR, true);}// Small code takes the original per-container direct copy below (no volume): it's fast// enough that a shared volume's helper round-trip would only add cold-start latency.@@ -287,6 +304,11 @@ public class ContainerLauncher {// /var/runtime/bootstrap on start, so code must be copied first.containerId = lifecycleManager.create(spec);LOG.infov("Created container {0} for function {1}", containerId, fn.getFunctionName());+ // Docker now holds the real container-to-volume reference, which removeVolume's own in-use+ // check protects from here on - release the in-flight marker that stood in for it before+ // this point, and null it out so the catch block below doesn't release it a second time.+ releaseCodeVolumeReference(reservedCodeVolume);+ reservedCodeVolume = null;// Copy code into container via Docker API tar stream (works inside Docker too).// Hot-reload functions skip the tar-copy — the bind-mount already wires the host path.@@ -387,6 +409,10 @@ public class ContainerLauncher {if (containerId != null) {try { lifecycleManager.stopAndRemove(containerId, null); } catch (Exception ignore) { /* best effort */ }}+ // No-op if create() already succeeded and released this above (reservedCodeVolume is+ // null by then); otherwise the failure happened before Docker ever saw the volume, so+ // its in-flight reference must be released here or cleanup would wait on it forever.+ releaseCodeVolumeReference(reservedCodeVolume);try { runtimeApiServerFactory.release(runtimeApiServer); } catch (Exception ignore) { /* best effort */ }throw e;}@@ -439,12 +465,75 @@ public class ContainerLauncher {// Per-function-version code volumes: populated once, then mounted read-only into every// container so cold starts skip the ~95s node_modules copy. Tracks which volumes are already- // populated and a per-volume lock so a concurrent cold-start storm populates only once. The- // lock entry is dropped after a successful populate so this map only ever holds in-flight- // populates (see ensureCodeVolume) rather than growing across redeploys.+ // populated and a per-volume lock so a concurrent cold-start storm populates only once - and so+ // ensureCodeVolume's bookkeeping and cleanupSupersededVolumes' claim-and-delete for the same+ // volume name can never interleave (see both call sites).+ //+ // This in-memory bookkeeping is a cache of Docker's actual state, not a source of truth: a+ // volume manually removed mid-session (docker volume rm) or reclaimed by an out-of-band+ // `docker volume prune` would otherwise still read as "populated" here even though Docker has+ // nothing under that name, silently mounting an empty /var/task into the next container.+ // ensureCodeVolume re-checks lifecycleManager.volumeExists() rather than trusting this alone.private final java.util.Set<String> populatedCodeVolumes = java.util.concurrent.ConcurrentHashMap.newKeySet();+ // Deliberately never pruned: removing an entry while a caller elsewhere still held a reference+ // to its lock object let a third caller's computeIfAbsent create a replacement lock for the same+ // volume name, so the waiter (once granted the old lock) and the new caller (holding the new+ // one) could run their supposedly-exclusive sections concurrently - the exact race this map+ // exists to prevent. Bounded in practice by the number of distinct function+code-version+ // combinations ever launched, each entry a single Object; negligible next to the disk space the+ // rest of this fix reclaims.private final java.util.concurrent.ConcurrentHashMap<String, Object> codeVolumeLocks = new java.util.concurrent.ConcurrentHashMap<>();+ /** Each function's current code volume, so a redeploy can queue the superseded one for cleanup. */+ private final java.util.concurrent.ConcurrentHashMap<String, String> functionCurrentVolume =+ new java.util.concurrent.ConcurrentHashMap<>();++ /**+ * Serializes the functionCurrentVolume/volumesPendingCleanup transition below, keyed by function+ * name rather than volume name. Two code versions of the same function resolve to two different+ * volume names, so they hold two different codeVolumeLocks entries and can run that per-volume+ * critical section concurrently - which is fine for the populate work, but not for this+ * transition: without a shared lock, a rapid back-to-back redeploy (v2 then v3) can have v2's+ * functionCurrentVolume.put land after v3's, leaving v3 - the actually-current volume - recorded+ * as superseded and queued for cleanup while stale v2 is left marked current. Never pruned, for+ * the same lock-identity reason as codeVolumeLocks, and bounded the same way (one entry per+ * distinct function name ever launched).+ */+ private final java.util.concurrent.ConcurrentHashMap<String, Object> functionVolumeTransitionLocks =+ new java.util.concurrent.ConcurrentHashMap<>();++ /**+ * Superseded code volumes queued for cleanup, mapped to the time they were superseded (millis+ * since epoch). Deletion is deferred rather than immediate: Docker auto-creates an empty named+ * volume for a container mount that doesn't reference an existing one rather than failing, so a+ * launch that resolved the old volume name just before a redeploy, but hasn't created its+ * container yet, could otherwise be handed an empty /var/task by an immediate delete. Waiting+ * out a grace period comfortably longer than a single container launch takes crosses that+ * window safely; removeVolume() also no-ops if the volume is still genuinely in use by then.+ */+ private final java.util.concurrent.ConcurrentHashMap<String, Long> volumesPendingCleanup =+ new java.util.concurrent.ConcurrentHashMap<>();++ /**+ * Counts launches that hold a resolved volume name but haven't yet handed it to Docker via+ * {@code create()} - i.e. no real container-to-volume reference exists yet for Docker's own+ * in-use check to protect. The grace period in {@link #volumesPendingCleanup} assumes a launch+ * completes create() well within it, but create() itself has no proven upper bound (observed+ * up to ~80s under daemon load, longer than the default 60s grace period), so a slow launch can+ * still be mid-flight when a sweep would otherwise delete its volume. cleanupSupersededVolumes+ * re-queues rather than deletes while a volume's count here is nonzero, regardless of elapsed+ * time. Entries are removed once their count returns to zero, so this stays bounded like the+ * other per-volume maps.+ */+ private final java.util.concurrent.ConcurrentHashMap<String, java.util.concurrent.atomic.AtomicInteger> volumesInFlight =+ new java.util.concurrent.ConcurrentHashMap<>();+ /** Non-final and package-private, like {@link #CODE_VOLUME_MIN_BYTES}, so tests can shrink it+ * instead of waiting out a real 60s window (restore it in a finally). */+ static long VOLUME_CLEANUP_GRACE_MS = 60_000L;+ private final java.util.concurrent.ScheduledExecutorService volumeCleanupScheduler =+ java.util.concurrent.Executors.newSingleThreadScheduledExecutor(+ r -> { Thread t = new Thread(r, "floci-code-volume-cleanup"); t.setDaemon(true); return t; });+/*** Returns the name of a read-only Docker volume holding this function's {@code /var/task} code,* populating it once (per code version) on first use. Populating streams the code into a@@ -453,35 +542,145 @@ public class ContainerLauncher {*/private String ensureCodeVolume(LambdaFunction fn, String image) {String volName = codeVolumeName(fn);- if (populatedCodeVolumes.contains(volName)) {- return volName;- }+ // Held for the whole resolve-and-reconcile, not just the populate branch: this is the same+ // lock cleanupSupersededVolumes acquires before claiming a volume for deletion, so a launch+ // that resolves a volume can never race a sweep that's about to delete that exact volume out+ // from under it - whichever gets the lock first, the other reliably observes the result+ // (rescued-from-cleanup, or actually-gone-and-needs-repopulating) rather than acting on+ // stale state. Population itself is the only expensive part, and it happens at most once+ // per code version regardless, so holding this for cheap map bookkeeping too costs nothing+ // meaningful. codeVolumeLocks is never pruned (see the comment on that field): removing an+ // entry while a waiter still held a reference to its lock object let a third caller create a+ // replacement lock for the same name and run concurrently with the waiter once released,+ // defeating this exclusion entirely.Object lock = codeVolumeLocks.computeIfAbsent(volName, k -> new Object());synchronized (lock) {- if (populatedCodeVolumes.contains(volName)) {- return volName;+ if (!(populatedCodeVolumes.contains(volName) && lifecycleManager.volumeExists(volName))) {+ // Either never populated, or the bookkeeping says populated but Docker disagrees+ // (removed out of band). Repopulate rather than trust a stale flag.+ long t0 = System.currentTimeMillis();+ LOG.infov("Populating code volume {0} for function {1} (one-time per code version)",+ volName, fn.getFunctionName());+ populateCodeVolume(volName, fn, image);+ populatedCodeVolumes.add(volName);+ LOG.infov("Populated code volume {0} in {1}ms; future cold starts mount it instead of copying",+ volName, System.currentTimeMillis() - t0);+ }+ // Reconcile bookkeeping for the resolved volume even when the check above already found+ // it populated: a redeploy rolled back to an older code version resolves a volume name+ // that's already populated but may have been queued as superseded by the deploy this+ // just rolled back. Without pulling it back out of volumesPendingCleanup and marking it+ // current again, the next sweep would delete the volume this function is actively using.+ //+ // Under the per-function lock too: this specific read-modify-write (put, then queue+ // whatever was previously current) must be atomic across different code versions of the+ // same function, or two concurrent resolves can interleave their put()/queue steps and+ // leave the actually-current volume queued for cleanup instead of the stale one - see the+ // comment on functionVolumeTransitionLocks.+ Object functionLock = functionVolumeTransitionLocks.computeIfAbsent(+ fn.getFunctionName(), k -> new Object());+ synchronized (functionLock) {+ volumesPendingCleanup.remove(volName);+ String previous = functionCurrentVolume.put(fn.getFunctionName(), volName);+ if (previous != null && !previous.equals(volName)) {+ volumesPendingCleanup.put(previous, System.currentTimeMillis());+ }}- long t0 = System.currentTimeMillis();- LOG.infov("Populating code volume {0} for function {1} (one-time per code version)",- volName, fn.getFunctionName());- populateCodeVolume(volName, fn, image);- populatedCodeVolumes.add(volName);- // Bound codeVolumeLocks: drop the lock now that the volume is populated, so the map only- // ever holds in-flight populates rather than growing across redeploys. Safe under the- // lock: a late thread that computeIfAbsent's a fresh lock object will still hit the- // populatedCodeVolumes.contains check inside its synchronized block and return without- // re-populating.- codeVolumeLocks.remove(volName);- LOG.infov("Populated code volume {0} in {1}ms; future cold starts mount it instead of copying",- volName, System.currentTimeMillis() - t0);- }- // We do NOT eagerly delete a function's previous code-version volume here: a concurrent- // in-flight launch may have already resolved that name and be about to mount it, so deleting- // it would give that container an empty /var/task. Stale volumes are labeled floci=true (see- // ContainerLifecycleManager.ensureVolume) and reclaimed by `docker volume prune --filter label=floci`.+ // Mark this volume as having an unconfirmed reference before releasing the lock: the+ // caller now holds this name but hasn't handed it to Docker yet, so a sweep that acquires+ // this same lock next must see the increment and skip deleting it. The caller releases+ // this via releaseCodeVolumeReference once create() succeeds (or the launch fails).+ volumesInFlight.computeIfAbsent(volName, k -> new java.util.concurrent.atomic.AtomicInteger())+ .incrementAndGet();+ }return volName;}+ /**+ * Releases the in-flight reference {@link #ensureCodeVolume} placed on {@code volName}. Safe to+ * call with {@code null} (nothing was reserved) and idempotent bookkeeping-wise: the counter+ * only ever reflects reservations actually made, since callers null out their local reference+ * after releasing it (see {@link #launch}).+ */+ private void releaseCodeVolumeReference(String volName) {+ if (volName == null) {+ return;+ }+ volumesInFlight.computeIfPresent(volName, (k, count) -> count.decrementAndGet() > 0 ? count : null);+ }++ /**+ * Removes superseded code volumes whose grace period has elapsed. Scheduled at the same+ * interval as the grace period itself, so a volume is deleted within roughly one to two+ * intervals of becoming superseded. Not a hard deadline, since this is best-effort cleanup,+ * not a correctness requirement (a volume that outlives a few extra sweeps is still reclaimed by+ * `docker volume prune --filter label=floci` same as before this existed).+ */+ void cleanupSupersededVolumes() {+ long cutoff = System.currentTimeMillis() - VOLUME_CLEANUP_GRACE_MS;+ for (String volName : new java.util.ArrayList<>(volumesPendingCleanup.keySet())) {+ Long queuedAt = volumesPendingCleanup.get(volName);+ if (queuedAt == null || queuedAt > cutoff) {+ continue;+ }+ // Same lock ensureCodeVolume holds for this volume name: claiming and deleting it here+ // must not interleave with a concurrent launch resolving (and possibly rescuing) it.+ Object lock = codeVolumeLocks.computeIfAbsent(volName, k -> new Object());+ synchronized (lock) {+ // Re-check under the lock: a concurrent ensureCodeVolume may have rescued this+ // volume (removed it from volumesPendingCleanup) or requeued it with a fresher+ // timestamp between the check above and acquiring this lock.+ Long stillQueuedAt = volumesPendingCleanup.get(volName);+ if (stillQueuedAt == null || stillQueuedAt > cutoff) {+ continue;+ }+ java.util.concurrent.atomic.AtomicInteger inFlight = volumesInFlight.get(volName);+ if (inFlight != null && inFlight.get() > 0) {+ // A launch has resolved this volume but hasn't handed it to Docker yet, so no+ // real container-to-volume reference exists for removeVolume's own in-use check+ // to catch. The grace period alone isn't a safe upper bound on that launch's+ // duration (create() has been observed taking ~80s under daemon load), so requeue+ // for a later sweep instead of trusting elapsed time here.+ volumesPendingCleanup.put(volName, System.currentTimeMillis());+ continue;+ }+ volumesPendingCleanup.remove(volName, stillQueuedAt);+ if (lifecycleManager.removeVolume(volName)) {+ populatedCodeVolumes.remove(volName);+ LOG.debugv("Removed superseded code volume {0}", volName);+ } else {+ // Not confirmed gone: still in use (e.g. a slow-draining in-flight container+ // outlived the grace period), or the removal attempt itself failed for some+ // other reason (e.g. a transient daemon error). Either way, without this the+ // entry would be lost with no later sweep ever retrying it. Leave+ // populatedCodeVolumes alone too, since for all we know it's still there and+ // still valid.+ volumesPendingCleanup.put(volName, System.currentTimeMillis());+ }+ }+ }+ }++ /** Test-only: whether a per-volume lock object has ever been created for this volume name. */+ boolean hasCodeVolumeLock(String volName) {+ return codeVolumeLocks.containsKey(volName);+ }++ /**+ * Test-only: the same lock object ensureCodeVolume synchronizes on for this function's+ * current-volume transition, so a test can hold it directly to prove a concurrent launch blocks+ * on it rather than racing past it.+ */+ Object functionVolumeTransitionLockFor(String functionName) {+ return functionVolumeTransitionLocks.computeIfAbsent(functionName, k -> new Object());+ }++ /** Test-only: the current in-flight reference count for this volume name (0 if none). */+ int inFlightCount(String volName) {+ java.util.concurrent.atomic.AtomicInteger count = volumesInFlight.get(volName);+ return count == null ? 0 : count.get();+ }+private void populateCodeVolume(String volName, LambdaFunction fn, String image) {lifecycleManager.ensureVolume(volName);String shortId = java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8);diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.javaindex 4de92c240c..5e89c941f2 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java@@ -48,6 +48,7 @@ import java.util.Optional;import java.util.concurrent.atomic.AtomicBoolean;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;@@ -592,7 +593,9 @@ class ContainerLauncherTest {verify(lifecycleManager, times(2)).create(any());verify(lifecycleManager, atLeastOnce()).ensureVolume(any());verify(lifecycleManager, times(1)).stopAndRemove(any(), any()); // the helper only- // Old code-version volumes are NOT eagerly deleted (race fix); they are label-pruned instead.+ // A superseded code-version volume is never deleted synchronously within a single launch+ // (see the cleanupSupersededVolumes tests below for the deferred sweep; this fn has no+ // prior volume to supersede, since it's the fn's first deploy here).verify(lifecycleManager, never()).removeVolume(any());}@@ -631,6 +634,461 @@ class ContainerLauncherTest {verify(lifecycleManager, never()).startCreated(any(), any());}+ @Test+ void launchFunction_reprovisionsCodeVolume_whenDockerHasNoRecordOfIt() throws Exception {+ // Regression for #2164: the "populated" bookkeeping is only ever an in-memory cache of+ // Docker's state, so a volume removed out of band (manual `docker volume rm`, or a stale+ // in-memory flag left over from a process restart racing a prune) must not be trusted.+ Path codePath = Files.createDirectory(tempDir.resolve("revalidate-code"));+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("revalidate-fn");+ fn.setRuntime("nodejs20.x");+ fn.setHandler("index.handler");+ fn.setCodeLocalPath(codePath.toString());+ fn.setCodeSha256("revalidate-fn-sha-v1");++ // Docker never reports this volume as existing, simulating it being gone every time+ // ensureCodeVolume checks, regardless of what the in-memory flag says.+ when(lifecycleManager.volumeExists(anyString())).thenReturn(false);++ long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ try {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;+ launcher.launch(fn);+ launcher.launch(fn);+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;+ }++ // Populated twice, once per launch, because the bookkeeping alone is never trusted.+ assertEquals(2, capturedRemotePaths.stream().filter("/var/task"::equals).count(),+ "each launch should repopulate since Docker never confirms the volume exists");+ }++ @Test+ void cleanupSupersededVolumes_removesPreviousVersionVolume_onceGracePeriodElapses() throws Exception {+ Path codePath = Files.createDirectory(tempDir.resolve("cleanup-code"));+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB+ // No volumeExists stub needed: v1 and v2 are each a first-ever population of their own+ // distinct volume name, so populatedCodeVolumes.contains(volName) is false both times+ // ensureCodeVolume checks it, short-circuiting the volumeExists call away entirely.++ LambdaFunction v1 = new LambdaFunction();+ v1.setFunctionName("cleanup-fn");+ v1.setRuntime("nodejs20.x");+ v1.setHandler("index.handler");+ v1.setCodeLocalPath(codePath.toString());+ v1.setCodeSha256("cleanup-fn-sha-v1");+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);++ LambdaFunction v2 = new LambdaFunction();+ v2.setFunctionName("cleanup-fn");+ v2.setRuntime("nodejs20.x");+ v2.setHandler("index.handler");+ v2.setCodeLocalPath(codePath.toString());+ v2.setCodeSha256("cleanup-fn-sha-v2");+ String volumeV2 = ContainerLauncher.codeVolumeName(v2);++ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;+ try {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;+ launcher.launch(v1);+ launcher.launch(v2);++ // The v1 volume is superseded but not yet cleaned up: the grace period hasn't elapsed.+ launcher.cleanupSupersededVolumes();+ verify(lifecycleManager, never()).removeVolume(volumeV1);++ // Once the grace period has (trivially) elapsed, the sweep removes exactly the+ // superseded v1 volume, not the current v2 one.+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;+ launcher.cleanupSupersededVolumes();+ verify(lifecycleManager, times(1)).removeVolume(volumeV1);+ verify(lifecycleManager, never()).removeVolume(volumeV2);+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;+ }+ }++ @Test+ void rollingBackToAPreviousCodeVersion_rescuesItsVolumeFromCleanup() throws Exception {+ // Regression: v1 -> v2 -> back to v1 (e.g. a CloudFormation rollback) resolves v1's volume+ // name again, which is already populated - a fast-path hit in ensureCodeVolume. Without+ // reconciling functionCurrentVolume/volumesPendingCleanup on that path too, v1 would stay+ // queued as superseded from the v1 -> v2 step and the next sweep would delete the volume+ // this function is actively using again.+ Path codePath = Files.createDirectory(tempDir.resolve("rollback-code"));+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB++ LambdaFunction v1 = new LambdaFunction();+ v1.setFunctionName("rollback-fn");+ v1.setRuntime("nodejs20.x");+ v1.setHandler("index.handler");+ v1.setCodeLocalPath(codePath.toString());+ v1.setCodeSha256("rollback-fn-sha-v1");+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);++ LambdaFunction v2 = new LambdaFunction();+ v2.setFunctionName("rollback-fn");+ v2.setRuntime("nodejs20.x");+ v2.setHandler("index.handler");+ v2.setCodeLocalPath(codePath.toString());+ v2.setCodeSha256("rollback-fn-sha-v2");+ String volumeV2 = ContainerLauncher.codeVolumeName(v2);++ // Needed for the rollback launch below: v1's volume is already populated by then, so its+ // fast path actually evaluates volumeExists instead of short-circuiting past it.+ when(lifecycleManager.volumeExists(volumeV1)).thenReturn(true);++ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;+ try {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;+ launcher.launch(v1);+ launcher.launch(v2);+ launcher.launch(v1); // roll back++ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;+ launcher.cleanupSupersededVolumes();+ verify(lifecycleManager, never()).removeVolume(volumeV1);+ verify(lifecycleManager, times(1)).removeVolume(volumeV2);+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;+ }+ }++ @Test+ void cleanupSupersededVolumes_retriesOnALaterSweep_whenTheVolumeIsStillInUse() throws Exception {+ // Regression: removeVolume() silently no-ops when Docker refuses because the volume is+ // still in use (e.g. a slow-draining in-flight container outliving the grace period). The+ // pending-cleanup entry must not be discarded in that case, or the volume is orphaned until+ // someone manually runs `docker volume prune`- the exact problem this fix exists to avoid.+ Path codePath = Files.createDirectory(tempDir.resolve("retry-code"));+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB++ LambdaFunction v1 = new LambdaFunction();+ v1.setFunctionName("retry-fn");+ v1.setRuntime("nodejs20.x");+ v1.setHandler("index.handler");+ v1.setCodeLocalPath(codePath.toString());+ v1.setCodeSha256("retry-fn-sha-v1");+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);++ LambdaFunction v2 = new LambdaFunction();+ v2.setFunctionName("retry-fn");+ v2.setRuntime("nodejs20.x");+ v2.setHandler("index.handler");+ v2.setCodeLocalPath(codePath.toString());+ v2.setCodeSha256("retry-fn-sha-v2");++ // v1's volume persists (still in use) no matter how many times removal is attempted.+ when(lifecycleManager.removeVolume(volumeV1)).thenReturn(false);++ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;+ try {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;+ launcher.launch(v1);+ launcher.launch(v2);++ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;+ launcher.cleanupSupersededVolumes();+ verify(lifecycleManager, times(1)).removeVolume(volumeV1);++ // A later sweep must still retry it, not have silently dropped it after the first+ // no-op'd attempt.+ launcher.cleanupSupersededVolumes();+ verify(lifecycleManager, times(2)).removeVolume(volumeV1);+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;+ }+ }++ @Test+ void cleanupAndAConcurrentRollback_areMutuallyExclusiveForTheSameVolume() throws Exception {+ // Regression: without a shared per-volume lock, a sweep that has already claimed a+ // superseded volume (removed its pending-cleanup entry) could race a concurrent rollback+ // that resolves the same volume name, marks it current, and hands it to a new container -+ // while the sweep proceeds to actually delete it underneath that launch. Proving this+ // requires real concurrency: this test forces cleanupSupersededVolumes to block mid-deletion+ // (still holding the volume's lock) and asserts a concurrent rollback blocks too, rather than+ // racing past it.+ Path codePath = Files.createDirectory(tempDir.resolve("race-code"));+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB++ LambdaFunction v1 = new LambdaFunction();+ v1.setFunctionName("race-fn");+ v1.setRuntime("nodejs20.x");+ v1.setHandler("index.handler");+ v1.setCodeLocalPath(codePath.toString());+ v1.setCodeSha256("race-fn-sha-v1");+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);++ LambdaFunction v2 = new LambdaFunction();+ v2.setFunctionName("race-fn");+ v2.setRuntime("nodejs20.x");+ v2.setHandler("index.handler");+ v2.setCodeLocalPath(codePath.toString());+ v2.setCodeSha256("race-fn-sha-v2");++ when(lifecycleManager.volumeExists(volumeV1)).thenReturn(true);++ java.util.concurrent.CountDownLatch cleanupHoldingLock = new java.util.concurrent.CountDownLatch(1);+ java.util.concurrent.CountDownLatch releaseCleanup = new java.util.concurrent.CountDownLatch(1);+ doAnswer(inv -> {+ cleanupHoldingLock.countDown();+ // Blocks here while still holding volumeV1's per-volume lock, simulating a slow delete.+ assertTrue(releaseCleanup.await(5, java.util.concurrent.TimeUnit.SECONDS),+ "test did not release cleanup in time");+ return null;+ }).when(lifecycleManager).removeVolume(volumeV1);++ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;+ try {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;+ launcher.launch(v1);+ launcher.launch(v2);+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;++ Thread cleanupThread = new Thread(launcher::cleanupSupersededVolumes);+ cleanupThread.start();+ assertTrue(cleanupHoldingLock.await(5, java.util.concurrent.TimeUnit.SECONDS),+ "cleanup never reached removeVolume");++ java.util.concurrent.atomic.AtomicBoolean rollbackReturned =+ new java.util.concurrent.atomic.AtomicBoolean(false);+ Thread rollbackThread = new Thread(() -> {+ launcher.launch(v1);+ rollbackReturned.set(true);+ });+ rollbackThread.start();++ // Give the rollback every chance to (wrongly) proceed if the lock weren't shared.+ Thread.sleep(200);+ assertFalse(rollbackReturned.get(),+ "rollback must block while cleanup holds the volume's lock, not race past it");++ releaseCleanup.countDown();+ cleanupThread.join(5000);+ rollbackThread.join(5000);+ assertTrue(rollbackReturned.get(), "rollback should complete once cleanup releases the lock");+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;+ }+ }++ @Test+ void cleanupSkipsAVolumeStillInFlight_evenPastItsGracePeriod() throws Exception {+ // Regression: a launch that has resolved its code volume (ensureCodeVolume returned, its+ // per-volume lock released) but hasn't yet reached lifecycleManager.create() has no real+ // Docker container-to-volume reference for removeVolume's own in-use check to protect - and+ // create() itself has no proven upper bound (observed ~80s under daemon load, longer than+ // the default 60s grace period). This forces a second launch of v1 to block right before+ // create() while a redeploy to v2 supersedes its volume and the grace period is made to+ // elapse instantly, then asserts cleanup does NOT delete v1's volume while that launch is+ // still in flight - only once it completes and releases its reference.+ Path codePath = Files.createDirectory(tempDir.resolve("inflight-code"));+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB++ LambdaFunction v1 = new LambdaFunction();+ v1.setFunctionName("inflight-fn");+ v1.setRuntime("nodejs20.x");+ v1.setHandler("index.handler");+ v1.setCodeLocalPath(codePath.toString());+ v1.setCodeSha256("inflight-fn-sha-v1");+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);++ LambdaFunction v2 = new LambdaFunction();+ v2.setFunctionName("inflight-fn");+ v2.setRuntime("nodejs20.x");+ v2.setHandler("index.handler");+ v2.setCodeLocalPath(codePath.toString());+ v2.setCodeSha256("inflight-fn-sha-v2");++ // Needed for the delayed re-launch below: v1's volume is already populated by then, so its+ // fast path actually evaluates volumeExists instead of short-circuiting past it.+ when(lifecycleManager.volumeExists(volumeV1)).thenReturn(true);++ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;+ try {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;++ // Pre-populate v1's volume with an ordinary launch, before installing the create()+ // blocking stub below - otherwise that stub would catch populateCodeVolume's own+ // helper-container create() call instead of the real container's.+ launcher.launch(v1);++ java.util.concurrent.CountDownLatch launchReachedCreate = new java.util.concurrent.CountDownLatch(1);+ java.util.concurrent.CountDownLatch releaseLaunch = new java.util.concurrent.CountDownLatch(1);+ java.util.concurrent.atomic.AtomicBoolean blockedOnce = new java.util.concurrent.atomic.AtomicBoolean(false);+ doAnswer(inv -> {+ if (blockedOnce.compareAndSet(false, true)) {+ // Only the delayed re-launch of v1 (the first caller after this stub is+ // installed) blocks here; v2's later launch below must not, or the test would+ // deadlock itself waiting on its own main thread to release the latch.+ launchReachedCreate.countDown();+ assertTrue(releaseLaunch.await(5, java.util.concurrent.TimeUnit.SECONDS),+ "test did not release the delayed launch in time");+ }+ return "container-123";+ }).when(lifecycleManager).create(any());++ Thread launchThread = new Thread(() -> launcher.launch(v1));+ launchThread.start();+ assertTrue(launchReachedCreate.await(5, java.util.concurrent.TimeUnit.SECONDS),+ "v1's delayed re-launch never reached create()");++ assertEquals(1, launcher.inFlightCount(volumeV1),+ "ensureCodeVolume must mark the volume in-flight before create() confirms it");++ // A redeploy resolves v2 and supersedes v1's volume while v1's own launch is still stuck.+ launcher.launch(v2);+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1; // grace period elapses instantly++ launcher.cleanupSupersededVolumes();+ verify(lifecycleManager, never()).removeVolume(volumeV1);++ // Let the delayed launch finish; it releases its in-flight reference on success.+ releaseLaunch.countDown();+ launchThread.join(5000);+ assertEquals(0, launcher.inFlightCount(volumeV1),+ "in-flight count must be released once create() succeeds");++ // Nothing is in flight anymore, so a later sweep is free to actually delete it.+ launcher.cleanupSupersededVolumes();+ verify(lifecycleManager, times(1)).removeVolume(volumeV1);+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;+ }+ }++ @Test+ void concurrentRedeploysOfTheSameFunction_serializeTheCurrentVolumeTransition() throws Exception {+ // Regression: functionCurrentVolume/volumesPendingCleanup reconciliation ran under the+ // per-volume lock only, but two code versions of the same function resolve to two different+ // volume names and so hold two different per-volume locks - meaning that reconciliation+ // could interleave across a rapid back-to-back redeploy (v2 then v3), leaving the actually-+ // current v3 volume mistakenly queued for cleanup while stale v2 stayed recorded as current.+ // The three statements involved (remove pending-cleanup entry, swap in the new current+ // volume, queue whatever was displaced) are plain map operations with no interception point+ // inside them, so forcing that exact interleaving isn't reliably doable in a test. This+ // instead verifies the fix's actual mechanism directly - a concurrent launch for a second+ // code version of the same function must block on the same per-function lock object while+ // another is still transitioning - the same style cleanupAndAConcurrentRollback... already+ // uses to prove the per-volume lock.+ Path codePath = Files.createDirectory(tempDir.resolve("xfn-code"));+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB++ LambdaFunction v1 = new LambdaFunction();+ v1.setFunctionName("xfn");+ v1.setRuntime("nodejs20.x");+ v1.setHandler("index.handler");+ v1.setCodeLocalPath(codePath.toString());+ v1.setCodeSha256("xfn-sha-v1");++ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ try {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;++ Object functionLock = launcher.functionVolumeTransitionLockFor("xfn");+ java.util.concurrent.CountDownLatch testHoldingLock = new java.util.concurrent.CountDownLatch(1);+ java.util.concurrent.CountDownLatch releaseLock = new java.util.concurrent.CountDownLatch(1);+ Thread holderThread = new Thread(() -> {+ synchronized (functionLock) {+ testHoldingLock.countDown();+ try {+ assertTrue(releaseLock.await(5, java.util.concurrent.TimeUnit.SECONDS),+ "test did not release the function lock in time");+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+ });+ holderThread.start();+ assertTrue(testHoldingLock.await(5, java.util.concurrent.TimeUnit.SECONDS),+ "test thread never acquired the function lock");++ AtomicBoolean launchReturned = new AtomicBoolean(false);+ Thread launchThread = new Thread(() -> {+ launcher.launch(v1);+ launchReturned.set(true);+ });+ launchThread.start();++ // Give the launch every chance to (wrongly) proceed if the transition weren't guarded+ // by this same lock.+ Thread.sleep(200);+ assertFalse(launchReturned.get(),+ "launch must block on the function lock while another holder has it, not race past it");++ releaseLock.countDown();+ holderThread.join(5000);+ launchThread.join(5000);+ assertTrue(launchReturned.get(), "launch should complete once the function lock is released");+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;+ }+ }++ @Test+ void codeVolumeLocks_isNeverPruned_evenAfterCleanupDeletesTheVolume() throws Exception {+ // Regression: pruning a volume's lock entry while a waiter still held a reference to the+ // removed entry's lock object let a third caller's computeIfAbsent create a *different* lock+ // object for the same volume name, so the waiter (once granted the old lock) and that third+ // caller (holding the new one) could run their supposedly mutually-exclusive sections+ // concurrently - defeating the whole point. The actual JVM interleaving needed to trigger+ // that is timing-dependent and not reliably forceable in a test, so this instead verifies the+ // fix's real invariant directly: the lock entry must survive a full populate-then-delete+ // cycle unchanged, which is what actually guarantees computeIfAbsent always returns the same+ // object for a given volume name for the life of the process.+ Path codePath = Files.createDirectory(tempDir.resolve("lock-persist-code"));+ Files.write(codePath.resolve("bundle.bin"), new byte[8 * 1024]); // 8 KiB++ LambdaFunction v1 = new LambdaFunction();+ v1.setFunctionName("lock-persist-fn");+ v1.setRuntime("nodejs20.x");+ v1.setHandler("index.handler");+ v1.setCodeLocalPath(codePath.toString());+ v1.setCodeSha256("lock-persist-fn-sha-v1");+ String volumeV1 = ContainerLauncher.codeVolumeName(v1);++ LambdaFunction v2 = new LambdaFunction();+ v2.setFunctionName("lock-persist-fn");+ v2.setRuntime("nodejs20.x");+ v2.setHandler("index.handler");+ v2.setCodeLocalPath(codePath.toString());+ v2.setCodeSha256("lock-persist-fn-sha-v2");++ long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;+ try {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;+ launcher.launch(v1);+ assertTrue(launcher.hasCodeVolumeLock(volumeV1), "lock must exist right after populate");++ launcher.launch(v2); // supersedes v1's volume, queues it for cleanup+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;+ launcher.cleanupSupersededVolumes(); // deletes v1's volume for real++ assertTrue(launcher.hasCodeVolumeLock(volumeV1),+ "lock must still exist even after the volume itself was deleted - "+ + "pruning it here is exactly the bug this test guards against");+ } finally {+ ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;+ }+ }+/** Builds a tar archive matching what {@code docker cp}/{@code copyArchiveFromContainerCmd}* returns for a directory: the directory itself as a leading entry, then each name as a direct* child, executable. */