Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

perf(lambda): reuse a populated code volume across restarts via a completion marker (#1827)

Alex Banna5d61e9523938
diff --git a/docker/Dockerfile.native-package b/docker/Dockerfile.native-package
index 6255a716c3..b9a0aba991 100644
--- a/docker/Dockerfile.native-package
+++ b/docker/Dockerfile.native-package
@@ -20,7 +20,8 @@ RUN set -eux; \
aarch64) gosuArch='arm64'; gosuSha256="$GOSU_ARM64_SHA256" ;; \
*) echo >&2 "unsupported arch: $arch"; exit 1 ;; \
esac; \
- curl -fsSL -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${gosuArch}"; \
+ curl -fsSL --retry 5 --retry-all-errors --retry-delay 2 \
+ -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${gosuArch}"; \
echo "${gosuSha256} /usr/local/bin/gosu" | sha256sum -c -; \
chmod +x /usr/local/bin/gosu; \
gosu --version; \
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 7eb9606a55..9ceeb79a0d 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -5,6 +5,8 @@ import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CreateContainerCmd;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
+import com.github.dockerjava.api.command.ListVolumesResponse;
import com.github.dockerjava.api.exception.DockerException;
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.Bind;
@@ -26,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@@ -539,6 +542,34 @@ public class ContainerLifecycleManager {
}
}
+ /**
+ * Attempts to take an authoritative snapshot of all named volumes in the container runtime.
+ * An empty optional means the runtime could not be queried; it is intentionally distinct from
+ * a successful query that returned an empty set so cleanup callers fail closed.
+ */
+ public Optional<Set<String>> tryListVolumeNames() {
+ try {
+ ListVolumesResponse response = dockerClient.listVolumesCmd().exec();
+ if (response == null) {
+ LOG.warn("Container runtime returned no volume-list response");
+ return Optional.empty();
+ }
+ List<InspectVolumeResponse> volumes = response.getVolumes();
+ if (volumes == null) {
+ LOG.warn("Container runtime returned a volume-list response without an inventory");
+ return Optional.empty();
+ }
+ Set<String> names = volumes.stream()
+ .map(InspectVolumeResponse::getName)
+ .filter(name -> name != null && !name.isBlank())
+ .collect(java.util.stream.Collectors.toUnmodifiableSet());
+ return Optional.of(names);
+ } catch (RuntimeException e) {
+ LOG.warnv("Failed to list container-runtime volumes: {0}", e.getMessage());
+ return Optional.empty();
+ }
+ }
+
private HostConfig buildHostConfig(ContainerSpec spec) {
HostConfig hostConfig = HostConfig.newHostConfig();
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
index 9cba571cec..3ad399afd6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
@@ -33,8 +33,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateFactory;
@@ -48,6 +50,7 @@ import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
/**
* Starts and stops Docker containers for Lambda function execution.
@@ -523,6 +526,8 @@ public class ContainerLauncher {
// `docker volume prune` would otherwise still read as "populated" here even though Docker has
// nothing under that name, silently mounting an empty /var/task into the next container.
// ensureCodeVolume re-checks lifecycleManager.volumeExists() rather than trusting this alone.
+ private static final String CODE_VOLUME_MARKER_DIR = "lambda-codevol-markers";
+ private static final String CODE_VOLUME_MARKER_PREFIX = "floci-code-";
private final java.util.Set<String> populatedCodeVolumes = java.util.concurrent.ConcurrentHashMap.newKeySet();
// Deliberately never pruned: removing an entry while a caller elsewhere still held a reference
// to its lock object let a third caller's computeIfAbsent create a replacement lock for the same
@@ -532,6 +537,7 @@ public class ContainerLauncher {
// combinations ever launched, each entry a single Object; negligible next to the disk space the
// rest of this fix reclaims.
private final java.util.concurrent.ConcurrentHashMap<String, Object> codeVolumeLocks = new java.util.concurrent.ConcurrentHashMap<>();
+ private final Object codeVolumeMarkerIoLock = new Object();
/** Each function's current code volume, so a redeploy can queue the superseded one for cleanup. */
private final java.util.concurrent.ConcurrentHashMap<String, String> functionCurrentVolume =
@@ -589,7 +595,7 @@ public class ContainerLauncher {
* throwaway helper container that has the volume mounted read-write; every real container then
* just mounts the volume read-only, turning a ~95s per-container copy into a ~0.2s mount.
*/
- private String ensureCodeVolume(LambdaFunction fn, String image) {
+ String ensureCodeVolume(LambdaFunction fn, String image) {
String volName = codeVolumeName(fn);
// Held for the whole resolve-and-reconcile, not just the populate branch: this is the same
// lock cleanupSupersededVolumes acquires before claiming a volume for deletion, so a launch
@@ -605,15 +611,22 @@ public class ContainerLauncher {
Object lock = codeVolumeLocks.computeIfAbsent(volName, k -> new Object());
synchronized (lock) {
if (!(populatedCodeVolumes.contains(volName) && lifecycleManager.volumeExists(volName))) {
- // Either never populated, or the bookkeeping says populated but Docker disagrees
- // (removed out of band). Repopulate rather than trust a stale flag.
- long t0 = System.currentTimeMillis();
- LOG.infov("Populating code volume {0} for function {1} (one-time per code version)",
- volName, fn.getFunctionName());
- populateCodeVolume(volName, fn, image);
+ // Either never populated in this process, or the bookkeeping says populated but
+ // Docker disagrees (removed out of band). A completion marker from an earlier run
+ // is authoritative only while its independently persisted volume still exists.
+ if (reusePersistedCodeVolumeOrRemoveStaleMarker(volName)) {
+ LOG.infov("Reusing code volume {0} populated by a previous run (marker present)",
+ volName);
+ } else {
+ long t0 = System.currentTimeMillis();
+ LOG.infov("Populating code volume {0} for function {1} (one-time per code version)",
+ volName, fn.getFunctionName());
+ populateCodeVolume(volName, fn, image);
+ writeCodeVolumeMarkerAndPruneOrphans(volName);
+ LOG.infov("Populated code volume {0} in {1}ms; future cold starts mount it instead of copying",
+ volName, System.currentTimeMillis() - t0);
+ }
populatedCodeVolumes.add(volName);
- LOG.infov("Populated code volume {0} in {1}ms; future cold starts mount it instead of copying",
- volName, System.currentTimeMillis() - t0);
}
// Reconcile bookkeeping for the resolved volume even when the check above already found
// it populated: a redeploy rolled back to an older code version resolves a volume name
@@ -730,7 +743,7 @@ public class ContainerLauncher {
return count == null ? 0 : count.get();
}
- private void populateCodeVolume(String volName, LambdaFunction fn, String image) {
+ void populateCodeVolume(String volName, LambdaFunction fn, String image) {
lifecycleManager.ensureVolume(volName);
String shortId = java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8);
// A minimal helper container (sleep) with the volume mounted read-write at /var/task; we
@@ -749,16 +762,113 @@ public class ContainerLauncher {
try {
helperId = lifecycleManager.create(helperSpec);
lifecycleManager.startCreated(helperId, helperSpec);
- copyDirToContainer(lifecycleManager.getDockerClient(), helperId,
+ copyDirToContainerStrict(lifecycleManager.getDockerClient(), helperId,
Path.of(fn.getCodeLocalPath()), TASK_DIR, fn.getFunctionName());
} finally {
if (helperId != null) {
- try { lifecycleManager.stopAndRemove(helperId, null); } catch (Exception ignore) { /* best effort */ }
+ try {
+ lifecycleManager.stopAndRemove(helperId, null);
+ } catch (Exception e) {
+ LOG.warnv("Could not remove code-volume helper {0}: {1}", helperId, e.getMessage());
+ }
}
POPULATE_SEMAPHORE.release();
}
}
+ // A code volume survives an emulator restart, but the in-process populatedCodeVolumes set does not.
+ // A persisted completion marker lets a later run recognize an already-populated, content-addressed
+ // volume and skip re-copying its files. The marker is stored under storage.persistent-path
+ // regardless of storage mode; reuse still requires the independently persisted Docker volume.
+ private Path codeVolumeMarkerPath(String volName) {
+ return Path.of(config.storage().persistentPath(), CODE_VOLUME_MARKER_DIR, volName);
+ }
+
+ /**
+ * Reuses a completed volume when both marker and volume exist. If the volume is absent or cannot
+ * be verified, removes the marker before repopulation so a failed copy cannot leave the old
+ * completion signal beside partial data.
+ */
+ private boolean reusePersistedCodeVolumeOrRemoveStaleMarker(String volName) {
+ synchronized (codeVolumeMarkerIoLock) {
+ Path marker = codeVolumeMarkerPath(volName);
+ if (!Files.isRegularFile(marker, java.nio.file.LinkOption.NOFOLLOW_LINKS)) {
+ return false;
+ }
+ java.util.Optional<java.util.Set<String>> volumeNames = lifecycleManager.tryListVolumeNames();
+ boolean inventoryUnavailable = volumeNames.isEmpty();
+ if (!inventoryUnavailable && volumeNames.get().contains(volName)) {
+ return true;
+ }
+ try {
+ Files.deleteIfExists(marker);
+ if (inventoryUnavailable) {
+ LOG.warnv("Could not verify backing volume {0}; removed its marker and re-populating",
+ volName);
+ } else {
+ LOG.debugv("Removed stale code-volume marker {0}; backing volume is absent", marker);
+ }
+ } catch (IOException e) {
+ throw new IllegalStateException("Could not invalidate code-volume marker " + marker, e);
+ }
+ return false;
+ }
+ }
+
+ private void writeCodeVolumeMarkerAndPruneOrphans(String volName) {
+ synchronized (codeVolumeMarkerIoLock) {
+ Path marker = codeVolumeMarkerPath(volName);
+ try {
+ Path markerDir = marker.getParent();
+ Files.createDirectories(markerDir);
+ Path tempMarker = Files.createTempFile(markerDir, marker.getFileName().toString(), ".tmp");
+ try {
+ Files.writeString(tempMarker, "");
+ try {
+ Files.move(tempMarker, marker, StandardCopyOption.REPLACE_EXISTING,
+ StandardCopyOption.ATOMIC_MOVE);
+ } catch (AtomicMoveNotSupportedException e) {
+ Files.move(tempMarker, marker, StandardCopyOption.REPLACE_EXISTING);
+ }
+ } catch (IOException | RuntimeException e) {
+ try {
+ Files.deleteIfExists(tempMarker);
+ } catch (IOException cleanupError) {
+ e.addSuppressed(cleanupError);
+ }
+ throw e;
+ }
+ } catch (IOException e) {
+ // A missing marker only costs one re-populate on the next boot; never fail a launch over it.
+ LOG.warnv("Could not write code-volume marker {0}: {1}", marker, e.getMessage());
+ }
+ pruneOrphanCodeVolumeMarkers(marker.getParent());
+ }
+ }
+
+ private void pruneOrphanCodeVolumeMarkers(Path markerDir) {
+ java.util.Optional<java.util.Set<String>> volumeNames = lifecycleManager.tryListVolumeNames();
+ if (volumeNames.isEmpty() || !Files.isDirectory(markerDir)) {
+ return;
+ }
+ try (java.util.stream.Stream<Path> markers = Files.list(markerDir)) {
+ markers.filter(path -> Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS))
+ .filter(path -> path.getFileName().toString().startsWith(CODE_VOLUME_MARKER_PREFIX))
+ .filter(path -> !volumeNames.get().contains(path.getFileName().toString()))
+ .forEach(path -> {
+ try {
+ Files.deleteIfExists(path);
+ LOG.debugv("Pruned orphan code-volume marker {0}", path);
+ } catch (IOException e) {
+ LOG.warnv("Could not prune orphan code-volume marker {0}: {1}",
+ path, e.getMessage());
+ }
+ });
+ } catch (IOException e) {
+ LOG.warnv("Could not scan code-volume marker directory {0}: {1}", markerDir, e.getMessage());
+ }
+ }
+
/**
* Total code size (bytes) at or above which a function's {@code /var/task} is served from a
* shared read-only volume rather than copied into every container. This is roughly the point
@@ -843,26 +953,68 @@ public class ContainerLauncher {
*/
private static final int TAR_PIPE_BUFFER_BYTES = 16 * 1024 * 1024;
+ @FunctionalInterface
+ interface DirectoryTarWriter {
+ void write(Path sourceDir, OutputStream out) throws IOException;
+ }
+
private void copyDirToContainer(DockerClient dockerClient, String containerId,
Path sourceDir, String remotePath, String functionName) {
+ copyDirToContainer(dockerClient, containerId, sourceDir, remotePath, functionName,
+ ContainerLauncher::createTarFromDir);
+ }
+
+ static void copyDirToContainer(DockerClient dockerClient, String containerId,
+ Path sourceDir, String remotePath, String functionName,
+ DirectoryTarWriter tarWriter) {
+ copyDirToContainer(dockerClient, containerId, sourceDir, remotePath, functionName,
+ tarWriter, false);
+ }
+
+ private void copyDirToContainerStrict(DockerClient dockerClient, String containerId,
+ Path sourceDir, String remotePath, String functionName) {
+ copyDirToContainerStrict(dockerClient, containerId, sourceDir, remotePath, functionName,
+ ContainerLauncher::createTarFromDir);
+ }
+
+ static void copyDirToContainerStrict(DockerClient dockerClient, String containerId,
+ Path sourceDir, String remotePath, String functionName,
+ DirectoryTarWriter tarWriter) {
+ copyDirToContainer(dockerClient, containerId, sourceDir, remotePath, functionName,
+ tarWriter, true);
+ }
+
+ private static void copyDirToContainer(DockerClient dockerClient, String containerId,
+ Path sourceDir, String remotePath, String functionName,
+ DirectoryTarWriter tarWriter, boolean failOnTarFailure) {
// No per-copy gating here: the heavy /var/task populate for large code already holds a
// POPULATE_SEMAPHORE permit; small-code direct copies and layer copies are light enough
// to run unthrottled.
try (java.io.PipedOutputStream pos = new java.io.PipedOutputStream();
java.io.PipedInputStream pis = new java.io.PipedInputStream(pos, TAR_PIPE_BUFFER_BYTES)) {
- new Thread(() -> {
+ AtomicReference<IOException> tarFailure = new AtomicReference<>();
+ Thread tarStreamer = new Thread(() -> {
try (pos) {
- createTarFromDir(sourceDir, pos);
+ tarWriter.write(sourceDir, pos);
} catch (IOException e) {
- LOG.errorv("Failed to stream tar for function {0}: {1}", functionName, e.getMessage());
+ if (failOnTarFailure) {
+ tarFailure.set(e);
+ } else {
+ LOG.errorv("Failed to stream directory tar for function {0}: {1}",
+ functionName, e.getMessage());
+ }
}
- }, "tar-streamer-dir-" + functionName).start();
+ }, "tar-streamer-dir-" + functionName);
+ tarStreamer.start();
dockerClient.copyArchiveToContainerCmd(containerId)
.withRemotePath(remotePath)
.withTarInputStream(pis)
.exec();
+ if (failOnTarFailure) {
+ waitForTarStreamer(tarStreamer, tarFailure, functionName, sourceDir);
+ }
LOG.debugv("Copied directory {0} into container {1} at {2}", sourceDir, containerId, remotePath);
} catch (Exception e) {
// Fail loudly so launch() cleans up the half-built container instead of leaking it.
@@ -902,6 +1054,22 @@ public class ContainerLauncher {
}
}
+ private static void waitForTarStreamer(Thread tarStreamer, AtomicReference<IOException> tarFailure,
+ String functionName, Path sourcePath) throws IOException {
+ try {
+ tarStreamer.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while streaming tar for function " + functionName
+ + " from " + sourcePath, e);
+ }
+ IOException failure = tarFailure.get();
+ if (failure != null) {
+ throw new IOException("Failed to stream tar for function " + functionName
+ + " from " + sourcePath, failure);
+ }
+ }
+
private static boolean isProvidedRuntime(String runtime) {
return runtime != null && runtime.startsWith("provided");
}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
index e7f757d627..c4811d1452 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
@@ -5,6 +5,8 @@ import com.github.dockerjava.api.command.CreateContainerCmd;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.command.InspectVolumeCmd;
import com.github.dockerjava.api.command.InspectVolumeResponse;
+import com.github.dockerjava.api.command.ListVolumesCmd;
+import com.github.dockerjava.api.command.ListVolumesResponse;
import com.github.dockerjava.api.command.RemoveContainerCmd;
import com.github.dockerjava.api.command.StartContainerCmd;
import com.github.dockerjava.api.command.StopContainerCmd;
@@ -23,6 +25,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.OptionalInt;
+import java.util.Set;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
@@ -155,6 +158,62 @@ class ContainerLifecycleManagerVolumeTest {
assertFalse(manager.volumeExists("some-volume"));
}
+ @Test
+ void tryListVolumeNamesReturnsAuthoritativeSnapshot() {
+ ListVolumesCmd cmd = mock(ListVolumesCmd.class);
+ ListVolumesResponse response = mock(ListVolumesResponse.class);
+ InspectVolumeResponse first = mock(InspectVolumeResponse.class);
+ InspectVolumeResponse second = mock(InspectVolumeResponse.class);
+ when(dockerClient.listVolumesCmd()).thenReturn(cmd);
+ when(cmd.exec()).thenReturn(response);
+ when(response.getVolumes()).thenReturn(java.util.List.of(first, second));
+ when(first.getName()).thenReturn("floci-code-first");
+ when(second.getName()).thenReturn("shared-data");
+
+ assertEquals(Optional.of(Set.of("floci-code-first", "shared-data")),
+ manager.tryListVolumeNames());
+ }
+
+ @Test
+ void tryListVolumeNamesDistinguishesEmptyInventory() {
+ ListVolumesCmd cmd = mock(ListVolumesCmd.class);
+ ListVolumesResponse response = mock(ListVolumesResponse.class);
+ when(dockerClient.listVolumesCmd()).thenReturn(cmd);
+ when(cmd.exec()).thenReturn(response);
+ when(response.getVolumes()).thenReturn(java.util.List.of());
+
+ assertEquals(Optional.of(Set.of()), manager.tryListVolumeNames());
+ }
+
+ @Test
+ void tryListVolumeNamesReturnsEmptyOptionalForNullResponse() {
+ ListVolumesCmd cmd = mock(ListVolumesCmd.class);
+ when(dockerClient.listVolumesCmd()).thenReturn(cmd);
+ when(cmd.exec()).thenReturn(null);
+
+ assertEquals(Optional.empty(), manager.tryListVolumeNames());
+ }
+
+ @Test
+ void tryListVolumeNamesReturnsEmptyOptionalForNullVolumeList() {
+ ListVolumesCmd cmd = mock(ListVolumesCmd.class);
+ ListVolumesResponse response = mock(ListVolumesResponse.class);
+ when(dockerClient.listVolumesCmd()).thenReturn(cmd);
+ when(cmd.exec()).thenReturn(response);
+ when(response.getVolumes()).thenReturn(null);
+
+ assertEquals(Optional.empty(), manager.tryListVolumeNames());
+ }
+
+ @Test
+ void tryListVolumeNamesReturnsEmptyOptionalOnDockerFailure() {
+ ListVolumesCmd cmd = mock(ListVolumesCmd.class);
+ when(dockerClient.listVolumesCmd()).thenReturn(cmd);
+ when(cmd.exec()).thenThrow(new DockerException("Connection refused", 500));
+
+ assertEquals(Optional.empty(), manager.tryListVolumeNames());
+ }
+
@Test
void ensureSharedVolume_noOwnershipConfig_createsVolumeButNoHelperContainer() {
InspectVolumeCmd cmd = mock(InspectVolumeCmd.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherCodeVolumeReuseTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherCodeVolumeReuseTest.java
new file mode 100644
index 0000000000..30b214ae1e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherCodeVolumeReuseTest.java
@@ -0,0 +1,271 @@
+package io.github.hectorvent.floci.services.lambda.launcher;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
+import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
+import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
+import io.github.hectorvent.floci.services.lambda.LambdaLayerService;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for {@link ContainerLauncher#ensureCodeVolume} reuse across restarts. A code volume is
+ * content-addressed by the function's codeSha256 and survives an emulator restart, but the in-process
+ * "already populated" set does not. A persisted completion marker lets a later run recognize the
+ * populated volume and mount it as-is instead of re-copying its (e.g. ~34k) files.
+ *
+ * <p>Uses a subclass that overrides {@code populateCodeVolume} (rather than a Mockito spy) so the
+ * internal self-call from {@code ensureCodeVolume} is intercepted — a spy would not intercept it.
+ */
+class ContainerLauncherCodeVolumeReuseTest {
+
+ @TempDir
+ Path tempDir;
+
+ private ContainerLifecycleManager lifecycleManager;
+ private RecordingLauncher launcher;
+
+ /** Records populate calls instead of running the real Docker helper-container copy. */
+ private static final class RecordingLauncher extends ContainerLauncher {
+ final List<String> populated = new ArrayList<>();
+ Runnable beforePopulate = () -> {};
+ RuntimeException populateFailure;
+
+ RecordingLauncher(ContainerLifecycleManager lifecycleManager, EmulatorConfig config) {
+ super(mock(ContainerBuilder.class), lifecycleManager, mock(ContainerLogStreamer.class),
+ mock(ImageResolver.class), mock(RuntimeApiServerFactory.class),
+ mock(DockerHostResolver.class), config, mock(EcrRegistryManager.class),
+ mock(LambdaLayerService.class), mock(LaunchedContainerAwsEnv.class));
+ }
+
+ @Override
+ void populateCodeVolume(String volName, LambdaFunction fn, String image) {
+ beforePopulate.run();
+ if (populateFailure != null) {
+ throw populateFailure;
+ }
+ populated.add(volName);
+ }
+ }
+
+ @BeforeEach
+ void setUp() {
+ lifecycleManager = mock(ContainerLifecycleManager.class);
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ EmulatorConfig.StorageConfig storage = mock(EmulatorConfig.StorageConfig.class);
+ when(config.storage()).thenReturn(storage);
+ when(storage.persistentPath()).thenReturn(tempDir.toString());
+ launcher = new RecordingLauncher(lifecycleManager, config);
+ }
+
+ private static LambdaFunction fn(String sha) {
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("orders");
+ fn.setCodeSha256(sha);
+ return fn;
+ }
+
+ private void writeMarker(String vol) throws Exception {
+ Path dir = tempDir.resolve("lambda-codevol-markers");
+ Files.createDirectories(dir);
+ Files.writeString(dir.resolve(vol), "");
+ }
+
+ @Test
+ void reusesExistingVolumeWhenMarkerPresentAndVolumeExists() throws Exception {
+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");
+ String vol = ContainerLauncher.codeVolumeName(fn);
+ writeMarker(vol);
+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.of(Set.of(vol)));
+
+ String result = launcher.ensureCodeVolume(fn, "public.ecr.aws/lambda/nodejs:20");
+
+ assertEquals(vol, result);
+ assertTrue(launcher.populated.isEmpty(),
+ "the ~34k-file populate/copy must be skipped when a previously-populated volume is reused");
+ }
+
+ @Test
+ void populatesAndWritesMarkerWhenMarkerAbsent() throws Exception {
+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");
+ String vol = ContainerLauncher.codeVolumeName(fn);
+ // Volume exists but no completion marker (e.g. a prior populate crashed mid-copy): re-populate.
+
+ String result = launcher.ensureCodeVolume(fn, "img");
+
+ assertEquals(vol, result);
+ assertEquals(List.of(vol), launcher.populated, "populate must run when no completion marker exists");
+ assertTrue(Files.isRegularFile(tempDir.resolve("lambda-codevol-markers").resolve(vol)),
+ "a completion marker must be written after a successful populate so the next boot reuses it");
+ }
+
+ @Test
+ void deletesStaleMarkerBeforeRepopulatingMissingVolume() throws Exception {
+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");
+ String vol = ContainerLauncher.codeVolumeName(fn);
+ Path marker = tempDir.resolve("lambda-codevol-markers").resolve(vol);
+ writeMarker(vol);
+ // The marker lingers but the volume was pruned; reusing it would mount an empty /var/task.
+ when(lifecycleManager.tryListVolumeNames())
+ .thenReturn(Optional.of(Set.of()), Optional.of(Set.of(vol)));
+ launcher.beforePopulate = () -> assertFalse(Files.exists(marker),
+ "the stale completion marker must be deleted before repopulation starts");
+
+ launcher.ensureCodeVolume(fn, "img");
+
+ assertEquals(List.of(vol), launcher.populated,
+ "a lingering marker without the backing volume must not trigger reuse");
+ assertTrue(Files.isRegularFile(marker), "successful repopulation must write a fresh marker");
+ }
+
+ @Test
+ void failedRepopulateCannotLeaveOldCompletionMarker() throws Exception {
+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");
+ String vol = ContainerLauncher.codeVolumeName(fn);
+ Path marker = tempDir.resolve("lambda-codevol-markers").resolve(vol);
+ writeMarker(vol);
+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.of(Set.of()));
+ launcher.populateFailure = new IllegalStateException("copy failed");
+
+ assertThrows(IllegalStateException.class, () -> launcher.ensureCodeVolume(fn, "img"));
+
+ assertFalse(Files.exists(marker),
+ "a failed repopulation must not retain the previous run's completion marker");
+ }
+
+ @Test
+ void volumeInventoryFailureRepopulatesAfterInvalidatingMarker() throws Exception {
+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");
+ String vol = ContainerLauncher.codeVolumeName(fn);
+ Path marker = tempDir.resolve("lambda-codevol-markers").resolve(vol);
+ writeMarker(vol);
+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.empty());
+ launcher.beforePopulate = () -> assertFalse(Files.exists(marker),
+ "the old completion marker must be removed before fallback population starts");
+
+ String result = launcher.ensureCodeVolume(fn, "img");
+
+ assertEquals(vol, result);
+ assertEquals(List.of(vol), launcher.populated,
+ "an unavailable inventory must degrade to the safe population path");
+ assertTrue(Files.isRegularFile(marker),
+ "successful fallback population must write a fresh completion marker");
+ }
+
+ @Test
+ void failedInventoryFallbackCannotLeaveOldCompletionMarker() throws Exception {
+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");
+ String vol = ContainerLauncher.codeVolumeName(fn);
+ Path marker = tempDir.resolve("lambda-codevol-markers").resolve(vol);
+ writeMarker(vol);
+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.empty());
+ launcher.populateFailure = new IllegalStateException("copy failed");
+
+ assertThrows(IllegalStateException.class, () -> launcher.ensureCodeVolume(fn, "img"));
+
+ assertFalse(Files.exists(marker),
+ "a failed fallback population must not retain the previous completion marker");
+ assertTrue(launcher.populated.isEmpty());
+ }
+
+ @Test
+ void successfulPopulationPrunesOnlyOrphanInternalMarkers() throws Exception {
+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");
+ String vol = ContainerLauncher.codeVolumeName(fn);
+ String live = "floci-code-live";
+ String orphan = "floci-code-orphan";
+ writeMarker(live);
+ writeMarker(orphan);
+ writeMarker("unrelated-marker");
+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.of(Set.of(vol, live)));
+
+ launcher.ensureCodeVolume(fn, "img");
+
+ Path markerDir = tempDir.resolve("lambda-codevol-markers");
+ assertTrue(Files.isRegularFile(markerDir.resolve(vol)));
+ assertTrue(Files.isRegularFile(markerDir.resolve(live)));
+ assertFalse(Files.exists(markerDir.resolve(orphan)));
+ assertTrue(Files.isRegularFile(markerDir.resolve("unrelated-marker")),
+ "marker cleanup must not delete files outside Floci's code-volume namespace");
+ }
+
+ @Test
+ void successfulPopulationReplacesSymlinkMarkerWithoutFollowingIt() throws Exception {
+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");
+ String vol = ContainerLauncher.codeVolumeName(fn);
+ Path markerDir = tempDir.resolve("lambda-codevol-markers");
+ Path marker = markerDir.resolve(vol);
+ Path symlinkTarget = tempDir.resolve("outside-marker-target");
+ Files.createDirectories(markerDir);
+ Files.writeString(symlinkTarget, "keep");
+ Files.createSymbolicLink(marker, symlinkTarget);
+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.of(Set.of(vol)));
+
+ launcher.ensureCodeVolume(fn, "img");
+
+ assertEquals("keep", Files.readString(symlinkTarget),
+ "writing a completion marker must not follow and truncate a pre-existing symlink");
+ assertTrue(Files.isRegularFile(marker, java.nio.file.LinkOption.NOFOLLOW_LINKS),
+ "successful population should replace the symlink with a real marker file");
+ assertFalse(Files.isSymbolicLink(marker));
+ }
+
+ @Test
+ void partialTarProducerFailureIsStrictOnlyForCodeVolumePopulation() throws Exception {
+ DockerClient dockerClient = mock(DockerClient.class);
+ CopyArchiveToContainerCmd copyCommand = mock(CopyArchiveToContainerCmd.class);
+ java.io.InputStream[] capturedInput = {null};
+ when(dockerClient.copyArchiveToContainerCmd("container-123")).thenReturn(copyCommand);
+ when(copyCommand.withRemotePath("/var/task")).thenReturn(copyCommand);
+ when(copyCommand.withTarInputStream(org.mockito.ArgumentMatchers.any()))
+ .thenAnswer(invocation -> {
+ capturedInput[0] = invocation.getArgument(0);
+ return copyCommand;
+ });
+ org.mockito.Mockito.doAnswer(invocation -> {
+ capturedInput[0].transferTo(java.io.OutputStream.nullOutputStream());
+ return null;
+ }).when(copyCommand).exec();
+
+ assertDoesNotThrow(() -> ContainerLauncher.copyDirToContainer(
+ dockerClient, "container-123", tempDir, "/var/task", "orders",
+ (sourceDir, out) -> {
+ out.write(new byte[]{1, 2, 3});
+ throw new java.io.IOException("ordinary copy producer closed");
+ }));
+
+ RuntimeException failure = assertThrows(RuntimeException.class,
+ () -> ContainerLauncher.copyDirToContainerStrict(
+ dockerClient, "container-123", tempDir, "/var/task", "orders",
+ (sourceDir, out) -> {
+ out.write(new byte[]{1, 2, 3});
+ throw new java.io.IOException("producer failed after partial archive");
+ }));
+
+ assertTrue(failure.getMessage().contains("Failed to copy directory"), failure.getMessage());
+ assertTrue(failure.getCause().getMessage().contains("Failed to stream tar"), failure.toString());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
index 918b230a65..74852b90bd 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
@@ -104,7 +104,9 @@ class ContainerLauncherTest {
lenient().when(tls.enabled()).thenReturn(false);
lenient().when(config.defaultRegion()).thenReturn("us-east-1");
lenient().when(config.hostname()).thenReturn(Optional.empty());
+ // The large-code path resolves a code-volume completion marker under the storage persistent path.
lenient().when(config.storage()).thenReturn(storage);
+ lenient().when(storage.persistentPath()).thenReturn(tempDir.toString());
lenient().when(storage.efs()).thenReturn(efs);
lenient().when(efs.ownerUid()).thenReturn(OptionalInt.empty());
lenient().when(efs.ownerGid()).thenReturn(OptionalInt.empty());