Light, fluffy, and always free - The AWS Local Emulator alternative
perf(lambda): reuse a populated code volume across restarts via a completion marker (#1827)
Alex Banna5d61e9523938
diff --git a/docker/Dockerfile.native-package b/docker/Dockerfile.native-packageindex 6255a716c3..b9a0aba991 100644--- a/docker/Dockerfile.native-package+++ b/docker/Dockerfile.native-package@@ -20,7 +20,8 @@ RUN set -eux; \aarch64) gosuArch='arm64'; gosuSha256="$GOSU_ARM64_SHA256" ;; \*) echo >&2 "unsupported arch: $arch"; exit 1 ;; \esac; \- curl -fsSL -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${gosuArch}"; \+ curl -fsSL --retry 5 --retry-all-errors --retry-delay 2 \+ -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${gosuArch}"; \echo "${gosuSha256} /usr/local/bin/gosu" | sha256sum -c -; \chmod +x /usr/local/bin/gosu; \gosu --version; \diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 7eb9606a55..9ceeb79a0d 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -5,6 +5,8 @@ import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.command.CreateContainerCmd;import com.github.dockerjava.api.command.CreateContainerResponse;import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.command.InspectVolumeResponse;+import com.github.dockerjava.api.command.ListVolumesResponse;import com.github.dockerjava.api.exception.DockerException;import com.github.dockerjava.api.exception.NotFoundException;import com.github.dockerjava.api.model.Bind;@@ -26,6 +28,7 @@ import java.util.List;import java.util.Map;import java.util.Optional;import java.util.OptionalInt;+import java.util.Set;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.TimeUnit;@@ -539,6 +542,34 @@ public class ContainerLifecycleManager {}}+ /**+ * Attempts to take an authoritative snapshot of all named volumes in the container runtime.+ * An empty optional means the runtime could not be queried; it is intentionally distinct from+ * a successful query that returned an empty set so cleanup callers fail closed.+ */+ public Optional<Set<String>> tryListVolumeNames() {+ try {+ ListVolumesResponse response = dockerClient.listVolumesCmd().exec();+ if (response == null) {+ LOG.warn("Container runtime returned no volume-list response");+ return Optional.empty();+ }+ List<InspectVolumeResponse> volumes = response.getVolumes();+ if (volumes == null) {+ LOG.warn("Container runtime returned a volume-list response without an inventory");+ return Optional.empty();+ }+ Set<String> names = volumes.stream()+ .map(InspectVolumeResponse::getName)+ .filter(name -> name != null && !name.isBlank())+ .collect(java.util.stream.Collectors.toUnmodifiableSet());+ return Optional.of(names);+ } catch (RuntimeException e) {+ LOG.warnv("Failed to list container-runtime volumes: {0}", e.getMessage());+ return Optional.empty();+ }+ }+private HostConfig buildHostConfig(ContainerSpec spec) {HostConfig hostConfig = HostConfig.newHostConfig();diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex 9cba571cec..3ad399afd6 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -33,8 +33,10 @@ import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.nio.charset.StandardCharsets;+import java.nio.file.AtomicMoveNotSupportedException;import java.nio.file.Files;import java.nio.file.Path;+import java.nio.file.StandardCopyOption;import java.security.MessageDigest;import java.security.NoSuchAlgorithmException;import java.security.cert.CertificateFactory;@@ -48,6 +50,7 @@ import java.util.Optional;import java.util.concurrent.ExecutionException;import java.util.concurrent.TimeUnit;import java.util.concurrent.TimeoutException;+import java.util.concurrent.atomic.AtomicReference;/*** Starts and stops Docker containers for Lambda function execution.@@ -523,6 +526,8 @@ public class ContainerLauncher {// `docker volume prune` would otherwise still read as "populated" here even though Docker has// nothing under that name, silently mounting an empty /var/task into the next container.// ensureCodeVolume re-checks lifecycleManager.volumeExists() rather than trusting this alone.+ private static final String CODE_VOLUME_MARKER_DIR = "lambda-codevol-markers";+ private static final String CODE_VOLUME_MARKER_PREFIX = "floci-code-";private final java.util.Set<String> populatedCodeVolumes = java.util.concurrent.ConcurrentHashMap.newKeySet();// Deliberately never pruned: removing an entry while a caller elsewhere still held a reference// to its lock object let a third caller's computeIfAbsent create a replacement lock for the same@@ -532,6 +537,7 @@ public class ContainerLauncher {// combinations ever launched, each entry a single Object; negligible next to the disk space the// rest of this fix reclaims.private final java.util.concurrent.ConcurrentHashMap<String, Object> codeVolumeLocks = new java.util.concurrent.ConcurrentHashMap<>();+ private final Object codeVolumeMarkerIoLock = new Object();/** Each function's current code volume, so a redeploy can queue the superseded one for cleanup. */private final java.util.concurrent.ConcurrentHashMap<String, String> functionCurrentVolume =@@ -589,7 +595,7 @@ public class ContainerLauncher {* throwaway helper container that has the volume mounted read-write; every real container then* just mounts the volume read-only, turning a ~95s per-container copy into a ~0.2s mount.*/- private String ensureCodeVolume(LambdaFunction fn, String image) {+ String ensureCodeVolume(LambdaFunction fn, String image) {String volName = codeVolumeName(fn);// Held for the whole resolve-and-reconcile, not just the populate branch: this is the same// lock cleanupSupersededVolumes acquires before claiming a volume for deletion, so a launch@@ -605,15 +611,22 @@ public class ContainerLauncher {Object lock = codeVolumeLocks.computeIfAbsent(volName, k -> new Object());synchronized (lock) {if (!(populatedCodeVolumes.contains(volName) && lifecycleManager.volumeExists(volName))) {- // Either never populated, or the bookkeeping says populated but Docker disagrees- // (removed out of band). Repopulate rather than trust a stale flag.- long t0 = System.currentTimeMillis();- LOG.infov("Populating code volume {0} for function {1} (one-time per code version)",- volName, fn.getFunctionName());- populateCodeVolume(volName, fn, image);+ // Either never populated in this process, or the bookkeeping says populated but+ // Docker disagrees (removed out of band). A completion marker from an earlier run+ // is authoritative only while its independently persisted volume still exists.+ if (reusePersistedCodeVolumeOrRemoveStaleMarker(volName)) {+ LOG.infov("Reusing code volume {0} populated by a previous run (marker present)",+ volName);+ } else {+ long t0 = System.currentTimeMillis();+ LOG.infov("Populating code volume {0} for function {1} (one-time per code version)",+ volName, fn.getFunctionName());+ populateCodeVolume(volName, fn, image);+ writeCodeVolumeMarkerAndPruneOrphans(volName);+ LOG.infov("Populated code volume {0} in {1}ms; future cold starts mount it instead of copying",+ volName, System.currentTimeMillis() - t0);+ }populatedCodeVolumes.add(volName);- LOG.infov("Populated code volume {0} in {1}ms; future cold starts mount it instead of copying",- volName, System.currentTimeMillis() - t0);}// Reconcile bookkeeping for the resolved volume even when the check above already found// it populated: a redeploy rolled back to an older code version resolves a volume name@@ -730,7 +743,7 @@ public class ContainerLauncher {return count == null ? 0 : count.get();}- private void populateCodeVolume(String volName, LambdaFunction fn, String image) {+ void populateCodeVolume(String volName, LambdaFunction fn, String image) {lifecycleManager.ensureVolume(volName);String shortId = java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8);// A minimal helper container (sleep) with the volume mounted read-write at /var/task; we@@ -749,16 +762,113 @@ public class ContainerLauncher {try {helperId = lifecycleManager.create(helperSpec);lifecycleManager.startCreated(helperId, helperSpec);- copyDirToContainer(lifecycleManager.getDockerClient(), helperId,+ copyDirToContainerStrict(lifecycleManager.getDockerClient(), helperId,Path.of(fn.getCodeLocalPath()), TASK_DIR, fn.getFunctionName());} finally {if (helperId != null) {- try { lifecycleManager.stopAndRemove(helperId, null); } catch (Exception ignore) { /* best effort */ }+ try {+ lifecycleManager.stopAndRemove(helperId, null);+ } catch (Exception e) {+ LOG.warnv("Could not remove code-volume helper {0}: {1}", helperId, e.getMessage());+ }}POPULATE_SEMAPHORE.release();}}+ // A code volume survives an emulator restart, but the in-process populatedCodeVolumes set does not.+ // A persisted completion marker lets a later run recognize an already-populated, content-addressed+ // volume and skip re-copying its files. The marker is stored under storage.persistent-path+ // regardless of storage mode; reuse still requires the independently persisted Docker volume.+ private Path codeVolumeMarkerPath(String volName) {+ return Path.of(config.storage().persistentPath(), CODE_VOLUME_MARKER_DIR, volName);+ }++ /**+ * Reuses a completed volume when both marker and volume exist. If the volume is absent or cannot+ * be verified, removes the marker before repopulation so a failed copy cannot leave the old+ * completion signal beside partial data.+ */+ private boolean reusePersistedCodeVolumeOrRemoveStaleMarker(String volName) {+ synchronized (codeVolumeMarkerIoLock) {+ Path marker = codeVolumeMarkerPath(volName);+ if (!Files.isRegularFile(marker, java.nio.file.LinkOption.NOFOLLOW_LINKS)) {+ return false;+ }+ java.util.Optional<java.util.Set<String>> volumeNames = lifecycleManager.tryListVolumeNames();+ boolean inventoryUnavailable = volumeNames.isEmpty();+ if (!inventoryUnavailable && volumeNames.get().contains(volName)) {+ return true;+ }+ try {+ Files.deleteIfExists(marker);+ if (inventoryUnavailable) {+ LOG.warnv("Could not verify backing volume {0}; removed its marker and re-populating",+ volName);+ } else {+ LOG.debugv("Removed stale code-volume marker {0}; backing volume is absent", marker);+ }+ } catch (IOException e) {+ throw new IllegalStateException("Could not invalidate code-volume marker " + marker, e);+ }+ return false;+ }+ }++ private void writeCodeVolumeMarkerAndPruneOrphans(String volName) {+ synchronized (codeVolumeMarkerIoLock) {+ Path marker = codeVolumeMarkerPath(volName);+ try {+ Path markerDir = marker.getParent();+ Files.createDirectories(markerDir);+ Path tempMarker = Files.createTempFile(markerDir, marker.getFileName().toString(), ".tmp");+ try {+ Files.writeString(tempMarker, "");+ try {+ Files.move(tempMarker, marker, StandardCopyOption.REPLACE_EXISTING,+ StandardCopyOption.ATOMIC_MOVE);+ } catch (AtomicMoveNotSupportedException e) {+ Files.move(tempMarker, marker, StandardCopyOption.REPLACE_EXISTING);+ }+ } catch (IOException | RuntimeException e) {+ try {+ Files.deleteIfExists(tempMarker);+ } catch (IOException cleanupError) {+ e.addSuppressed(cleanupError);+ }+ throw e;+ }+ } catch (IOException e) {+ // A missing marker only costs one re-populate on the next boot; never fail a launch over it.+ LOG.warnv("Could not write code-volume marker {0}: {1}", marker, e.getMessage());+ }+ pruneOrphanCodeVolumeMarkers(marker.getParent());+ }+ }++ private void pruneOrphanCodeVolumeMarkers(Path markerDir) {+ java.util.Optional<java.util.Set<String>> volumeNames = lifecycleManager.tryListVolumeNames();+ if (volumeNames.isEmpty() || !Files.isDirectory(markerDir)) {+ return;+ }+ try (java.util.stream.Stream<Path> markers = Files.list(markerDir)) {+ markers.filter(path -> Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS))+ .filter(path -> path.getFileName().toString().startsWith(CODE_VOLUME_MARKER_PREFIX))+ .filter(path -> !volumeNames.get().contains(path.getFileName().toString()))+ .forEach(path -> {+ try {+ Files.deleteIfExists(path);+ LOG.debugv("Pruned orphan code-volume marker {0}", path);+ } catch (IOException e) {+ LOG.warnv("Could not prune orphan code-volume marker {0}: {1}",+ path, e.getMessage());+ }+ });+ } catch (IOException e) {+ LOG.warnv("Could not scan code-volume marker directory {0}: {1}", markerDir, e.getMessage());+ }+ }+/*** Total code size (bytes) at or above which a function's {@code /var/task} is served from a* shared read-only volume rather than copied into every container. This is roughly the point@@ -843,26 +953,68 @@ public class ContainerLauncher {*/private static final int TAR_PIPE_BUFFER_BYTES = 16 * 1024 * 1024;+ @FunctionalInterface+ interface DirectoryTarWriter {+ void write(Path sourceDir, OutputStream out) throws IOException;+ }+private void copyDirToContainer(DockerClient dockerClient, String containerId,Path sourceDir, String remotePath, String functionName) {+ copyDirToContainer(dockerClient, containerId, sourceDir, remotePath, functionName,+ ContainerLauncher::createTarFromDir);+ }++ static void copyDirToContainer(DockerClient dockerClient, String containerId,+ Path sourceDir, String remotePath, String functionName,+ DirectoryTarWriter tarWriter) {+ copyDirToContainer(dockerClient, containerId, sourceDir, remotePath, functionName,+ tarWriter, false);+ }++ private void copyDirToContainerStrict(DockerClient dockerClient, String containerId,+ Path sourceDir, String remotePath, String functionName) {+ copyDirToContainerStrict(dockerClient, containerId, sourceDir, remotePath, functionName,+ ContainerLauncher::createTarFromDir);+ }++ static void copyDirToContainerStrict(DockerClient dockerClient, String containerId,+ Path sourceDir, String remotePath, String functionName,+ DirectoryTarWriter tarWriter) {+ copyDirToContainer(dockerClient, containerId, sourceDir, remotePath, functionName,+ tarWriter, true);+ }++ private static void copyDirToContainer(DockerClient dockerClient, String containerId,+ Path sourceDir, String remotePath, String functionName,+ DirectoryTarWriter tarWriter, boolean failOnTarFailure) {// No per-copy gating here: the heavy /var/task populate for large code already holds a// POPULATE_SEMAPHORE permit; small-code direct copies and layer copies are light enough// to run unthrottled.try (java.io.PipedOutputStream pos = new java.io.PipedOutputStream();java.io.PipedInputStream pis = new java.io.PipedInputStream(pos, TAR_PIPE_BUFFER_BYTES)) {- new Thread(() -> {+ AtomicReference<IOException> tarFailure = new AtomicReference<>();+ Thread tarStreamer = new Thread(() -> {try (pos) {- createTarFromDir(sourceDir, pos);+ tarWriter.write(sourceDir, pos);} catch (IOException e) {- LOG.errorv("Failed to stream tar for function {0}: {1}", functionName, e.getMessage());+ if (failOnTarFailure) {+ tarFailure.set(e);+ } else {+ LOG.errorv("Failed to stream directory tar for function {0}: {1}",+ functionName, e.getMessage());+ }}- }, "tar-streamer-dir-" + functionName).start();+ }, "tar-streamer-dir-" + functionName);+ tarStreamer.start();dockerClient.copyArchiveToContainerCmd(containerId).withRemotePath(remotePath).withTarInputStream(pis).exec();+ if (failOnTarFailure) {+ waitForTarStreamer(tarStreamer, tarFailure, functionName, sourceDir);+ }LOG.debugv("Copied directory {0} into container {1} at {2}", sourceDir, containerId, remotePath);} catch (Exception e) {// Fail loudly so launch() cleans up the half-built container instead of leaking it.@@ -902,6 +1054,22 @@ public class ContainerLauncher {}}+ private static void waitForTarStreamer(Thread tarStreamer, AtomicReference<IOException> tarFailure,+ String functionName, Path sourcePath) throws IOException {+ try {+ tarStreamer.join();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IOException("Interrupted while streaming tar for function " + functionName+ + " from " + sourcePath, e);+ }+ IOException failure = tarFailure.get();+ if (failure != null) {+ throw new IOException("Failed to stream tar for function " + functionName+ + " from " + sourcePath, failure);+ }+ }+private static boolean isProvidedRuntime(String runtime) {return runtime != null && runtime.startsWith("provided");}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.javaindex e7f757d627..c4811d1452 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java@@ -5,6 +5,8 @@ import com.github.dockerjava.api.command.CreateContainerCmd;import com.github.dockerjava.api.command.CreateContainerResponse;import com.github.dockerjava.api.command.InspectVolumeCmd;import com.github.dockerjava.api.command.InspectVolumeResponse;+import com.github.dockerjava.api.command.ListVolumesCmd;+import com.github.dockerjava.api.command.ListVolumesResponse;import com.github.dockerjava.api.command.RemoveContainerCmd;import com.github.dockerjava.api.command.StartContainerCmd;import com.github.dockerjava.api.command.StopContainerCmd;@@ -23,6 +25,7 @@ import java.util.ArrayList;import java.util.List;import java.util.Optional;import java.util.OptionalInt;+import java.util.Set;import static org.junit.jupiter.api.Assertions.*;import static org.mockito.Mockito.*;@@ -155,6 +158,62 @@ class ContainerLifecycleManagerVolumeTest {assertFalse(manager.volumeExists("some-volume"));}+ @Test+ void tryListVolumeNamesReturnsAuthoritativeSnapshot() {+ ListVolumesCmd cmd = mock(ListVolumesCmd.class);+ ListVolumesResponse response = mock(ListVolumesResponse.class);+ InspectVolumeResponse first = mock(InspectVolumeResponse.class);+ InspectVolumeResponse second = mock(InspectVolumeResponse.class);+ when(dockerClient.listVolumesCmd()).thenReturn(cmd);+ when(cmd.exec()).thenReturn(response);+ when(response.getVolumes()).thenReturn(java.util.List.of(first, second));+ when(first.getName()).thenReturn("floci-code-first");+ when(second.getName()).thenReturn("shared-data");++ assertEquals(Optional.of(Set.of("floci-code-first", "shared-data")),+ manager.tryListVolumeNames());+ }++ @Test+ void tryListVolumeNamesDistinguishesEmptyInventory() {+ ListVolumesCmd cmd = mock(ListVolumesCmd.class);+ ListVolumesResponse response = mock(ListVolumesResponse.class);+ when(dockerClient.listVolumesCmd()).thenReturn(cmd);+ when(cmd.exec()).thenReturn(response);+ when(response.getVolumes()).thenReturn(java.util.List.of());++ assertEquals(Optional.of(Set.of()), manager.tryListVolumeNames());+ }++ @Test+ void tryListVolumeNamesReturnsEmptyOptionalForNullResponse() {+ ListVolumesCmd cmd = mock(ListVolumesCmd.class);+ when(dockerClient.listVolumesCmd()).thenReturn(cmd);+ when(cmd.exec()).thenReturn(null);++ assertEquals(Optional.empty(), manager.tryListVolumeNames());+ }++ @Test+ void tryListVolumeNamesReturnsEmptyOptionalForNullVolumeList() {+ ListVolumesCmd cmd = mock(ListVolumesCmd.class);+ ListVolumesResponse response = mock(ListVolumesResponse.class);+ when(dockerClient.listVolumesCmd()).thenReturn(cmd);+ when(cmd.exec()).thenReturn(response);+ when(response.getVolumes()).thenReturn(null);++ assertEquals(Optional.empty(), manager.tryListVolumeNames());+ }++ @Test+ void tryListVolumeNamesReturnsEmptyOptionalOnDockerFailure() {+ ListVolumesCmd cmd = mock(ListVolumesCmd.class);+ when(dockerClient.listVolumesCmd()).thenReturn(cmd);+ when(cmd.exec()).thenThrow(new DockerException("Connection refused", 500));++ assertEquals(Optional.empty(), manager.tryListVolumeNames());+ }+@Testvoid ensureSharedVolume_noOwnershipConfig_createsVolumeButNoHelperContainer() {InspectVolumeCmd cmd = mock(InspectVolumeCmd.class);diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherCodeVolumeReuseTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherCodeVolumeReuseTest.javanew file mode 100644index 0000000000..30b214ae1e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherCodeVolumeReuseTest.java@@ -0,0 +1,271 @@+package io.github.hectorvent.floci.services.lambda.launcher;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;+import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;+import io.github.hectorvent.floci.services.lambda.LambdaLayerService;+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;+import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.nio.file.Files;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;+import java.util.Optional;+import java.util.Set;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * Unit tests for {@link ContainerLauncher#ensureCodeVolume} reuse across restarts. A code volume is+ * content-addressed by the function's codeSha256 and survives an emulator restart, but the in-process+ * "already populated" set does not. A persisted completion marker lets a later run recognize the+ * populated volume and mount it as-is instead of re-copying its (e.g. ~34k) files.+ *+ * <p>Uses a subclass that overrides {@code populateCodeVolume} (rather than a Mockito spy) so the+ * internal self-call from {@code ensureCodeVolume} is intercepted — a spy would not intercept it.+ */+class ContainerLauncherCodeVolumeReuseTest {++ @TempDir+ Path tempDir;++ private ContainerLifecycleManager lifecycleManager;+ private RecordingLauncher launcher;++ /** Records populate calls instead of running the real Docker helper-container copy. */+ private static final class RecordingLauncher extends ContainerLauncher {+ final List<String> populated = new ArrayList<>();+ Runnable beforePopulate = () -> {};+ RuntimeException populateFailure;++ RecordingLauncher(ContainerLifecycleManager lifecycleManager, EmulatorConfig config) {+ super(mock(ContainerBuilder.class), lifecycleManager, mock(ContainerLogStreamer.class),+ mock(ImageResolver.class), mock(RuntimeApiServerFactory.class),+ mock(DockerHostResolver.class), config, mock(EcrRegistryManager.class),+ mock(LambdaLayerService.class), mock(LaunchedContainerAwsEnv.class));+ }++ @Override+ void populateCodeVolume(String volName, LambdaFunction fn, String image) {+ beforePopulate.run();+ if (populateFailure != null) {+ throw populateFailure;+ }+ populated.add(volName);+ }+ }++ @BeforeEach+ void setUp() {+ lifecycleManager = mock(ContainerLifecycleManager.class);+ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.StorageConfig storage = mock(EmulatorConfig.StorageConfig.class);+ when(config.storage()).thenReturn(storage);+ when(storage.persistentPath()).thenReturn(tempDir.toString());+ launcher = new RecordingLauncher(lifecycleManager, config);+ }++ private static LambdaFunction fn(String sha) {+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("orders");+ fn.setCodeSha256(sha);+ return fn;+ }++ private void writeMarker(String vol) throws Exception {+ Path dir = tempDir.resolve("lambda-codevol-markers");+ Files.createDirectories(dir);+ Files.writeString(dir.resolve(vol), "");+ }++ @Test+ void reusesExistingVolumeWhenMarkerPresentAndVolumeExists() throws Exception {+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");+ String vol = ContainerLauncher.codeVolumeName(fn);+ writeMarker(vol);+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.of(Set.of(vol)));++ String result = launcher.ensureCodeVolume(fn, "public.ecr.aws/lambda/nodejs:20");++ assertEquals(vol, result);+ assertTrue(launcher.populated.isEmpty(),+ "the ~34k-file populate/copy must be skipped when a previously-populated volume is reused");+ }++ @Test+ void populatesAndWritesMarkerWhenMarkerAbsent() throws Exception {+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");+ String vol = ContainerLauncher.codeVolumeName(fn);+ // Volume exists but no completion marker (e.g. a prior populate crashed mid-copy): re-populate.++ String result = launcher.ensureCodeVolume(fn, "img");++ assertEquals(vol, result);+ assertEquals(List.of(vol), launcher.populated, "populate must run when no completion marker exists");+ assertTrue(Files.isRegularFile(tempDir.resolve("lambda-codevol-markers").resolve(vol)),+ "a completion marker must be written after a successful populate so the next boot reuses it");+ }++ @Test+ void deletesStaleMarkerBeforeRepopulatingMissingVolume() throws Exception {+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");+ String vol = ContainerLauncher.codeVolumeName(fn);+ Path marker = tempDir.resolve("lambda-codevol-markers").resolve(vol);+ writeMarker(vol);+ // The marker lingers but the volume was pruned; reusing it would mount an empty /var/task.+ when(lifecycleManager.tryListVolumeNames())+ .thenReturn(Optional.of(Set.of()), Optional.of(Set.of(vol)));+ launcher.beforePopulate = () -> assertFalse(Files.exists(marker),+ "the stale completion marker must be deleted before repopulation starts");++ launcher.ensureCodeVolume(fn, "img");++ assertEquals(List.of(vol), launcher.populated,+ "a lingering marker without the backing volume must not trigger reuse");+ assertTrue(Files.isRegularFile(marker), "successful repopulation must write a fresh marker");+ }++ @Test+ void failedRepopulateCannotLeaveOldCompletionMarker() throws Exception {+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");+ String vol = ContainerLauncher.codeVolumeName(fn);+ Path marker = tempDir.resolve("lambda-codevol-markers").resolve(vol);+ writeMarker(vol);+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.of(Set.of()));+ launcher.populateFailure = new IllegalStateException("copy failed");++ assertThrows(IllegalStateException.class, () -> launcher.ensureCodeVolume(fn, "img"));++ assertFalse(Files.exists(marker),+ "a failed repopulation must not retain the previous run's completion marker");+ }++ @Test+ void volumeInventoryFailureRepopulatesAfterInvalidatingMarker() throws Exception {+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");+ String vol = ContainerLauncher.codeVolumeName(fn);+ Path marker = tempDir.resolve("lambda-codevol-markers").resolve(vol);+ writeMarker(vol);+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.empty());+ launcher.beforePopulate = () -> assertFalse(Files.exists(marker),+ "the old completion marker must be removed before fallback population starts");++ String result = launcher.ensureCodeVolume(fn, "img");++ assertEquals(vol, result);+ assertEquals(List.of(vol), launcher.populated,+ "an unavailable inventory must degrade to the safe population path");+ assertTrue(Files.isRegularFile(marker),+ "successful fallback population must write a fresh completion marker");+ }++ @Test+ void failedInventoryFallbackCannotLeaveOldCompletionMarker() throws Exception {+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");+ String vol = ContainerLauncher.codeVolumeName(fn);+ Path marker = tempDir.resolve("lambda-codevol-markers").resolve(vol);+ writeMarker(vol);+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.empty());+ launcher.populateFailure = new IllegalStateException("copy failed");++ assertThrows(IllegalStateException.class, () -> launcher.ensureCodeVolume(fn, "img"));++ assertFalse(Files.exists(marker),+ "a failed fallback population must not retain the previous completion marker");+ assertTrue(launcher.populated.isEmpty());+ }++ @Test+ void successfulPopulationPrunesOnlyOrphanInternalMarkers() throws Exception {+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");+ String vol = ContainerLauncher.codeVolumeName(fn);+ String live = "floci-code-live";+ String orphan = "floci-code-orphan";+ writeMarker(live);+ writeMarker(orphan);+ writeMarker("unrelated-marker");+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.of(Set.of(vol, live)));++ launcher.ensureCodeVolume(fn, "img");++ Path markerDir = tempDir.resolve("lambda-codevol-markers");+ assertTrue(Files.isRegularFile(markerDir.resolve(vol)));+ assertTrue(Files.isRegularFile(markerDir.resolve(live)));+ assertFalse(Files.exists(markerDir.resolve(orphan)));+ assertTrue(Files.isRegularFile(markerDir.resolve("unrelated-marker")),+ "marker cleanup must not delete files outside Floci's code-volume namespace");+ }++ @Test+ void successfulPopulationReplacesSymlinkMarkerWithoutFollowingIt() throws Exception {+ LambdaFunction fn = fn("sha-v1-abcdef0123456789");+ String vol = ContainerLauncher.codeVolumeName(fn);+ Path markerDir = tempDir.resolve("lambda-codevol-markers");+ Path marker = markerDir.resolve(vol);+ Path symlinkTarget = tempDir.resolve("outside-marker-target");+ Files.createDirectories(markerDir);+ Files.writeString(symlinkTarget, "keep");+ Files.createSymbolicLink(marker, symlinkTarget);+ when(lifecycleManager.tryListVolumeNames()).thenReturn(Optional.of(Set.of(vol)));++ launcher.ensureCodeVolume(fn, "img");++ assertEquals("keep", Files.readString(symlinkTarget),+ "writing a completion marker must not follow and truncate a pre-existing symlink");+ assertTrue(Files.isRegularFile(marker, java.nio.file.LinkOption.NOFOLLOW_LINKS),+ "successful population should replace the symlink with a real marker file");+ assertFalse(Files.isSymbolicLink(marker));+ }++ @Test+ void partialTarProducerFailureIsStrictOnlyForCodeVolumePopulation() throws Exception {+ DockerClient dockerClient = mock(DockerClient.class);+ CopyArchiveToContainerCmd copyCommand = mock(CopyArchiveToContainerCmd.class);+ java.io.InputStream[] capturedInput = {null};+ when(dockerClient.copyArchiveToContainerCmd("container-123")).thenReturn(copyCommand);+ when(copyCommand.withRemotePath("/var/task")).thenReturn(copyCommand);+ when(copyCommand.withTarInputStream(org.mockito.ArgumentMatchers.any()))+ .thenAnswer(invocation -> {+ capturedInput[0] = invocation.getArgument(0);+ return copyCommand;+ });+ org.mockito.Mockito.doAnswer(invocation -> {+ capturedInput[0].transferTo(java.io.OutputStream.nullOutputStream());+ return null;+ }).when(copyCommand).exec();++ assertDoesNotThrow(() -> ContainerLauncher.copyDirToContainer(+ dockerClient, "container-123", tempDir, "/var/task", "orders",+ (sourceDir, out) -> {+ out.write(new byte[]{1, 2, 3});+ throw new java.io.IOException("ordinary copy producer closed");+ }));++ RuntimeException failure = assertThrows(RuntimeException.class,+ () -> ContainerLauncher.copyDirToContainerStrict(+ dockerClient, "container-123", tempDir, "/var/task", "orders",+ (sourceDir, out) -> {+ out.write(new byte[]{1, 2, 3});+ throw new java.io.IOException("producer failed after partial archive");+ }));++ assertTrue(failure.getMessage().contains("Failed to copy directory"), failure.getMessage());+ assertTrue(failure.getCause().getMessage().contains("Failed to stream tar"), failure.toString());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.javaindex 918b230a65..74852b90bd 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java@@ -104,7 +104,9 @@ class ContainerLauncherTest {lenient().when(tls.enabled()).thenReturn(false);lenient().when(config.defaultRegion()).thenReturn("us-east-1");lenient().when(config.hostname()).thenReturn(Optional.empty());+ // The large-code path resolves a code-volume completion marker under the storage persistent path.lenient().when(config.storage()).thenReturn(storage);+ lenient().when(storage.persistentPath()).thenReturn(tempDir.toString());lenient().when(storage.efs()).thenReturn(efs);lenient().when(efs.ownerUid()).thenReturn(OptionalInt.empty());lenient().when(efs.ownerGid()).thenReturn(OptionalInt.empty());