Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ec2): remove the volume helper container on shutdown (#5551)

Hector Venturaec5692dfa2c4
diff --git a/docs/services/ec2.md b/docs/services/ec2.md
index c1ec1b6e81..2f7fb729a5 100644
--- a/docs/services/ec2.md
+++ b/docs/services/ec2.md
@@ -753,7 +753,7 @@ A standalone ENI created via `CreateNetworkInterface` can also be handed to `Run
| DetachVolume | Detaches a volume from an instance, optionally forced; returns the attachment in `detaching` state. |
When `floci.services.ec2.volume-block-devices` is enabled (the default), attached EBS volumes are backed by real Linux loop devices inside running instance containers and synthesized EKS cluster node containers:
-- **Storage mechanism**: Backing sparse raw image files are allocated in Floci storage under `floci-aws-ec2-volumes` (named volume or host path). A privileged helper container (`floci-aws-ec2-volume-helper`, running `floci.services.ec2.volume-helper-image`, default `alpine:3.21`) manages Linux loop devices via `losetup`.
+- **Storage mechanism**: Backing sparse raw image files are allocated in Floci storage under `floci-aws-ec2-volumes` (named volume or host path). A privileged helper container (`floci-aws-ec2-volume-helper`, running `floci.services.ec2.volume-helper-image`, default `alpine:3.21`) manages Linux loop devices via `losetup`. It starts on the first volume operation and is removed when Floci shuts down; the backing files stay in storage.
- **Target device nodes**: Inside privileged target instance containers, the corresponding device node is created at the requested path (for example `/dev/xvdf` or `/dev/sdf`) using `mknod` or a symlink to the loop device, matching the exact size specified during volume creation. Target containers can format filesystems (such as ext4 or xfs), mount them, and persist data across detach and reattach.
- **Restart reconciliation**: When an instance container stops and starts, or when Floci restarts, attached volume device nodes are automatically restored inside the target container. Backing raw files remain preserved across reboots in persistent storage.
- **Graceful degradation**: If Docker is unavailable, the target instance container is not running, or the container is not privileged, volume attachments degrade gracefully to metadata-only tracking without failing the API call.
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceManager.java
index abce261231..412eb564a1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceManager.java
@@ -4,6 +4,7 @@ import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.exception.NotFoundException;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerExec;
@@ -39,7 +40,7 @@ import java.util.regex.Pattern;
* created inside the target container at the requested path.
*/
@ApplicationScoped
-public class Ec2VolumeBlockDeviceManager implements Resettable {
+public class Ec2VolumeBlockDeviceManager implements ContainerTeardown, Resettable {
private static final Logger LOG = Logger.getLogger(Ec2VolumeBlockDeviceManager.class);
private static final int DEFAULT_TIMEOUT_SECONDS = 30;
@@ -462,6 +463,52 @@ public class Ec2VolumeBlockDeviceManager implements Resettable {
}
}
+ /**
+ * Removes the helper container on shutdown and on a state reset. The helper is looked up by
+ * name and never created here, so one left behind by an earlier process is collected too.
+ * Backing files stay in storage, and the next volume operation recreates the helper.
+ */
+ @Override
+ public synchronized void stopManagedContainers() {
+ if (!isAvailable()) {
+ return;
+ }
+ String helperName = ContainerStorageHelper.resourceName(config, "ec2", null, "volume-helper");
+ InspectContainerResponse helper;
+ try {
+ helper = dockerClient.inspectContainerCmd(helperName).exec();
+ } catch (NotFoundException e) {
+ LOG.debugv("No EC2 volume helper container {0} to remove", helperName);
+ return;
+ } catch (Exception e) {
+ LOG.warnv("Failed to inspect EC2 volume helper container {0}: {1}", helperName, e.getMessage());
+ return;
+ }
+ if (Boolean.TRUE.equals(helper.getState().getRunning())) {
+ // One pass per file: a loop device still held open by an instance container is
+ // detached lazily and stays listed, so retrying it until it disappears would spin.
+ // Every loop is tried, and one that would not detach fails the script so it is reported.
+ String script = SHELL_LOOP_FINDER
+ + "failed=0\n"
+ + "for f in /volumes/*.raw; do\n"
+ + " [ -f \"$f\" ] || continue\n"
+ + " for loop in $(find_loops \"$f\"); do\n"
+ + " losetup -d \"$loop\" 2>/dev/null || { echo \"could not detach $loop from $f\"; failed=1; }\n"
+ + " done\n"
+ + "done\n"
+ + "exit $failed";
+ ContainerExec.Result released = execInContainer(helper.getId(),
+ new String[]{"sh", "-c", script}, DEFAULT_TIMEOUT_SECONDS);
+ if (released.exitCode() != 0) {
+ LOG.warnv("Failed to release loop devices before removing EC2 volume helper container {0}: {1}",
+ helperName, released.summary());
+ }
+ }
+ // The helper only sleeps and would ignore SIGTERM as PID 1, so do not wait on a graceful stop.
+ lifecycleManager.stopAndRemove(helper.getId(), null, 0);
+ activeLoopDevices.clear();
+ }
+
private ContainerExec.Result execInContainer(String containerId, String[] cmd, int timeoutSeconds) {
try {
return ContainerExec.runMerged(dockerClient, containerId, cmd, timeoutSeconds);
@@ -489,6 +536,9 @@ public class Ec2VolumeBlockDeviceManager implements Resettable {
+ " rm -f \"$f\"\n"
+ "done";
execInContainer(helperId, new String[]{"sh", "-c", script}, DEFAULT_TIMEOUT_SECONDS);
+ // The helper was only needed to delete the files, so a reset ends as a shutdown does:
+ // without one.
+ lifecycleManager.stopAndRemove(helperId, null, 0);
}
activeLoopDevices.clear();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceDockerIntegrationTest.java
index 3162a463c0..cb0f69f588 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceDockerIntegrationTest.java
@@ -33,6 +33,7 @@ import java.util.Optional;
import java.util.UUID;
import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -478,6 +479,45 @@ class Ec2VolumeBlockDeviceDockerIntegrationTest {
assertEquals(0, writeBeyond.exitCode(), "Writing beyond original capacity should succeed: " + writeBeyond.summary());
}
+ @Test
+ void stopManagedContainersRemovesHelperAndKeepsBackingFile() {
+ Volume volume = ec2Service.createVolume("us-east-1", "us-east-1a", "gp3", 1, false, 3000, 125, null, null);
+ volumesToClean.add(volume.getVolumeId());
+
+ String helperName = ContainerStorageHelper.resourceName(config, "ec2", null, "volume-helper");
+ assertTrue(lifecycleManager.findByName(helperName).isPresent());
+
+ volumeBlockDeviceManager.stopManagedContainers();
+
+ assertTrue(lifecycleManager.findByName(helperName).isEmpty(),
+ "Helper container should be removed, not just stopped");
+ assertDoesNotThrow(() -> volumeBlockDeviceManager.stopManagedContainers());
+
+ // The next volume operation recreates the helper against the surviving backing file.
+ assertTrue(volumeBlockDeviceManager.resizeVolume(volume.getVolumeId(), 2));
+ ContainerExec.Result checkSize = execIn(helperName, "stat -c %s /volumes/" + volume.getVolumeId() + ".raw");
+ assertEquals(0, checkSize.exitCode());
+ assertEquals("2147483648", checkSize.stdout().trim());
+ }
+
+ @Test
+ void clearDeletesBackingFilesAndLeavesNoHelper() {
+ Volume volume = ec2Service.createVolume("us-east-1", "us-east-1a", "gp3", 1, false, 3000, 125, null, null);
+ volumesToClean.add(volume.getVolumeId());
+ String helperName = ContainerStorageHelper.resourceName(config, "ec2", null, "volume-helper");
+
+ volumeBlockDeviceManager.stopManagedContainers();
+ volumeBlockDeviceManager.clear();
+
+ assertTrue(lifecycleManager.findByName(helperName).isEmpty(),
+ "A reset should end without a helper container");
+ // The next volume operation recreates the helper, which shows the old file is gone.
+ Volume next = ec2Service.createVolume("us-east-1", "us-east-1a", "gp3", 1, false, 3000, 125, null, null);
+ volumesToClean.add(next.getVolumeId());
+ ContainerExec.Result wiped = execIn(helperName, "test -e /volumes/" + volume.getVolumeId() + ".raw");
+ assertEquals(1, wiped.exitCode(), "The reset should have deleted the backing file");
+ }
+
@Test
void modifyVolumeUnattachedGrowsBackingFile() {
Volume volume = ec2Service.createVolume("us-east-1", "us-east-1a", "gp3", 1, false, 3000, 125, null, null);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceManagerTest.java
index 62953ea9de..fc9d5e8150 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2VolumeBlockDeviceManagerTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.ec2;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.github.dockerjava.api.exception.NotFoundException;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerExecStubs;
@@ -15,9 +16,14 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
/**
@@ -29,6 +35,8 @@ class Ec2VolumeBlockDeviceManagerTest {
private static final String HELPER_ID = "helper-container-id";
private DockerClient dockerClient;
+ private InspectContainerResponse helper;
+ private ContainerLifecycleManager lifecycleManager;
private Ec2VolumeBlockDeviceManager manager;
@BeforeEach
@@ -38,13 +46,91 @@ class Ec2VolumeBlockDeviceManagerTest {
when(config.services().ec2().mock()).thenReturn(false);
dockerClient = mock(DockerClient.class, RETURNS_DEEP_STUBS);
- InspectContainerResponse helper = mock(InspectContainerResponse.class, RETURNS_DEEP_STUBS);
+ helper = mock(InspectContainerResponse.class, RETURNS_DEEP_STUBS);
when(helper.getState().getRunning()).thenReturn(true);
when(helper.getId()).thenReturn(HELPER_ID);
when(dockerClient.inspectContainerCmd(anyString()).exec()).thenReturn(helper);
+ lifecycleManager = mock(ContainerLifecycleManager.class);
manager = new Ec2VolumeBlockDeviceManager(dockerClient, mock(ContainerBuilder.class),
- mock(ContainerLifecycleManager.class), config);
+ lifecycleManager, config);
+ }
+
+ @Test
+ void stopManagedContainersReleasesLoopDevicesAndRemovesTheHelper() {
+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(dockerClient, HELPER_ID, 0, "", "");
+
+ manager.stopManagedContainers();
+
+ assertEquals(1, commands.size());
+ String script = commands.getFirst().get(2);
+ assertTrue(script.contains("losetup -d \"$loop\""));
+ assertFalse(script.contains("rm -f"), "backing files must survive a shutdown");
+ verify(lifecycleManager).stopAndRemove(HELPER_ID, null, 0);
+ }
+
+ @Test
+ void stopManagedContainersRemovesAStoppedHelperWithoutRunningAnythingInIt() {
+ when(helper.getState().getRunning()).thenReturn(false);
+
+ manager.stopManagedContainers();
+
+ verify(dockerClient, never()).execCreateCmd(anyString());
+ verify(lifecycleManager).stopAndRemove(HELPER_ID, null, 0);
+ }
+
+ @Test
+ void stopManagedContainersNeverCreatesAHelperThatDoesNotExist() {
+ when(dockerClient.inspectContainerCmd(anyString()).exec()).thenThrow(new NotFoundException("no such container"));
+
+ manager.stopManagedContainers();
+
+ verifyNoInteractions(lifecycleManager);
+ }
+
+ @Test
+ void stopManagedContainersStillRemovesTheHelperWhenReleasingLoopDevicesFails() {
+ when(dockerClient.execCreateCmd(HELPER_ID)).thenThrow(new IllegalStateException("daemon went away"));
+
+ assertDoesNotThrow(() -> manager.stopManagedContainers());
+
+ verify(lifecycleManager).stopAndRemove(HELPER_ID, null, 0);
+ }
+
+ @Test
+ void stopManagedContainersFailsTheReleaseScriptWhenALoopDeviceWillNotDetach() {
+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(dockerClient, HELPER_ID, 1,
+ "could not detach /dev/loop3 from /volumes/vol-1.raw", "");
+
+ assertDoesNotThrow(() -> manager.stopManagedContainers());
+
+ String script = commands.getFirst().get(2);
+ assertTrue(script.contains("|| { echo \"could not detach $loop from $f\"; failed=1; }"));
+ assertTrue(script.endsWith("exit $failed"));
+ verify(lifecycleManager).stopAndRemove(HELPER_ID, null, 0);
+ }
+
+ @Test
+ void clearDeletesTheBackingFilesAndRemovesTheHelper() {
+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(dockerClient, HELPER_ID, 0, "", "");
+
+ manager.clear();
+
+ assertEquals(1, commands.size());
+ assertTrue(commands.getFirst().get(2).contains("rm -f \"$f\""));
+ verify(lifecycleManager).stopAndRemove(HELPER_ID, null, 0);
+ }
+
+ @Test
+ void stopManagedContainersDoesNothingWhenBlockDevicesAreDisabled() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ec2().volumeBlockDevices()).thenReturn(false);
+ Ec2VolumeBlockDeviceManager disabledManager = new Ec2VolumeBlockDeviceManager(dockerClient,
+ mock(ContainerBuilder.class), lifecycleManager, config);
+
+ disabledManager.stopManagedContainers();
+
+ verify(lifecycleManager, never()).stopAndRemove(anyString(), any(), anyInt());
}
@Test