Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): reclaim anonymous task volumes (#6793)

Andre Kurait9fcf1683eeac
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index bf55b508cb..57b5858d0e 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -591,10 +591,18 @@ public class ContainerLifecycleManager {
}
public void stopAndRemoveStrict(String containerId, Closeable logStream, int stopTimeoutSeconds) {
- withTeardownSlot(() -> stopAndRemoveStrictAdmitted(containerId, logStream, stopTimeoutSeconds));
+ stopAndRemoveStrict(containerId, logStream, stopTimeoutSeconds, false);
}
- private void stopAndRemoveStrictAdmitted(String containerId, Closeable logStream, int stopTimeoutSeconds) {
+ /** Strict teardown that can also reclaim the container's anonymous volumes. */
+ public void stopAndRemoveStrict(String containerId, Closeable logStream, int stopTimeoutSeconds,
+ boolean removeAnonymousVolumes) {
+ withTeardownSlot(() -> stopAndRemoveStrictAdmitted(
+ containerId, logStream, stopTimeoutSeconds, removeAnonymousVolumes));
+ }
+
+ private void stopAndRemoveStrictAdmitted(String containerId, Closeable logStream, int stopTimeoutSeconds,
+ boolean removeAnonymousVolumes) {
LOG.infov("Stopping container {0}", containerId);
Exception stopFailure = null;
@@ -613,7 +621,11 @@ public class ContainerLifecycleManager {
}
try {
- dockerClient.removeContainerCmd(containerId).withForce(true).exec();
+ RemoveContainerCmd remove = dockerClient.removeContainerCmd(containerId).withForce(true);
+ if (removeAnonymousVolumes) {
+ remove.withRemoveVolumes(true);
+ }
+ remove.exec();
LOG.debugv("Removed container {0}", containerId);
} catch (NotFoundException e) {
// Already gone, so cleanup succeeded.
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
index dc2921db54..ecc94af0c7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
@@ -5,6 +5,7 @@ import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.command.InspectVolumeResponse;
import com.github.dockerjava.api.command.ListVolumesResponse;
+import com.github.dockerjava.api.command.RemoveContainerCmd;
import com.github.dockerjava.api.command.StatsCmd;
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.ContainerNetwork;
@@ -567,7 +568,7 @@ public class EcsContainerManager {
}
fluentHost = inspectContainerIp(dockerId);
} else {
- ContainerInfo info = lifecycleManager.createAndStart(spec);
+ ContainerInfo info = lifecycleManager.createAndStartEphemeral(spec);
dockerId = info.containerId();
}
@@ -1241,7 +1242,7 @@ public class EcsContainerManager {
continue;
}
try {
- lifecycleManager.removeIfExistsStrict(container.getId());
+ lifecycleManager.removeIfExistsStrict(container.getId(), true);
leftoverRemovalFailuresReported.remove(container.getId());
LOG.infov("Removed ECS container {0} ({1}) left by a previous run", container.getId(),
container.getNames() == null ? "" : String.join(",", container.getNames()));
@@ -1466,7 +1467,7 @@ public class EcsContainerManager {
*/
private boolean stopAndConfirmRemoved(String dockerId) {
try {
- lifecycleManager.stopAndRemoveStrict(dockerId, null);
+ lifecycleManager.stopAndRemoveStrict(dockerId, null, 5, true);
return true;
} catch (RuntimeException e) {
LOG.warnv("Error removing ECS container {0}: {1}", dockerId, e.getMessage());
@@ -1522,7 +1523,10 @@ public class EcsContainerManager {
// Read before the removal below, which is the last moment the daemon still knows it.
handle.recordFinishedAt(name, getFinishedAtIfStopped(dockerId));
try {
- lifecycleManager.getDockerClient().removeContainerCmd(dockerId).withForce(true).exec();
+ RemoveContainerCmd remove = lifecycleManager.getDockerClient()
+ .removeContainerCmd(dockerId).withForce(true);
+ remove.withRemoveVolumes(true);
+ remove.exec();
terminatedContainerIds.add(dockerId);
handle.recordContainerRemoved(name);
} catch (NotFoundException ignored) {
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerTeardownTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerTeardownTest.java
index 1db50c6f28..91585f2af5 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerTeardownTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerTeardownTest.java
@@ -6,6 +6,8 @@ import com.github.dockerjava.api.command.StopContainerCmd;
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.exception.NotModifiedException;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.Closeable;
import java.util.ArrayList;
@@ -26,6 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -34,6 +37,35 @@ import static org.mockito.Mockito.when;
class ContainerLifecycleManagerTeardownTest {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void strictRemovalRetainsLogStreamForRetryWithOptionalAnonymousVolumeCleanup(boolean removeAnonymousVolumes)
+ throws Exception {
+ DockerClient docker = mock(DockerClient.class);
+ ContainerLifecycleManager manager = new ContainerLifecycleManager(docker, null, null, null);
+ StopContainerCmd stop = mock(StopContainerCmd.class);
+ when(docker.stopContainerCmd("retry")).thenReturn(stop);
+ when(stop.withTimeout(5)).thenReturn(stop);
+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);
+ when(docker.removeContainerCmd("retry")).thenReturn(remove);
+ when(remove.withForce(true)).thenReturn(remove);
+ doThrow(new IllegalStateException("daemon refused removal")).doNothing().when(remove).exec();
+ Closeable logStream = mock(Closeable.class);
+ Runnable teardown = removeAnonymousVolumes
+ ? () -> manager.stopAndRemoveStrict("retry", logStream, 5, true)
+ : () -> manager.stopAndRemoveStrict("retry", logStream);
+
+ assertThrows(IllegalStateException.class, teardown::run);
+ verify(logStream, never()).close();
+ if (removeAnonymousVolumes) {
+ verify(remove).withRemoveVolumes(true);
+ } else {
+ verify(remove, never()).withRemoveVolumes(true);
+ }
+ teardown.run();
+ verify(logStream).close();
+ }
+
@Test
void interruptedWaitersStillCleanUpWithoutReleasingAnotherOwnersSlot() throws Exception {
DockerClient docker = mock(DockerClient.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerAwsBaselineTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerAwsBaselineTest.java
index e1901ced28..5221abd0ec 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerAwsBaselineTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerAwsBaselineTest.java
@@ -66,7 +66,7 @@ class EcsContainerManagerAwsBaselineTest {
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any()))
+ when(lifecycleManager.createAndStartEphemeral(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEcrRewriteTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEcrRewriteTest.java
index 574fceed73..6ae99f2801 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEcrRewriteTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEcrRewriteTest.java
@@ -68,7 +68,7 @@ class EcsContainerManagerEcrRewriteTest {
lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any()))
+ when(lifecycleManager.createAndStartEphemeral(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);
@@ -145,6 +145,6 @@ class EcsContainerManagerEcrRewriteTest {
() -> manager.startTask(task, taskDef, List.of(), "us-east-1"));
// The earlier "sidecar" container must never have been created.
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
index ac4d0751c4..4bfbdd3957 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
@@ -100,7 +100,7 @@ class EcsContainerManagerFirelensTest {
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.removeVolume(anyString())).thenReturn(true);
- when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("app-id", Map.of()));
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(new ContainerInfo("app-id", Map.of()));
when(lifecycleManager.create(any())).thenReturn("router-id");
when(lifecycleManager.startCreated(anyString(), any())).thenReturn(new ContainerInfo("router-id", Map.of()));
@@ -169,7 +169,7 @@ class EcsContainerManagerFirelensTest {
InOrder order = inOrder(lifecycleManager);
order.verify(lifecycleManager).create(any(ContainerSpec.class));
order.verify(lifecycleManager).startCreated(anyString(), any());
- order.verify(lifecycleManager).createAndStart(any(ContainerSpec.class));
+ order.verify(lifecycleManager).createAndStartEphemeral(any(ContainerSpec.class));
ArgumentCaptor<LogConfig> logConfig = ArgumentCaptor.forClass(LogConfig.class);
verify(builder).withLogConfig(logConfig.capture());
@@ -222,7 +222,7 @@ class EcsContainerManagerFirelensTest {
InOrder order = inOrder(lifecycleManager);
order.verify(lifecycleManager).create(any(ContainerSpec.class));
order.verify(lifecycleManager).startCreated(anyString(), any());
- order.verify(lifecycleManager).createAndStart(any(ContainerSpec.class));
+ order.verify(lifecycleManager).createAndStartEphemeral(any(ContainerSpec.class));
ArgumentCaptor<LogConfig> logConfig = ArgumentCaptor.forClass(LogConfig.class);
verify(builder).withLogConfig(logConfig.capture());
@@ -349,7 +349,7 @@ class EcsContainerManagerFirelensTest {
+ "missing.conf from bucket firelens-configs: The specified key does not exist.",
failure.getMessage());
verify(lifecycleManager, never()).create(any(ContainerSpec.class));
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
@Test
@@ -365,7 +365,7 @@ class EcsContainerManagerFirelensTest {
assertEquals("ClientException", failure.getErrorCode());
assertEquals("awsfirelens log driver requires a firelensConfiguration container",
failure.getMessage());
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
@Test
@@ -380,7 +380,7 @@ class EcsContainerManagerFirelensTest {
assertEquals("ClientException", failure.getErrorCode());
assertEquals("A task definition can have only one FireLens log router.", failure.getMessage());
verify(lifecycleManager, never()).create(any(ContainerSpec.class));
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
@Test
@@ -467,7 +467,7 @@ class EcsContainerManagerFirelensTest {
order.verify(lifecycleManager).create(any());
order.verify(uploads.commands.get(1)).exec();
order.verify(lifecycleManager).startCreated(eq("router-id"), any());
- order.verify(lifecycleManager).createAndStart(any());
+ order.verify(lifecycleManager).createAndStartEphemeral(any());
verify(lifecycleManager, never()).removeIfExists(anyString());
if (external) {
verify(s3Service).getObject("firelens-configs", "extra.conf");
@@ -514,7 +514,7 @@ class EcsContainerManagerFirelensTest {
assertEquals(1, uploads.commands.size());
assertEquals(1, uploads.completed.size());
verify(lifecycleManager).startCreated(eq("router-id"), any());
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
verify(lifecycleManager).removeIfExists("router-id");
verifyFirelensVolumeRemoved();
}
@@ -546,7 +546,7 @@ class EcsContainerManagerFirelensTest {
assertThrows(AwsException.class, () -> manager.startTask(
task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1"));
verify(lifecycleManager, never()).create(any());
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
verifyFirelensVolumeRemoved();
}
@@ -612,7 +612,7 @@ class EcsContainerManagerFirelensTest {
private void verifyFailedLaunchCleanup() {
verify(lifecycleManager).create(any());
verify(lifecycleManager, never()).startCreated(anyString(), any());
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
verify(lifecycleManager).removeIfExists("router-id");
verifyFirelensVolumeRemoved();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerImagePullTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerImagePullTest.java
index cdb7445a9b..93398d95c5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerImagePullTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerImagePullTest.java
@@ -63,7 +63,7 @@ class EcsContainerManagerImagePullTest {
when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
lifecycleManager = mock(ContainerLifecycleManager.class);
- when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
when(config.services().ecs().imagePullBehavior()).thenReturn(ImagePullBehavior.ALWAYS);
@@ -91,7 +91,7 @@ class EcsContainerManagerImagePullTest {
InOrder order = inOrder(lifecycleManager);
order.verify(lifecycleManager).resolveImageForLaunch(APP_IMAGE, ImagePullBehavior.ALWAYS);
order.verify(lifecycleManager).resolveImageForLaunch(SIDECAR_IMAGE, ImagePullBehavior.ALWAYS);
- order.verify(lifecycleManager).createAndStart(any());
+ order.verify(lifecycleManager).createAndStartEphemeral(any());
List<Container> containers = task.getContainers();
assertEquals("sha256:new-manifest", containers.get(0).getImageDigest());
assertNull(containers.get(1).getImageDigest(), "a locally built image has no manifest digest");
@@ -136,7 +136,7 @@ class EcsContainerManagerImagePullTest {
assertThrows(NotFoundException.class,
() -> manager.startTask(task(), taskDefinition(), List.of(), "us-east-1"));
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
private static TaskDefinition taskDefinition() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java
index 9367fa96e1..f8bb207c89 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java
@@ -36,6 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.RETURNS_SELF;
import static org.mockito.Mockito.doAnswer;
@@ -239,11 +240,11 @@ class EcsContainerManagerLifecycleTest {
assertTrue(manager(lifecycleManager, namespacedConfig()).removeLeftoverContainers("current"));
- verify(lifecycleManager).removeIfExistsStrict("new-only");
- verify(lifecycleManager).removeIfExistsStrict("legacy-only");
- verify(lifecycleManager).removeIfExistsStrict("both");
- verify(lifecycleManager, never()).removeIfExistsStrict("current-run");
- verify(lifecycleManager, never()).removeIfExistsStrict("legacy-current-run");
+ verify(lifecycleManager).removeIfExistsStrict("new-only", true);
+ verify(lifecycleManager).removeIfExistsStrict("legacy-only", true);
+ verify(lifecycleManager).removeIfExistsStrict("both", true);
+ verify(lifecycleManager, never()).removeIfExistsStrict("current-run", true);
+ verify(lifecycleManager, never()).removeIfExistsStrict("legacy-current-run", true);
}
@Test
@@ -263,7 +264,7 @@ class EcsContainerManagerLifecycleTest {
assertTrue(manager(lifecycleManager, namespacedConfig()).removeLeftoverContainers("current"));
- verify(lifecycleManager, never()).removeIfExistsStrict(anyString());
+ verify(lifecycleManager, never()).removeIfExistsStrict(anyString(), eq(true));
}
@Test
@@ -297,7 +298,7 @@ class EcsContainerManagerLifecycleTest {
stubLeftoverListing(lifecycle, Map.of("old-container", leftover("old-container",
Map.of("io.floci.service", "ecs", "io.floci.owner", "alpha/4566", "io.floci.ecs.run", "previous"))),
Map.of());
- doThrow(new IllegalStateException("busy")).when(lifecycle).removeIfExistsStrict("old-container");
+ doThrow(new IllegalStateException("busy")).when(lifecycle).removeIfExistsStrict("old-container", true);
assertFalse(manager(lifecycle, namespacedConfig()).removeLeftoverContainers("current"));
verify(lifecycle.getDockerClient(), never()).listVolumesCmd();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLogConfigurationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLogConfigurationTest.java
index a785b8e502..fa43bd20e8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLogConfigurationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLogConfigurationTest.java
@@ -55,7 +55,7 @@ class EcsContainerManagerLogConfigurationTest {
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any()))
+ when(lifecycleManager.createAndStartEphemeral(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
logStreamer = mock(ContainerLogStreamer.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerOverridesTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerOverridesTest.java
index f6c40c3326..77a317dc27 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerOverridesTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerOverridesTest.java
@@ -66,7 +66,7 @@ class EcsContainerManagerOverridesTest {
lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any()))
+ when(lifecycleManager.createAndStartEphemeral(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerPortMappingsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerPortMappingsTest.java
index 79eba41090..0d67d77eff 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerPortMappingsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerPortMappingsTest.java
@@ -86,7 +86,7 @@ class EcsContainerManagerPortMappingsTest {
lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any()))
+ when(lifecycleManager.createAndStartEphemeral(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
// resolveNetworkBindings() inspects the container after launch; deep stubs
// make the empty-binding readback (Map.get(...) -> null) NPE-free.
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecretsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecretsTest.java
index a0b37b8cdc..35d3c427f5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecretsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecretsTest.java
@@ -61,7 +61,7 @@ class EcsContainerManagerSecretsTest {
lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any()))
+ when(lifecycleManager.createAndStartEphemeral(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);
@@ -187,7 +187,7 @@ class EcsContainerManagerSecretsTest {
assertTrue(thrown.getMessage().contains("ResourceInitializationError"));
assertTrue(thrown.getMessage().contains("/missing"));
verify(containerBuilder, never()).newContainer(anyString());
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
@Test
@@ -207,7 +207,7 @@ class EcsContainerManagerSecretsTest {
assertTrue(thrown.getMessage().contains("ResourceInitializationError"));
assertTrue(thrown.getMessage().contains(secretArn));
verify(containerBuilder, never()).newContainer(anyString());
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
@Test
@@ -273,7 +273,7 @@ class EcsContainerManagerSecretsTest {
assertTrue(thrown.getMessage().contains("ResourceInitializationError"));
assertTrue(thrown.getMessage().contains("did not contain json key token"));
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
@Test
@@ -305,7 +305,7 @@ class EcsContainerManagerSecretsTest {
assertTrue(thrown.getMessage().contains("ResourceInitializationError"));
assertTrue(thrown.getMessage().contains(
"an invalid ARN format for the AWS Secrets Manager secret was specified"));
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
private static ContainerDefinition containerDef(String name, List<Secret> secrets) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java
index 99fe650e0b..a5af392f1d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java
@@ -77,7 +77,7 @@ class EcsContainerManagerSecurityGroupTest {
lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
dockerClient = mock(DockerClient.class, RETURNS_DEEP_STUBS);
when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
@@ -245,7 +245,7 @@ class EcsContainerManagerSecurityGroupTest {
verify(firewallManager).createNamespace(eq("ecs"), eq("abc123"), any(), any(), any(), any(), any(), any());
verify(lifecycleManager).create(any());
verify(lifecycleManager).startCreated(eq("router-id"), any());
- verify(lifecycleManager).createAndStart(any());
+ verify(lifecycleManager).createAndStartEphemeral(any());
}
private static EcsTask awsvpcTask() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerTaskRoleCredentialsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerTaskRoleCredentialsTest.java
index ed931e5422..b3b542cfc6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerTaskRoleCredentialsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerTaskRoleCredentialsTest.java
@@ -75,7 +75,7 @@ class EcsContainerManagerTaskRoleCredentialsTest {
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
when(config.services().ecs().taskRoleCredentials().enabled()).thenReturn(true);
@@ -259,11 +259,11 @@ class EcsContainerManagerTaskRoleCredentialsTest {
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
// The reconciler's cleanup path is the other way a task's containers come off, and Docker
// can refuse a removal here just as it can on the stop path.
doThrow(new IllegalStateException("in use"))
- .when(lifecycleManager).stopAndRemoveStrict("docker-id", null);
+ .when(lifecycleManager).stopAndRemoveStrict("docker-id", null, 5, true);
EcsContainerManager stopping = new EcsContainerManager(containerBuilder, lifecycleManager,
mock(ContainerLogStreamer.class), mock(ContainerDetector.class), config,
mock(RegionResolver.class), baselineAwsEnv(), mock(SsmService.class),
@@ -288,11 +288,11 @@ class EcsContainerManagerTaskRoleCredentialsTest {
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
// The first container starts and the second does not, so the launch unwinds with one
// container already up and holding an address.
- when(failing.createAndStart(any()))
+ when(failing.createAndStartEphemeral(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()))
.thenThrow(new IllegalStateException("no daemon"));
doThrow(new IllegalStateException("in use"))
- .when(failing).stopAndRemoveStrict("docker-id", null);
+ .when(failing).stopAndRemoveStrict("docker-id", null, 5, true);
EcsContainerManager failingManager = new EcsContainerManager(containerBuilder, failing,
mock(ContainerLogStreamer.class), mock(ContainerDetector.class), config,
mock(RegionResolver.class), baselineAwsEnv(), mock(SsmService.class),
@@ -363,7 +363,7 @@ class EcsContainerManagerTaskRoleCredentialsTest {
ContainerLifecycleManager failing = mock(ContainerLifecycleManager.class);
when(failing.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(failing.createAndStart(any())).thenThrow(new IllegalStateException("no daemon"));
+ when(failing.createAndStartEphemeral(any())).thenThrow(new IllegalStateException("no daemon"));
EcsContainerManager failingManager = new EcsContainerManager(containerBuilder, failing,
mock(ContainerLogStreamer.class), mock(ContainerDetector.class), config,
mock(RegionResolver.class), mock(LaunchedContainerAwsEnv.class), mock(SsmService.class),
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromDockerIntegrationTest.java
index 533620c5e4..d0c6db7ee9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromDockerIntegrationTest.java
@@ -3,11 +3,20 @@ package io.github.hectorvent.floci.services.ecs.container;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.api.model.Bind;
+import com.github.dockerjava.api.model.Event;
+import com.github.dockerjava.api.model.EventType;
import com.github.dockerjava.api.model.Frame;
+import com.github.dockerjava.api.model.HostConfig;
+import com.github.dockerjava.api.model.Volume;
import com.github.dockerjava.core.command.BuildImageResultCallback;
import com.github.dockerjava.core.command.WaitContainerResultCallback;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.services.ecs.model.Container;
import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDependency;
import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
import io.github.hectorvent.floci.services.ecs.model.VolumeFrom;
@@ -23,14 +32,26 @@ import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
@@ -49,7 +70,11 @@ class EcsContainerManagerVolumesFromDockerIntegrationTest {
@Inject
DockerClient dockerClient;
+ @Inject
+ EmulatorConfig config;
+
private String sourceImage;
+ private final List<String> ownedVolumes = new ArrayList<>();
@BeforeEach
void requireDocker() {
@@ -59,11 +84,21 @@ class EcsContainerManagerVolumesFromDockerIntegrationTest {
@AfterEach
void removeSourceImage() {
- if (sourceImage != null) {
- try {
- dockerClient.removeImageCmd(sourceImage).withForce(true).exec();
- } catch (NotFoundException ignored) {
- // A failed build may not have produced an image to remove.
+ try {
+ for (String volume : ownedVolumes) {
+ try {
+ dockerClient.removeVolumeCmd(volume).exec();
+ } catch (NotFoundException ignored) {
+ // Successful task teardown already removed its anonymous volume.
+ }
+ }
+ } finally {
+ if (sourceImage != null) {
+ try {
+ dockerClient.removeImageCmd(sourceImage).withForce(true).exec();
+ } catch (NotFoundException ignored) {
+ // A failed build may not have produced an image to remove.
+ }
}
}
}
@@ -72,8 +107,8 @@ class EcsContainerManagerVolumesFromDockerIntegrationTest {
void appEntrypointCanComeFromSourceContainerVolume() throws Exception {
String suffix = UUID.randomUUID().toString().substring(0, 8);
String sourceImageTag = "floci-ecs-volumes-from-test:" + suffix;
- buildSourceImage(sourceImageTag);
sourceImage = sourceImageTag;
+ buildSourceImage(sourceImageTag);
ContainerDefinition app = definition("app", BUSYBOX_IMAGE);
app.setEntryPoint(List.of("/shared/wrapper"));
@@ -93,11 +128,19 @@ class EcsContainerManagerVolumesFromDockerIntegrationTest {
task.setTaskArn("arn:aws:ecs:us-east-1:000000000000:task/volumes-from/" + suffix);
EcsTaskHandle handle = containerManager.startTask(task, taskDefinition, List.of(), "us-east-1");
+ String sharedVolume;
try {
+ String sourceId = handle.getContainerIds().get("source");
+ sharedVolume = dockerClient.inspectContainerCmd(sourceId).exec().getMounts().stream()
+ .filter(mount -> "/shared".equals(mount.getDestination().getPath()))
+ .findFirst().orElseThrow().getName();
+ ownedVolumes.add(sharedVolume);
String appId = handle.getContainerIds().get("app");
- Integer status = dockerClient.waitContainerCmd(appId)
- .exec(new WaitContainerResultCallback())
- .awaitStatusCode(60, TimeUnit.SECONDS);
+ Integer status;
+ try (WaitContainerResultCallback callback = new WaitContainerResultCallback()) {
+ status = dockerClient.waitContainerCmd(appId).exec(callback)
+ .awaitStatusCode(60, TimeUnit.SECONDS);
+ }
String output = logs(appId);
assertEquals(0, status, output);
@@ -107,6 +150,211 @@ class EcsContainerManagerVolumesFromDockerIntegrationTest {
} finally {
containerManager.stopTask(handle);
}
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectVolumeCmd(sharedVolume).exec(),
+ "stopping every container must reclaim the inherited anonymous volume");
+ }
+
+ @Test
+ void stoppingTaskPreservesAnExplicitlyNamedVolume() {
+ String volume = "floci-ecs-named-volume-test-" + UUID.randomUUID();
+ ownedVolumes.add(volume);
+ dockerClient.createVolumeCmd().withName(volume).exec();
+ String containerId = dockerClient.createContainerCmd(BUSYBOX_IMAGE)
+ .withHostConfig(HostConfig.newHostConfig().withBinds(new Bind(volume, new Volume("/data"))))
+ .withCmd("true").exec().getId();
+ try {
+ dockerClient.startContainerCmd(containerId).exec();
+ containerManager.stopTask(new EcsTaskHandle("named-volume-task",
+ Map.of("app", containerId), Map.of()));
+ assertEquals(volume, dockerClient.inspectVolumeCmd(volume).exec().getName());
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectContainerCmd(containerId).exec());
+ } finally {
+ try {
+ dockerClient.removeContainerCmd(containerId).withForce(true).withRemoveVolumes(true).exec();
+ } catch (NotFoundException ignored) {
+ // Successful task teardown already removed the container.
+ }
+ }
+ }
+
+ @Test
+ void cleanupStoppedTaskReclaimsAnonymousVolumeAndPreservesNamedVolume() throws Exception {
+ assertStoppedContainerVolumesCleaned(false);
+ }
+
+ @Test
+ void previousRunCleanupReclaimsAnonymousVolumeAndPreservesNamedVolume() throws Exception {
+ assertStoppedContainerVolumesCleaned(true);
+ }
+
+ private void assertStoppedContainerVolumesCleaned(boolean previousRun) throws Exception {
+ String suffix = UUID.randomUUID().toString();
+ sourceImage = "floci-ecs-volumes-from-test:" + suffix;
+ buildSourceImage(sourceImage);
+ String namedVolume = "floci-ecs-named-volume-test-" + suffix;
+ ownedVolumes.add(namedVolume);
+ dockerClient.createVolumeCmd().withName(namedVolume).exec();
+ String containerId = dockerClient.createContainerCmd(sourceImage)
+ .withLabels(Map.of(ContainerStorageHelper.SERVICE_LABEL, "ecs",
+ ContainerStorageHelper.OWNER_LABEL, ContainerStorageHelper.ownerIdentity(config),
+ EcsContainerManager.RUN_LABEL, "previous-" + suffix))
+ .withHostConfig(HostConfig.newHostConfig()
+ .withBinds(new Bind(namedVolume, new Volume("/data"))))
+ .withCmd("true").exec().getId();
+ try {
+ String anonymousVolume = dockerClient.inspectContainerCmd(containerId).exec().getMounts().stream()
+ .filter(mount -> "/shared".equals(mount.getDestination().getPath()))
+ .findFirst().orElseThrow().getName();
+ ownedVolumes.add(anonymousVolume);
+ dockerClient.startContainerCmd(containerId).exec();
+ try (WaitContainerResultCallback callback = new WaitContainerResultCallback()) {
+ assertEquals(0, dockerClient.waitContainerCmd(containerId).exec(callback)
+ .awaitStatusCode(60, TimeUnit.SECONDS));
+ }
+ if (previousRun) {
+ assertTrue(containerManager.removeLeftoverContainers("current-" + suffix));
+ } else {
+ containerManager.cleanupStoppedTask(new EcsTaskHandle("stopped-volume-task-" + suffix,
+ Map.of("app", containerId), Map.of()));
+ }
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectContainerCmd(containerId).exec());
+ assertEquals(namedVolume, dockerClient.inspectVolumeCmd(namedVolume).exec().getName());
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectVolumeCmd(anonymousVolume).exec(),
+ "cleanup must reclaim image-declared anonymous volumes after the container exits");
+ } finally {
+ try {
+ dockerClient.removeContainerCmd(containerId).withForce(true).withRemoveVolumes(true).exec();
+ } catch (NotFoundException ignored) {
+ // The cleanup path under test already removed this container.
+ }
+ }
+ }
+
+ @Test
+ void failedDependencyReclaimsTheObservedSourceContainersAnonymousVolume() throws Exception {
+ String suffix = UUID.randomUUID().toString();
+ sourceImage = "floci-ecs-volumes-from-test:" + suffix;
+ buildSourceImage(sourceImage);
+ ContainerDefinition source = definition("source", sourceImage);
+ source.setCommand(List.of("sleep", "300"));
+ ContainerDefinition app = definition("app", BUSYBOX_IMAGE);
+ app.setCommand(List.of("true"));
+ app.setDependsOn(List.of(new ContainerDependency("source", ContainerDependency.SUCCESS)));
+ app.setStartTimeout(60);
+ TaskDefinition definition = new TaskDefinition();
+ definition.setFamily("failed-dependency-" + suffix);
+ definition.setContainerDefinitions(List.of(app, source));
+ EcsTask task = new EcsTask();
+ task.setTaskArn("arn:aws:ecs:us-east-1:000000000000:task/cleanup/" + suffix);
+ String sourceName = ContainerStorageHelper.dockerName(config, "ecs-" + suffix + "-source");
+ CountDownLatch started = new CountDownLatch(1);
+ AtomicReference<String> observedContainer = new AtomicReference<>();
+ try (ResultCallback.Adapter<Event> events = dockerClient.eventsCmd()
+ .withEventTypeFilter(EventType.CONTAINER)
+ .withEventFilter("start")
+ .withSince(String.valueOf(Instant.now().getEpochSecond()))
+ .exec(new ResultCallback.Adapter<Event>() {
+ @Override
+ public void onNext(Event event) {
+ if (event.getActor() != null
+ && sourceName.equals(event.getActor().getAttributes().get("name"))) {
+ observedContainer.set(event.getActor().getId());
+ started.countDown();
+ }
+ }
+ });
+ ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) {
+ Future<EcsTaskHandle> launch = executor.submit(
+ () -> containerManager.startTask(task, definition, List.of(), "us-east-1"));
+ try {
+ assertTrue(started.await(60, TimeUnit.SECONDS), "the owned source container must start");
+ String containerId = observedContainer.get();
+ String anonymousVolume = dockerClient.inspectContainerCmd(containerId).exec().getMounts().stream()
+ .filter(mount -> "/shared".equals(mount.getDestination().getPath()))
+ .findFirst().orElseThrow().getName();
+ ownedVolumes.add(anonymousVolume);
+ dockerClient.killContainerCmd(containerId).exec();
+ ExecutionException failure = assertThrows(ExecutionException.class,
+ () -> launch.get(60, TimeUnit.SECONDS));
+ AwsException cause = assertInstanceOf(AwsException.class, failure.getCause());
+ assertTrue(cause.getMessage().contains("SUCCESS"), cause.getMessage());
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectContainerCmd(containerId).exec());
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectVolumeCmd(anonymousVolume).exec(),
+ "failed partial launch must reclaim its source container's anonymous volume");
+ } finally {
+ launch.cancel(true);
+ }
+ } finally {
+ String containerId = observedContainer.get();
+ if (containerId != null) {
+ try {
+ dockerClient.removeContainerCmd(containerId).withForce(true).withRemoveVolumes(true).exec();
+ } catch (NotFoundException ignored) {
+ // Successful rollback already removed the observed source container.
+ }
+ }
+ }
+ }
+
+ @Test
+ void rejectedContainerStartReclaimsItsAnonymousVolume() throws Exception {
+ String suffix = UUID.randomUUID().toString();
+ sourceImage = "floci-ecs-volumes-from-test:" + suffix;
+ buildSourceImage(sourceImage);
+ ContainerDefinition app = definition("app", sourceImage);
+ app.setEntryPoint(List.of("/missing-entrypoint-" + suffix));
+ TaskDefinition definition = new TaskDefinition();
+ definition.setFamily("rejected-start-" + suffix);
+ definition.setContainerDefinitions(List.of(app));
+ EcsTask task = new EcsTask();
+ task.setTaskArn("arn:aws:ecs:us-east-1:000000000000:task/rejected-start/" + suffix);
+ String containerName = ContainerStorageHelper.dockerName(config, "ecs-" + suffix + "-app");
+ AtomicReference<String> observedContainer = new AtomicReference<>();
+ AtomicReference<String> observedVolume = new AtomicReference<>();
+ CountDownLatch mounted = new CountDownLatch(1);
+ try (ResultCallback.Adapter<Event> events = dockerClient.eventsCmd()
+ .withSince(String.valueOf(Instant.now().getEpochSecond()))
+ .exec(new ResultCallback.Adapter<Event>() {
+ @Override
+ public void onNext(Event event) {
+ if (event.getActor() == null) {
+ return;
+ }
+ Map<String, String> attributes = event.getActor().getAttributes();
+ if (event.getType() == EventType.CONTAINER
+ && "create".equals(event.getAction())
+ && containerName.equals(attributes.get("name"))) {
+ observedContainer.set(event.getActor().getId());
+ }
+ if (event.getType() == EventType.VOLUME
+ && "mount".equals(event.getAction())
+ && attributes.get("container") != null
+ && attributes.get("container").equals(observedContainer.get())) {
+ observedVolume.set(event.getActor().getId());
+ mounted.countDown();
+ }
+ }
+ })) {
+ RuntimeException failure = assertThrows(RuntimeException.class,
+ () -> containerManager.startTask(task, definition, List.of(), "us-east-1"));
+ assertTrue(failure.getMessage().contains("missing-entrypoint-" + suffix), failure.getMessage());
+ assertTrue(mounted.await(30, TimeUnit.SECONDS), "observe the owned rejected container's volume mount");
+ String anonymousVolume = observedVolume.get();
+ ownedVolumes.add(anonymousVolume);
+ assertThrows(NotFoundException.class,
+ () -> dockerClient.inspectContainerCmd(observedContainer.get()).exec());
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectVolumeCmd(anonymousVolume).exec(),
+ "rejected start must remove its anonymous volume before ECS has a task handle");
+ } finally {
+ String containerId = observedContainer.get();
+ if (containerId != null) {
+ try {
+ dockerClient.removeContainerCmd(containerId).withForce(true).withRemoveVolumes(true).exec();
+ } catch (NotFoundException ignored) {
+ // Failed-start cleanup already removed this owned container.
+ }
+ }
+ }
}
private void buildSourceImage(String tag) throws Exception {
@@ -137,18 +385,20 @@ class EcsContainerManagerVolumesFromDockerIntegrationTest {
}
}
- private String logs(String containerId) throws InterruptedException {
+ private String logs(String containerId) throws InterruptedException, IOException {
StringBuilder output = new StringBuilder();
- dockerClient.logContainerCmd(containerId)
- .withStdOut(true)
- .withStdErr(true)
- .exec(new ResultCallback.Adapter<Frame>() {
- @Override
- public void onNext(Frame frame) {
- output.append(new String(frame.getPayload(), StandardCharsets.UTF_8));
- }
- })
- .awaitCompletion(10, TimeUnit.SECONDS);
+ try (ResultCallback.Adapter<Frame> callback = new ResultCallback.Adapter<>() {
+ @Override
+ public void onNext(Frame frame) {
+ output.append(new String(frame.getPayload(), StandardCharsets.UTF_8));
+ }
+ }) {
+ dockerClient.logContainerCmd(containerId)
+ .withStdOut(true)
+ .withStdErr(true)
+ .exec(callback)
+ .awaitCompletion(10, TimeUnit.SECONDS);
+ }
return output.toString();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java
index 6fd2b58005..a9c318e73a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java
@@ -74,7 +74,7 @@ class EcsContainerManagerVolumesFromTest {
lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any()))
+ when(lifecycleManager.createAndStartEphemeral(any()))
.thenReturn(new ContainerInfo("source-id", Map.of()))
.thenReturn(new ContainerInfo("app-id", Map.of()));
@@ -160,11 +160,11 @@ class EcsContainerManagerVolumesFromTest {
order.verify(containerBuilder).newContainer("router:latest");
order.verify(containerBuilder).newContainer("app:latest");
InOrder lifecycleOrder = inOrder(lifecycleManager, copyCmd);
- lifecycleOrder.verify(lifecycleManager).createAndStart(any(ContainerSpec.class));
+ lifecycleOrder.verify(lifecycleManager).createAndStartEphemeral(any(ContainerSpec.class));
lifecycleOrder.verify(lifecycleManager).create(any(ContainerSpec.class));
lifecycleOrder.verify(copyCmd).exec();
lifecycleOrder.verify(lifecycleManager).startCreated(eq("router-id"), any(ContainerSpec.class));
- lifecycleOrder.verify(lifecycleManager).createAndStart(any(ContainerSpec.class));
+ lifecycleOrder.verify(lifecycleManager).createAndStartEphemeral(any(ContainerSpec.class));
verify(routerBuilder).withVolumesFrom("source-id", false);
verify(appBuilder).withVolumesFrom("source-id", true);
verify(lifecycleManager).ensureVolume(eq(handle.getFirelensVolumeName()), any());
@@ -206,7 +206,7 @@ class EcsContainerManagerVolumesFromTest {
verify(containerBuilder, never()).newContainer(anyString());
verify(lifecycleManager, never()).create(any());
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
verify(lifecycleManager, never()).ensureVolume(anyString(), any());
}
@@ -235,7 +235,7 @@ class EcsContainerManagerVolumesFromTest {
verify(s3Service).getObject("firelens-configs", "missing.conf");
verify(containerBuilder, never()).newContainer(anyString());
verify(lifecycleManager, never()).create(any());
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
verify(lifecycleManager, never()).ensureVolume(anyString(), any());
}
@@ -247,7 +247,7 @@ class EcsContainerManagerVolumesFromTest {
assertThrows(IllegalArgumentException.class,
() -> manager.startTask(task(), taskDefinition(List.of(app)), List.of(), "us-east-1"));
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
@Test
@@ -260,7 +260,7 @@ class EcsContainerManagerVolumesFromTest {
assertThrows(IllegalArgumentException.class,
() -> manager.startTask(task(), taskDefinition(List.of(app, source)), List.of(), "us-east-1"));
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
private static ContainerDefinition definition(String name, String image) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java
index ab02341d75..cb4f2c5363 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java
@@ -139,10 +139,10 @@ class EcsContainerManagerVolumesTest {
@ValueSource(booleans = {false, true})
void failedLaunchRemovesItsEphemeralVolume(boolean firstContainerStarted) {
if (firstContainerStarted) {
- when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()))
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(new ContainerInfo("docker-id", Map.of()))
.thenThrow(new IllegalStateException("launch failed"));
} else {
- when(lifecycleManager.createAndStart(any())).thenThrow(new IllegalStateException("launch failed"));
+ when(lifecycleManager.createAndStartEphemeral(any())).thenThrow(new IllegalStateException("launch failed"));
}
assertThrows(IllegalStateException.class, () -> manager.startTask(task("000000000000", "us-east-1"),
ephemeralDefinition(new Volume("shared", null)), List.of(), "us-east-1"));
@@ -177,13 +177,13 @@ class EcsContainerManagerVolumesTest {
ephemeralDefinition(new Volume("shared", null)), List.of(), "us-east-1");
String name = handle.getTaskVolumeNames().iterator().next();
doThrow(new IllegalStateException("remove failed")).when(lifecycleManager)
- .stopAndRemoveStrict("docker-id", null);
+ .stopAndRemoveStrict("docker-id", null, 5, true);
manager.cleanupStoppedTask(handle);
assertTrue(handle.hasTaskVolumes());
verify(lifecycleManager, never()).removeVolume(name);
- doNothing().when(lifecycleManager).stopAndRemoveStrict("docker-id", null);
+ doNothing().when(lifecycleManager).stopAndRemoveStrict("docker-id", null, 5, true);
when(lifecycleManager.removeVolume(name)).thenReturn(true);
manager.cleanupStoppedTask(handle);
assertFalse(handle.hasTaskVolumes());
@@ -192,11 +192,11 @@ class EcsContainerManagerVolumesTest {
@ParameterizedTest
@ValueSource(booleans = {false, true})
void failedLaunchPreservesResourcesWhenCleanupFails(boolean containerRemovalFails) {
- when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()))
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(new ContainerInfo("docker-id", Map.of()))
.thenThrow(new IllegalStateException("launch failed"));
if (containerRemovalFails) {
doThrow(new IllegalStateException("remove failed")).when(lifecycleManager)
- .stopAndRemoveStrict("docker-id", null);
+ .stopAndRemoveStrict("docker-id", null, 5, true);
} else {
when(lifecycleManager.removeVolume(anyString())).thenReturn(false);
}
@@ -208,7 +208,7 @@ class EcsContainerManagerVolumesTest {
assertEquals(!containerRemovalFails, handle.allContainersRemoved());
assertEquals("launch failed", failure.getMessage());
- doNothing().when(lifecycleManager).stopAndRemoveStrict("docker-id", null);
+ doNothing().when(lifecycleManager).stopAndRemoveStrict("docker-id", null, 5, true);
when(lifecycleManager.removeVolume(anyString())).thenReturn(true);
manager.cleanupStoppedTask(handle);
assertTrue(handle.allContainersRemoved());
@@ -262,7 +262,7 @@ class EcsContainerManagerVolumesTest {
when(lifecycleManager.removeVolume(anyString())).thenReturn(true);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
- when(lifecycleManager.createAndStart(any()))
+ when(lifecycleManager.createAndStartEphemeral(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);
@@ -470,7 +470,7 @@ class EcsContainerManagerVolumesTest {
() -> managerWithEfs(efsFallbackConfig()).startTask(task("000000000000", "us-east-1"),
accessPointDefinition("fsap-test"), List.of(), "us-east-1"));
assertEquals("ClientException", failure.getErrorCode());
- verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
}
@Test