Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): mount and clean up ephemeral task volumes (#5511)

Mohith760c38ef99d2
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsEphemeralVolumeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsEphemeralVolumeTest.java
new file mode 100644
index 0000000000..2bcfab9f52
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsEphemeralVolumeTest.java
@@ -0,0 +1,112 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.core.waiters.WaiterOverrideConfiguration;
+import software.amazon.awssdk.services.ec2.Ec2Client;
+import software.amazon.awssdk.services.ecs.EcsClient;
+import software.amazon.awssdk.services.ecs.model.Compatibility;
+import software.amazon.awssdk.services.ecs.model.ContainerDefinition;
+import software.amazon.awssdk.services.ecs.model.DescribeTasksRequest;
+import software.amazon.awssdk.services.ecs.model.EFSVolumeConfiguration;
+import software.amazon.awssdk.services.ecs.model.HostVolumeProperties;
+import software.amazon.awssdk.services.ecs.model.InvalidParameterException;
+import software.amazon.awssdk.services.ecs.model.LaunchType;
+import software.amazon.awssdk.services.ecs.model.MountPoint;
+import software.amazon.awssdk.services.ecs.model.NetworkMode;
+import software.amazon.awssdk.services.ecs.model.RunTaskResponse;
+import software.amazon.awssdk.services.ecs.model.Task;
+import software.amazon.awssdk.services.ecs.model.TaskDefinition;
+import software.amazon.awssdk.services.ecs.model.Volume;
+
+import java.time.Duration;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+@DisplayName("ECS ephemeral volumes shared within a task")
+class EcsEphemeralVolumeTest {
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void efsVolumeRequiresFileSystemId(boolean blankId) {
+ String name = "sdk-invalid-efs-" + UUID.randomUUID().toString().substring(0, 8);
+ EFSVolumeConfiguration.Builder efs = EFSVolumeConfiguration.builder();
+ if (blankId) {
+ efs.fileSystemId("");
+ }
+ try (EcsClient ecs = TestFixtures.ecsClient()) {
+ assertThatThrownBy(() -> ecs.registerTaskDefinition(request -> request.family(name)
+ .containerDefinitions(container("reader", true, "true"))
+ .volumes(Volume.builder().name("shared").efsVolumeConfiguration(efs.build()).build())))
+ .isInstanceOf(InvalidParameterException.class);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void nameOnlyAndEmptyHostVolumesShareFilesBetweenContainers(boolean emptyHost) {
+ String name = "sdk-ephemeral-volume-" + UUID.randomUUID().toString().substring(0, 8);
+ Volume.Builder volume = Volume.builder().name("shared");
+ if (emptyHost) {
+ volume.host(HostVolumeProperties.builder().build());
+ }
+ try (EcsClient ecs = TestFixtures.ecsClient(); Ec2Client ec2 = TestFixtures.ec2Client()) {
+ String subnet = ec2.describeSubnets().subnets().get(0).subnetId();
+ ecs.createCluster(request -> request.clusterName(name));
+ TaskDefinition definition = ecs.registerTaskDefinition(request -> request.family(name)
+ .networkMode(NetworkMode.AWSVPC).requiresCompatibilities(Compatibility.FARGATE)
+ .cpu("256").memory("512").volumes(volume.build())
+ .containerDefinitions(
+ container("writer", false, "echo shared > /data/marker"),
+ container("reader", true, """
+ for attempt in 1 2 3 4 5; do
+ if [ "$(cat /data/marker 2>/dev/null)" = shared ]; then exit 0; fi
+ sleep 1
+ done
+ exit 42
+ """))).taskDefinition();
+ try {
+ assertThat(definition.volumes()).hasSize(1);
+ assertThat(definition.volumes().get(0).name()).isEqualTo("shared");
+ assertThat(definition.volumes().get(0).host() != null).isEqualTo(emptyHost);
+ RunTaskResponse started = ecs.runTask(request -> request.cluster(name)
+ .taskDefinition(definition.taskDefinitionArn()).launchType(LaunchType.FARGATE)
+ .networkConfiguration(network -> network.awsvpcConfiguration(vpc -> vpc.subnets(subnet))));
+ assertThat(started.failures()).isEmpty();
+ assertThat(started.tasks()).hasSize(1);
+ Task task = started.tasks().get(0);
+ try {
+ assertThat(task.containers()).hasSize(2);
+ assumeTrue(task.containers().stream().allMatch(container -> container.runtimeId() != null),
+ "ECS Docker mode is required to check files shared by running containers");
+ DescribeTasksRequest describe = DescribeTasksRequest.builder()
+ .cluster(name).tasks(task.taskArn()).build();
+ assertThat(ecs.waiter().waitUntilTasksStopped(describe,
+ WaiterOverrideConfiguration.builder().waitTimeout(Duration.ofSeconds(30)).build())
+ .matched().response()).isPresent();
+ Task stopped = ecs.describeTasks(describe).tasks().get(0);
+ assertThat(stopped.containers()).hasSize(2).allSatisfy(container -> {
+ assertThat(container.lastStatus()).isEqualTo("STOPPED");
+ assertThat(container.exitCode()).isZero();
+ });
+ } finally {
+ ecs.stopTask(request -> request.cluster(name).task(task.taskArn()));
+ }
+ } finally {
+ ecs.deregisterTaskDefinition(request -> request.taskDefinition(definition.taskDefinitionArn()));
+ ecs.deleteCluster(request -> request.cluster(name));
+ }
+ }
+ }
+
+ private static ContainerDefinition container(String name, boolean readOnly, String command) {
+ return ContainerDefinition.builder().name(name).image("busybox:stable").essential(true)
+ .command("sh", "-c", command).stopTimeout(1)
+ .mountPoints(MountPoint.builder().sourceVolume("shared").containerPath("/data")
+ .readOnly(readOnly).build()).build();
+ }
+}
diff --git a/docs/services/ecs.md b/docs/services/ecs.md
index b7e9ad4835..11d011dab0 100644
--- a/docs/services/ecs.md
+++ b/docs/services/ecs.md
@@ -862,6 +862,23 @@ for reasons that have nothing to do with the role's policy.
Each of these logs a warning naming the task and the role it asked for.
+### Ephemeral task volumes
+
+A volume declared with only a `name`, or with an empty `host` and no `sourcePath`, supplies
+ephemeral storage shared by the containers in that task. Each container's `mountPoints` selects
+the volume, mount path and read-only mode. Floci creates a Docker volume for each mounted task
+volume; different tasks get separate storage even when they use the same task definition.
+
+Task teardown removes this storage after the containers have been removed. Cleanup also runs
+when a task exits naturally or fails to launch. The task reconciler retries failed removals,
+including cleanup left pending by a failed launch or a state reset.
+With `reconcile-containers-on-startup` enabled, Floci also removes task volumes left by a previous
+run after their containers are gone. Failed volume cleanup retries on reconciler ticks without
+blocking new task launches. The sweep matches the emulator owner and run labels; it
+leaves current-run volumes and other emulator instances alone. Unlike AWS's configurable delayed
+EC2 agent cleanup, Floci attempts removal during teardown. EFS-backed volumes remain shared and persistent, and the
+unsupported external volume configurations listed above remain unimplemented.
+
### Host volume safety
A task definition's `volumes[].host.sourcePath` is a caller-controlled filesystem path that Floci bind-mounts straight into the launched container, so both `RegisterTaskDefinition` and the actual bind mount at `RunTask` time validate it (the second check narrows the window between validation and mount, and also covers task definitions registered before this policy existed).
@@ -886,7 +903,7 @@ Always rejected, regardless of configuration:
- `FLOCI_SERVICES_ECS_ALLOW_UNSAFE_HOST_VOLUMES=true`: allow any host path (for local development where any host path should be mountable). The traversal, bare-root, and Docker socket blocks above are never bypassed by this flag.
-A rejected `sourcePath` fails `RegisterTaskDefinition` with `InvalidParameterException`; a rejection caught again at `RunTask` time (e.g. a task definition registered before this policy existed) stops the task with that message as its `stoppedReason`. Named Docker volumes, EFS volumes, and host volumes with no `sourcePath` (ephemeral, container-local storage) are unaffected by these checks.
+A rejected `sourcePath` fails `RegisterTaskDefinition` with `InvalidParameterException`; a rejection caught again at `RunTask` time (e.g. a task definition registered before this policy existed) stops the task with that message as its `stoppedReason`. Named Docker volumes, EFS volumes, and host volumes with no `sourcePath` (ephemeral storage shared within a task) are unaffected by these checks.
### EFS volume ownership
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 818b437407..605d323bfe 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -583,14 +583,23 @@ public class ContainerLifecycleManager {
createVolumeIfAbsent(volumeName);
}
- /** Creates the named volume when it is missing; returns whether it was created. */
+ /** Creates a volume with resource ownership labels for recoverable cleanup. */
+ public void ensureVolume(String volumeName, Map<String, String> labels) {
+ createVolumeIfAbsent(volumeName, labels);
+ }
+
private boolean createVolumeIfAbsent(String volumeName) {
+ return createVolumeIfAbsent(volumeName, Map.of());
+ }
+
+ /** Creates the named volume when it is missing; returns whether it was created. */
+ private boolean createVolumeIfAbsent(String volumeName, Map<String, String> labels) {
if (volumeExists(volumeName)) {
return false;
}
dockerClient.createVolumeCmd()
.withName(volumeName)
- .withLabels(ContainerStorageHelper.defaultLabels(config))
+ .withLabels(mergedLabels(labels))
.exec();
LOG.debugv("Created volume {0}", volumeName);
return true;
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandler.java
index 61cbe91905..24e87852ab 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandler.java
@@ -1517,12 +1517,16 @@ public class EcsJsonHandler {
private static final Set<String> VOLUME_CONSUMED = Set.of("name", "efsVolumeConfiguration");
private EfsVolumeConfiguration parseEfsVolumeConfiguration(JsonNode node) {
- if (node == null || !node.isObject()) {
+ if (node == null || node.isMissingNode() || node.isNull()) {
return null;
}
+ if (!node.isObject()) {
+ throw new AwsException("InvalidParameterException", "EFS volume configuration must be an object.", 400);
+ }
String fileSystemId = node.path("fileSystemId").asText(null);
- if (fileSystemId == null) {
- return null;
+ if (fileSystemId == null || fileSystemId.isBlank()) {
+ throw new AwsException("InvalidParameterException",
+ "EFS volume configuration requires a fileSystemId.", 400);
}
Integer transitEncryptionPort = node.path("transitEncryptionPort").isNumber()
? node.path("transitEncryptionPort").asInt() : null;
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index 71704a8242..bfb24d09b2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -19,6 +19,7 @@ import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsS
import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;
import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;
+import io.github.hectorvent.floci.services.ecs.container.EcsTaskLaunchException;
import io.github.hectorvent.floci.services.ecs.exec.EcsExecChannelHandler;
import io.github.hectorvent.floci.services.ecs.exec.EcsExecSessionRegistry;
import io.github.hectorvent.floci.services.ecs.exec.ExecSession;
@@ -128,6 +129,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
private final Object reconcilerLock = new Object();
// False until a previous run's task containers are confirmed gone; no task starts before.
private volatile boolean leftoverContainersRemoved = true;
+ private volatile boolean leftoverTaskVolumesRemoved;
private volatile boolean leftoverContainersBlockReported;
// region::clusterName → EcsCluster
@@ -138,8 +140,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
private Map<String, Integer> latestRevisions = new ConcurrentHashMap<>();
// taskArn → EcsTask
private final Map<String, EcsTask> tasks = new ConcurrentHashMap<>();
- // taskArn → EcsTaskHandle (running containers or unresolved log readers)
+ // taskArn → EcsTaskHandle (running containers or pending resource cleanup)
private final Map<String, EcsTaskHandle> taskHandles = new ConcurrentHashMap<>();
+ private final Set<String> shutdownCleanupTasks = ConcurrentHashMap.newKeySet();
// region::clusterName/serviceName → EcsServiceModel
private Map<String, EcsServiceModel> services = new ConcurrentHashMap<>();
private AccountAwareStorageBackend<EcsServiceModel> servicesStore;
@@ -239,6 +242,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
this.containerManager = containerManager;
this.dockerMode = !config.services().ecs().mock();
this.sweepLeftoverContainers = dockerMode && config.services().ecs().reconcileContainersOnStartup();
+ this.leftoverTaskVolumesRemoved = !sweepLeftoverContainers;
this.baseUrl = config.effectiveBaseUrl();
this.lbRegistrar = lbRegistrar;
this.storageFactory = storageFactory;
@@ -287,6 +291,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
void releasePreviousRunLeftovers() {
if (sweepLeftoverContainers) {
leftoverContainersRemoved = containerManager.removeLeftoverContainers();
+ retryPreviousRunTaskVolumes();
}
if (lbRegistrar != null) {
lbRegistrar.releaseRecordedTargets();
@@ -384,8 +389,16 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
}
try {
containerManager.stopTask(claimed);
+ for (int attempt = 0; attempt < 2 && claimed.hasTaskVolumes(); attempt++) {
+ containerManager.cleanupStoppedTask(claimed);
+ }
} catch (Exception e) {
LOG.warnv("Failed to stop ECS task {0} on shutdown: {1}", taskArn, e.getMessage());
+ } finally {
+ if (claimed.hasTaskVolumes()) {
+ taskHandles.put(taskArn, claimed);
+ shutdownCleanupTasks.add(taskArn);
+ }
}
}
// Task state is memory-only, so an ENI left behind here would outlive every task that
@@ -1551,6 +1564,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
stopTask(cluster.getClusterName(), taskArn, task.getStoppedReason(), task.getStopCode(), region);
}
} catch (Exception e) {
+ if (e instanceof EcsTaskLaunchException failedLaunch) {
+ retainPendingCleanupHandle(taskArn, failedLaunch.getHandle());
+ }
LOG.errorv("Failed to start ECS task {0}: {1}", taskArn, e.getMessage());
task.setLastStatus(TaskStatus.STOPPED.name());
task.setDesiredStatus(TaskStatus.STOPPED.name());
@@ -1983,7 +1999,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
return task;
}
taskHandles.remove(task.getTaskArn(), handle);
- retainUnresolvedLogHandle(task.getTaskArn(), handle);
+ retainPendingCleanupHandle(task.getTaskArn(), handle);
finishedAt = handle.getFinishedAt();
}
@@ -4716,6 +4732,11 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
// ── Service Reconciliation ────────────────────────────────────────────────
void reconcile() {
+ try {
+ retryPreviousRunTaskVolumes();
+ } catch (Exception e) {
+ LOG.warnv(e, "ECS previous-run task volume cleanup failed: {0}", e.getMessage());
+ }
try {
reconcileTasks();
} catch (Exception e) {
@@ -4733,6 +4754,12 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
}
}
+ private void retryPreviousRunTaskVolumes() {
+ if (sweepLeftoverContainers && leftoverContainersRemoved && !leftoverTaskVolumesRemoved) {
+ leftoverTaskVolumesRemoved = containerManager.removeLeftoverTaskVolumes();
+ }
+ }
+
/**
* Whether a task may start: true once the containers a previous run left behind are gone,
* retrying the sweep otherwise. It runs before every task launch, RunTask and StartTask
@@ -4798,8 +4825,13 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
}
private void reconcileTask(String taskArn) {
+ if (shutdownCleanupTasks.contains(taskArn)) {
+ reconcileStoppedTask(taskArn);
+ return;
+ }
EcsTask task = tasks.get(taskArn);
if (task == null) {
+ reconcileStoppedTask(taskArn);
return;
}
synchronized (task) {
@@ -4854,6 +4886,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
// Close log streams and remove the stopped Docker containers without re-inspecting.
containerManager.cleanupStoppedTask(claimed);
+ retainPendingCleanupHandle(taskArn, claimed);
if (task.getContainers() != null) {
task.getContainers().forEach(c -> {
@@ -4961,14 +4994,17 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
containerManager.stopTaskAndCollectExitCodes(claimed);
} finally {
// A task can be reported STOPPED even if Docker rejected both teardown attempts.
- // Keep only the affected log readers and retry their teardown on later reconciliation ticks.
- retainUnresolvedLogHandle(taskArn, claimed);
+ // Keep pending log readers and task volumes for later cleanup attempts.
+ retainPendingCleanupHandle(taskArn, claimed);
}
}
- private void retainUnresolvedLogHandle(String taskArn, EcsTaskHandle handle) {
- if (handle != null && handle.hasOpenLogStreams()) {
+ private void retainPendingCleanupHandle(String taskArn, EcsTaskHandle handle) {
+ if (handle != null && (handle.hasOpenLogStreams() || handle.hasTaskVolumes()
+ || !handle.allContainersRemoved())) {
taskHandles.put(taskArn, handle);
+ } else {
+ shutdownCleanupTasks.remove(taskArn);
}
}
@@ -4986,8 +5022,10 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
containerIds.put(container.getName(), dockerId);
}
Map<String, Integer> stopTimeouts = new LinkedHashMap<>();
+ Set<String> taskVolumeNames = Set.of();
try {
TaskDefinition definition = resolveTaskDefinitionOrThrow(task.getTaskDefinitionArn(), region);
+ taskVolumeNames = containerManager.taskVolumeNames(task, definition);
for (ContainerDefinition container : definition.getContainerDefinitions()) {
if (container.getStopTimeout() != null) {
stopTimeouts.put(container.getName(), container.getStopTimeout());
@@ -4998,7 +5036,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
task.getTaskArn());
}
EcsTaskHandle recovered = new EcsTaskHandle(task.getTaskArn(), containerIds, Map.of(),
- null, task.getNetworkInterfaceId(), region, stopTimeouts);
+ null, task.getNetworkInterfaceId(), region, stopTimeouts, taskVolumeNames);
EcsTaskHandle existing = taskHandles.putIfAbsent(task.getTaskArn(), recovered);
return existing != null ? existing : recovered;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
index 7eba3b4e75..de674b2c49 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
@@ -3,6 +3,8 @@ package io.github.hectorvent.floci.services.ecs.container;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
+import com.github.dockerjava.api.command.ListVolumesResponse;
import com.github.dockerjava.api.command.StatsCmd;
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.ContainerNetwork;
@@ -75,6 +77,7 @@ import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -144,6 +147,8 @@ public class EcsContainerManager {
// A sweep failure is retried before each task launch; each distinct failure is a WARN once.
private volatile boolean leftoverListFailureReported;
private final Set<String> leftoverRemovalFailuresReported = ConcurrentHashMap.newKeySet();
+ private volatile boolean leftoverTaskVolumeListFailureReported;
+ private final Set<String> leftoverTaskVolumeRemovalFailuresReported = ConcurrentHashMap.newKeySet();
@Inject
public EcsContainerManager(ContainerBuilder containerBuilder,
@@ -230,15 +235,19 @@ public class EcsContainerManager {
// name -> EFS configuration (materialised below as a shared local Docker volume).
Map<String, String> volumeSourcePaths = new LinkedHashMap<>();
Map<String, EfsVolumeConfiguration> efsVolumes = new LinkedHashMap<>();
+ Map<String, String> ephemeralVolumes = new LinkedHashMap<>();
+ Set<String> taskVolumeNames = new LinkedHashSet<>();
if (taskDef.getVolumes() != null) {
for (Volume v : taskDef.getVolumes()) {
if (v.name() == null) {
continue;
}
- if (v.hostSourcePath() != null) {
+ if (v.hostSourcePath() != null && !v.hostSourcePath().isBlank()) {
volumeSourcePaths.put(v.name(), v.hostSourcePath());
} else if (v.efs() != null) {
efsVolumes.put(v.name(), v.efs());
+ } else if (isEphemeralVolume(v)) {
+ ephemeralVolumes.put(v.name(), taskVolumeName(task, v));
}
}
}
@@ -299,6 +308,13 @@ public class EcsContainerManager {
throw e;
}
+ Map<String, Integer> stopTimeouts = new LinkedHashMap<>();
+ for (ContainerDefinition def : taskDef.getContainerDefinitions()) {
+ if (def.getStopTimeout() != null) {
+ stopTimeouts.put(def.getName(), def.getStopTimeout());
+ }
+ }
+
String fluentHost = null;
String networkModeName = taskDef.getNetworkMode() != null
? taskDef.getNetworkMode().name()
@@ -445,6 +461,7 @@ public class EcsContainerManager {
}
String sourcePath = volumeSourcePaths.get(mp.sourceVolume());
EfsVolumeConfiguration efs = efsVolumes.get(mp.sourceVolume());
+ String ephemeralVolume = ephemeralVolumes.get(mp.sourceVolume());
if (sourcePath != null) {
// Host volume: bind-mount an absolute path on the Docker host. Re-validate
// here (not just at RegisterTaskDefinition time): this narrows the window
@@ -460,6 +477,13 @@ public class EcsContainerManager {
}
} else if (efs != null) {
mountEfsVolume(specBuilder, efs, mp);
+ } else if (ephemeralVolume != null) {
+ if (taskVolumeNames.add(ephemeralVolume)) {
+ Map<String, String> labels = new LinkedHashMap<>(ownerLabels());
+ labels.put(ContainerStorageHelper.SERVICE_LABEL, "ecs");
+ lifecycleManager.ensureVolume(ephemeralVolume, labels);
+ }
+ specBuilder.withNamedVolume(ephemeralVolume, mp.containerPath(), mp.readOnly());
} else {
LOG.warnv("Skipping mountPoint with unresolved volume {0} on container {1}",
mp.sourceVolume(), def.getName());
@@ -497,6 +521,8 @@ public class EcsContainerManager {
dockerId = info.containerId();
}
+ containerIds.put(def.getName(), dockerId);
+
LOG.infov("Created ECS container {0} for task {1} container {2}", dockerId, taskId, def.getName());
// Resolve network bindings for ECS-specific model
@@ -508,7 +534,6 @@ public class EcsContainerManager {
metadataIdsByContainer.get(def.getName()));
container.setImageDigest(imageDigestsByContainer.get(def.getName()));
runtimeContainers.add(container);
- containerIds.put(def.getName(), dockerId);
// awsfirelens containers are shipped to Fluent Bit by Docker; don't also scrape json-file.
// Only awslogs containers go to CloudWatch Logs; the others stay on the console.
@@ -528,23 +553,16 @@ public class EcsContainerManager {
}
}
} catch (Exception e) {
- boolean allRemoved = true;
- for (String dockerId : containerIds.values()) {
- if (!stopAndConfirmRemoved(dockerId)) {
- allRemoved = false;
- }
- }
- if (firelensVolumeName != null) {
- lifecycleManager.removeVolume(firelensVolumeName);
- }
+ EcsTaskHandle partial = new EcsTaskHandle(task.getTaskArn(), containerIds, logStreamsByContainerId,
+ firelensVolumeName,
+ protectedNetwork == null ? null : protectedNetwork.eni().getNetworkInterfaceId(), region,
+ stopTimeouts, taskVolumeNames);
+ cleanupStoppedTask(partial);
if (protectedNetwork != null) {
releaseTaskNetwork(task, region);
}
- revokeTaskRoleCredentials(task.getTaskArn());
- // A container left behind by a failed removal is still on the network holding its
- // address, exactly as on the stop path.
- if (allRemoved) {
- releaseTaskLinkLocalAddresses(task.getTaskArn());
+ if (!partial.allContainersRemoved() || partial.hasTaskVolumes() || partial.hasOpenLogStreams()) {
+ throw new EcsTaskLaunchException(e, partial);
}
throw e;
}
@@ -577,16 +595,44 @@ public class EcsContainerManager {
task.setDesiredStatus(TaskStatus.RUNNING.name());
task.setStartedAt(Instant.now());
- Map<String, Integer> stopTimeouts = new LinkedHashMap<>();
- for (ContainerDefinition def : taskDef.getContainerDefinitions()) {
- if (def.getStopTimeout() != null) {
- stopTimeouts.put(def.getName(), def.getStopTimeout());
- }
- }
return new EcsTaskHandle(task.getTaskArn(), containerIds, logStreamsByContainerId,
firelensVolumeName,
protectedNetwork == null ? null : protectedNetwork.eni().getNetworkInterfaceId(), region,
- stopTimeouts);
+ stopTimeouts, taskVolumeNames);
+ }
+
+ private String taskVolumeName(EcsTask task, Volume volume) {
+ return ContainerStorageHelper.dockerName(config,
+ "ecs-task-volume-" + Digests.sha256Hex(task.getTaskArn() + "|" + volume.name()));
+ }
+
+ /** Reconstructs only task-owned volumes referenced by a container, without creating them. */
+ public Set<String> taskVolumeNames(EcsTask task, TaskDefinition definition) {
+ Set<String> referenced = new HashSet<>();
+ for (ContainerDefinition container : definition.getContainerDefinitions()) {
+ if (container.getMountPoints() != null) {
+ container.getMountPoints().forEach(mount -> referenced.add(mount.sourceVolume()));
+ }
+ }
+ Set<String> names = new LinkedHashSet<>();
+ if (definition.getVolumes() != null) {
+ for (Volume volume : definition.getVolumes()) {
+ if (referenced.contains(volume.name()) && isEphemeralVolume(volume)) {
+ names.add(taskVolumeName(task, volume));
+ }
+ }
+ }
+ return names;
+ }
+
+ private static boolean isEphemeralVolume(Volume volume) {
+ Map<String, Object> properties = volume.unparsed();
+ return (volume.hostSourcePath() == null || volume.hostSourcePath().isBlank()) && volume.efs() == null
+ && (properties == null || (properties.get("efsVolumeConfiguration") == null
+ && properties.get("dockerVolumeConfiguration") == null
+ && properties.get("fsxWindowsFileServerVolumeConfiguration") == null
+ && properties.get("s3filesVolumeConfiguration") == null
+ && !Boolean.TRUE.equals(properties.get("configuredAtLaunch"))));
}
/** A task-level {@code memory} is MiB as a plain number, or a {@code "1GB"}-style string. */
@@ -1163,6 +1209,61 @@ public class EcsContainerManager {
return allRemoved;
}
+ /** Removes previous-run task volumes independently of the container launch gate. */
+ public boolean removeLeftoverTaskVolumes() {
+ return removeLeftoverTaskVolumes(runId);
+ }
+
+ /** {@link #removeLeftoverTaskVolumes()} keeping volumes of {@code currentRunId}. */
+ public boolean removeLeftoverTaskVolumes(String currentRunId) {
+ String owner = ContainerStorageHelper.ownerIdentity(config);
+ ContainerStorageHelper.LabelAliases aliases = ContainerStorageHelper.CONTAINER_LABEL_ALIASES;
+ try {
+ List<InspectVolumeResponse> volumes = aliases.listByLabels(
+ Map.of(ContainerStorageHelper.SERVICE_LABEL, "ecs", ContainerStorageHelper.OWNER_LABEL, owner),
+ filter -> {
+ ListVolumesResponse response = lifecycleManager.getDockerClient().listVolumesCmd()
+ .withFilter("label", filter.entrySet().stream()
+ .map(entry -> entry.getKey() + "=" + entry.getValue()).toList()).exec();
+ if (response == null || response.getVolumes() == null) {
+ throw new IllegalStateException("Docker returned no volume inventory");
+ }
+ return response.getVolumes();
+ }, InspectVolumeResponse::getName);
+ leftoverTaskVolumeListFailureReported = false;
+ boolean allRemoved = true;
+ for (InspectVolumeResponse volume : volumes) {
+ Map<String, String> labels = volume.getLabels();
+ String previousRun = aliases.labelValue(labels, RUN_LABEL);
+ if (!aliases.consistent(volume.getName(), labels)
+ || !aliases.matches(labels, ContainerStorageHelper.SERVICE_LABEL, "ecs")
+ || !aliases.matches(labels, ContainerStorageHelper.OWNER_LABEL, owner)
+ || previousRun == null || currentRunId.equals(previousRun)) {
+ continue;
+ }
+ try {
+ lifecycleManager.removeVolumeStrict(volume.getName());
+ leftoverTaskVolumeRemovalFailuresReported.remove(volume.getName());
+ } catch (RuntimeException e) {
+ allRemoved = false;
+ Logger.Level level = leftoverTaskVolumeRemovalFailuresReported.add(volume.getName())
+ ? Logger.Level.WARN : Logger.Level.DEBUG;
+ LOG.logv(level, "Could not remove ECS task volume {0} left by a previous run: {1}",
+ volume.getName(), e.getMessage());
+ }
+ }
+ if (allRemoved) {
+ leftoverTaskVolumeRemovalFailuresReported.clear();
+ }
+ return allRemoved;
+ } catch (RuntimeException e) {
+ LOG.logv(leftoverTaskVolumeListFailureReported ? Logger.Level.DEBUG : Logger.Level.WARN,
+ "Could not list ECS task volumes from a previous run: {0}", e.getMessage());
+ leftoverTaskVolumeListFailureReported = true;
+ return false;
+ }
+ }
+
/** The ENI {@link #attachTaskNetwork} allocated for this task, or null if it is already gone. */
private NetworkInterface existingTaskEni(EcsTask task, String region) {
if (task.getNetworkInterfaceId() == null) {
@@ -1274,8 +1375,10 @@ public class EcsContainerManager {
return;
}
boolean allRemoved = true;
- for (String dockerId : handle.getContainerIds().values()) {
- if (!stopAndConfirmRemoved(dockerId)) {
+ for (Map.Entry<String, String> entry : handle.getContainerIds().entrySet()) {
+ if (stopAndConfirmRemoved(entry.getValue())) {
+ handle.recordContainerRemoved(entry.getKey());
+ } else {
allRemoved = false;
}
}
@@ -1285,6 +1388,7 @@ public class EcsContainerManager {
removeFirelensVolume(handle);
revokeTaskRoleCredentials(handle.getTaskArn());
if (allRemoved) {
+ removeTaskVolumes(handle);
releaseTaskLinkLocalAddresses(handle.getTaskArn());
}
}
@@ -1373,6 +1477,7 @@ public class EcsContainerManager {
if (handle.allContainersRemoved()) {
cleanupProtectedNetwork(handle);
removeFirelensVolume(handle);
+ removeTaskVolumes(handle);
// Only now: a container that survived a failed removal still holds its address.
releaseTaskLinkLocalAddresses(handle.getTaskArn());
}
@@ -1423,6 +1528,14 @@ public class EcsContainerManager {
lifecycleManager.removeVolume(handle.getFirelensVolumeName());
}
+ private void removeTaskVolumes(EcsTaskHandle handle) {
+ for (String volumeName : handle.getTaskVolumeNames()) {
+ if (lifecycleManager.removeVolume(volumeName)) {
+ handle.recordTaskVolumeRemoved(volumeName);
+ }
+ }
+ }
+
private static ContainerDefinition findFirelensRouter(List<ContainerDefinition> defs) {
if (defs == null) {
return null;
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java
index b36da8baf3..19b0f47a26 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java
@@ -4,6 +4,7 @@ import java.io.Closeable;
import java.time.Instant;
import java.util.HashSet;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
@@ -29,6 +30,7 @@ public class EcsTaskHandle {
private final String firelensVolumeName;
private final String networkInterfaceId;
private final String region;
+ private final Set<String> taskVolumeNames;
/** Per-container {@code stopTimeout}, so teardown waits as long as the task definition asked. */
private final Map<String, Integer> stopTimeouts;
/**
@@ -57,12 +59,21 @@ public class EcsTaskHandle {
Map<String, Closeable> logStreamsByContainerId,
String firelensVolumeName, String networkInterfaceId, String region,
Map<String, Integer> stopTimeouts) {
+ this(taskArn, containerIds, logStreamsByContainerId, firelensVolumeName,
+ networkInterfaceId, region, stopTimeouts, Set.of());
+ }
+
+ public EcsTaskHandle(String taskArn, Map<String, String> containerIds,
+ Map<String, Closeable> logStreamsByContainerId,
+ String firelensVolumeName, String networkInterfaceId, String region,
+ Map<String, Integer> stopTimeouts, Set<String> taskVolumeNames) {
this.taskArn = taskArn;
this.containerIds = new LinkedHashMap<>(containerIds);
this.logStreamsByContainerId = new LinkedHashMap<>(logStreamsByContainerId);
this.firelensVolumeName = firelensVolumeName;
this.networkInterfaceId = networkInterfaceId;
this.region = region;
+ this.taskVolumeNames = new LinkedHashSet<>(taskVolumeNames);
this.stopTimeouts = new LinkedHashMap<>(stopTimeouts);
}
@@ -72,6 +83,15 @@ public class EcsTaskHandle {
public String getFirelensVolumeName() { return firelensVolumeName; }
public String getNetworkInterfaceId() { return networkInterfaceId; }
public String getRegion() { return region; }
+ public Set<String> getTaskVolumeNames() { return Set.copyOf(taskVolumeNames); }
+
+ public void recordTaskVolumeRemoved(String volumeName) {
+ taskVolumeNames.remove(volumeName);
+ }
+
+ public boolean hasTaskVolumes() {
+ return !taskVolumeNames.isEmpty();
+ }
/** The container's {@code stopTimeout} in seconds, or {@link #DEFAULT_STOP_TIMEOUT_SECONDS}. */
public int stopTimeoutFor(String containerName) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskLaunchException.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskLaunchException.java
new file mode 100644
index 0000000000..34b424cc8c
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskLaunchException.java
@@ -0,0 +1,20 @@
+package io.github.hectorvent.floci.services.ecs.container;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+
+/** A failed launch whose remaining resources still need teardown. */
+public class EcsTaskLaunchException extends AwsException {
+
+ private final EcsTaskHandle handle;
+
+ public EcsTaskLaunchException(Exception cause, EcsTaskHandle handle) {
+ super(cause instanceof AwsException aws ? aws.getErrorCode() : "ClientException",
+ cause.getMessage(), cause instanceof AwsException aws ? aws.getHttpStatus() : 400);
+ initCause(cause);
+ this.handle = handle;
+ }
+
+ public EcsTaskHandle getHandle() {
+ return handle;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/model/Volume.java b/src/main/java/io/github/hectorvent/floci/services/ecs/model/Volume.java
index 601dc7ca9e..778e47a2bb 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/model/Volume.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/model/Volume.java
@@ -5,14 +5,16 @@ import io.quarkus.runtime.annotations.RegisterForReflection;
import java.util.Map;
/**
- * A task-level volume in an ECS task definition. Two shapes are modelled:
+ * A task-level volume in an ECS task definition. Three shapes are modelled:
* <ul>
+ * <li>A volume without a host source path or external configuration supplies ephemeral
+ * storage shared by the containers in one task.</li>
* <li>EC2-launch-type {@code host} volume: {@code {"name": ..., "host": {"sourcePath": ...}}};
* {@code hostSourcePath} is the absolute path on the Docker host that the volume binds to.</li>
* <li>{@code efsVolumeConfiguration} volume: {@code {"name": ..., "efsVolumeConfiguration": {...}}};
* see {@link EfsVolumeConfiguration}.</li>
* </ul>
- * The two are mutually exclusive. A container references the volume by {@code name} via a
+ * The configurations are mutually exclusive. A container references the volume by {@code name} via a
* {@link MountPoint}.
*
* <p>{@code unparsed} keeps the volume members Floci backs with nothing, which is the rest of
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
index ff971a2f5f..e2ee42810e 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
@@ -29,6 +29,7 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.Set;
@@ -65,6 +66,22 @@ class ContainerLifecycleManagerVolumeTest {
dockerClient, imageCacheService, containerDetector, portAllocator, config);
}
+ @Test
+ void ensureVolumePersistsOwnershipLabelsAndTheirLegacyAliases() {
+ InspectVolumeCmd inspect = mock(InspectVolumeCmd.class);
+ when(dockerClient.inspectVolumeCmd("task-volume")).thenReturn(inspect);
+ when(inspect.exec()).thenThrow(new NotFoundException("missing"));
+ CreateVolumeCmd create = mock(CreateVolumeCmd.class, RETURNS_SELF);
+ when(dockerClient.createVolumeCmd()).thenReturn(create);
+ manager.ensureVolume("task-volume", Map.of("io.floci.service", "ecs",
+ "io.floci.owner", "alpha/4566", "io.floci.ecs.run", "run-id"));
+ verify(create).withLabels(argThat(labels -> "ecs".equals(labels.get("io.floci.service"))
+ && "alpha/4566".equals(labels.get("io.floci.owner"))
+ && "alpha/4566".equals(labels.get("floci_owner_port"))
+ && "run-id".equals(labels.get("io.floci.ecs.run"))
+ && "run-id".equals(labels.get("floci.ecs-run"))));
+ }
+
@Test
void stopAndRemoveStopsContainerBeforeClosingItsLogStream() {
StopContainerCmd stop = mock(StopContainerCmd.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandlerVolumesTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandlerVolumesTest.java
index c3a6a8d68f..d012bf9b20 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandlerVolumesTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandlerVolumesTest.java
@@ -13,6 +13,8 @@ import jakarta.ws.rs.core.Response;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.nio.file.Files;
@@ -49,6 +51,19 @@ class EcsJsonHandlerVolumesTest {
private EmulatorConfig config;
private EcsJsonHandler handler;
+ @ParameterizedTest
+ @ValueSource(strings = {"{}", "{\"fileSystemId\":\"\"}", "{\"fileSystemId\":\" \"}", "\"invalid\""})
+ void registerTaskDefinitionRejectsMalformedEfsVolume(String efs) throws Exception {
+ JsonNode request = objectMapper.readTree("""
+ {"family":"invalid-efs","containerDefinitions":[{"name":"app","image":"busybox:stable",
+ "mountPoints":[{"sourceVolume":"shared","containerPath":"/data"}]}],
+ "volumes":[{"name":"shared","efsVolumeConfiguration":%s}]}
+ """.formatted(efs));
+ AwsException error = assertThrows(AwsException.class,
+ () -> handler.handle("RegisterTaskDefinition", request, "us-east-1"));
+ assertEquals("InvalidParameterException", error.getErrorCode());
+ }
+
@BeforeEach
void setUp() {
objectMapper = new ObjectMapper();
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java
index 4327722b81..8a65628ec6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java
@@ -49,6 +49,63 @@ class EcsServiceRestartLeftoversTest {
private static final String TARGET_GROUP_ARN =
"arn:aws:elasticloadbalancing:us-east-1:000000000000:targetgroup/web/0123456789abcdef";
+ @Test
+ void pendingPreviousRunVolumesDoNotBlockRunTaskOrServiceTasksAndRetryOnTicks() {
+ SharedStorageFactory storage = new SharedStorageFactory();
+ persistServiceWithLoadBalancer(storage);
+ EcsContainerManager manager = mock(EcsContainerManager.class);
+ when(manager.removeLeftoverContainers()).thenReturn(true);
+ when(manager.removeLeftoverTaskVolumes()).thenReturn(false, false, true);
+ when(manager.getExitCodeIfStopped(anyString())).thenReturn(null);
+ when(manager.startTask(any(), any(), any(), anyString())).thenAnswer(invocation -> {
+ EcsTask task = invocation.getArgument(0);
+ return new EcsTaskHandle(task.getTaskArn(), Map.of("web", "docker-id"), Map.of());
+ });
+ EcsService restarted = service(storage, false, manager, mock(EcsLoadBalancerRegistrar.class), null);
+ restarted.releasePreviousRunLeftovers();
+ EcsTask task = restarted.runTask("app-cluster", "web", 1, LaunchType.EC2, null, null,
+ null, null, REGION).getFirst();
+ assertEquals("RUNNING", task.getLastStatus());
+ restarted.reconcileServices();
+ verify(manager, times(2)).startTask(any(), any(), any(), anyString());
+ verify(manager).removeLeftoverTaskVolumes();
+
+ restarted.reconcile();
+ restarted.reconcile();
+ restarted.reconcile();
+ verify(manager, times(3)).removeLeftoverTaskVolumes();
+ verify(manager).removeLeftoverContainers();
+ verify(manager, times(2)).startTask(any(), any(), any(), anyString());
+ }
+
+ @Test
+ void volumeCleanupWaitsForContainersAndRetriesWithoutAnyTaskLaunch() {
+ EcsContainerManager manager = mock(EcsContainerManager.class);
+ when(manager.removeLeftoverContainers()).thenReturn(false, true);
+ when(manager.removeLeftoverTaskVolumes()).thenReturn(false, true);
+ when(manager.getExitCodeIfStopped(anyString())).thenReturn(null);
+ when(manager.startTask(any(), any(), any(), anyString())).thenAnswer(invocation -> {
+ EcsTask task = invocation.getArgument(0);
+ return new EcsTaskHandle(task.getTaskArn(), Map.of("web", "docker-id"), Map.of());
+ });
+ EcsService restarted = service(new SharedStorageFactory(), false, manager,
+ mock(EcsLoadBalancerRegistrar.class), null);
+ restarted.releasePreviousRunLeftovers();
+ restarted.reconcile();
+ verify(manager, never()).removeLeftoverTaskVolumes();
+ ContainerDefinition definition = new ContainerDefinition();
+ definition.setName("web");
+ definition.setImage("busybox:stable");
+ restarted.registerTaskDefinition("web", List.of(definition), NetworkMode.bridge, null, null,
+ null, null, List.of(), REGION);
+ restarted.runTask(null, "web", 1, LaunchType.EC2, null, null, null, null, REGION);
+ restarted.reconcile();
+ restarted.reconcile();
+ restarted.reconcile();
+ verify(manager, times(2)).removeLeftoverTaskVolumes();
+ verify(manager, times(2)).removeLeftoverContainers();
+ }
+
@Test
void releasesTheRegistrationsThePreviousRunRecorded() {
EcsLoadBalancerRegistrar lbRegistrar = mock(EcsLoadBalancerRegistrar.class);
@@ -138,6 +195,7 @@ class EcsServiceRestartLeftoversTest {
mock(EcsLoadBalancerRegistrar.class), null).releasePreviousRunLeftovers();
verify(containerManager, never()).removeLeftoverContainers();
+ verify(containerManager, never()).removeLeftoverTaskVolumes();
}
@Test
@@ -158,6 +216,7 @@ class EcsServiceRestartLeftoversTest {
restarted.runTask("app-cluster", "web", 1, LaunchType.EC2, null, null, null, null, REGION);
verify(containerManager, never()).removeLeftoverContainers();
+ verify(containerManager, never()).removeLeftoverTaskVolumes();
verify(containerManager).startTask(any(), any(), any(), anyString());
verify(lbRegistrar).releaseRecordedTargets();
verify(discoveryRegistrar).releaseRecordedInstances();
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
index 3f41617aaf..0baa89c38a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
@@ -9,16 +9,20 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;
+import io.github.hectorvent.floci.services.ecs.container.EcsTaskLaunchException;
import io.github.hectorvent.floci.services.ecs.model.Container;
import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.LaunchType;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.Closeable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -33,6 +37,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -47,6 +52,127 @@ class EcsServiceTeardownTest {
private static final String REGION = "us-east-1";
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void stoppedTaskKeepsItsHandleUntilEphemeralVolumeCleanupSucceeds(boolean exitsNaturally) {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(false);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ EcsContainerManager manager = mock(EcsContainerManager.class);
+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"), Map.of(),
+ null, null, REGION, Map.of(), Set.of("task-volume"));
+ when(manager.startTask(any(), any(), any(), anyString())).thenReturn(handle);
+ when(manager.getExitCodeIfStopped("docker-id")).thenReturn(0);
+ AtomicInteger attempts = new AtomicInteger();
+ doAnswer(ignored -> {
+ attempts.incrementAndGet();
+ handle.recordContainerRemoved("app");
+ return null;
+ }).when(manager).cleanupStoppedTask(handle);
+ when(manager.stopTaskAndCollectExitCodes(handle)).thenAnswer(ignored -> {
+ handle.recordContainerRemoved("app");
+ if (attempts.incrementAndGet() > 1) {
+ handle.recordTaskVolumeRemoved("task-volume");
+ }
+ return Map.of("app", 0);
+ });
+ EcsService service = new EcsService(new RegionResolver(REGION, "000000000000"), manager, config,
+ mock(EcsLoadBalancerRegistrar.class), new SingleUseStorageFactory(), null);
+ service.initializeStorage();
+ ContainerDefinition definition = new ContainerDefinition();
+ definition.setName("app");
+ definition.setImage("app:latest");
+ service.registerTaskDefinition("volume-retry", List.of(definition), null, null, null,
+ null, null, List.of(), REGION);
+ String taskArn = service.runTask(null, "volume-retry", 1, LaunchType.FARGATE, null, null,
+ List.of(), null, REGION).getFirst().getTaskArn();
+
+ if (exitsNaturally) {
+ service.reconcile();
+ } else {
+ service.stopTask(null, taskArn, null, REGION);
+ }
+ assertEquals("STOPPED", service.describeTasks(null, List.of(taskArn), REGION).getFirst().getLastStatus());
+ assertTrue(handle.hasTaskVolumes());
+
+ service.reconcile();
+ assertFalse(handle.hasTaskVolumes());
+ service.reconcile();
+ assertEquals(2, attempts.get(), "cleanup stops retrying after the volume is removed");
+ }
+
+ @Test
+ void failedLaunchCleanupIsRetriedByStoppedTaskReconciliation() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(false);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ EcsContainerManager manager = mock(EcsContainerManager.class);
+ EcsTaskHandle partial = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"), Map.of(),
+ null, null, REGION, Map.of(), Set.of("task-volume"));
+ when(manager.startTask(any(), any(), any(), anyString()))
+ .thenThrow(new EcsTaskLaunchException(new IllegalStateException("launch failed"), partial));
+ when(manager.stopTaskAndCollectExitCodes(partial)).thenAnswer(ignored -> {
+ partial.recordContainerRemoved("app");
+ partial.recordTaskVolumeRemoved("task-volume");
+ return Map.of("app", 137);
+ });
+ EcsService service = new EcsService(new RegionResolver(REGION, "000000000000"), manager, config,
+ mock(EcsLoadBalancerRegistrar.class), new SingleUseStorageFactory(), null);
+ service.initializeStorage();
+ ContainerDefinition definition = new ContainerDefinition();
+ definition.setName("app");
+ definition.setImage("busybox:stable");
+ service.registerTaskDefinition("failed-volume-launch", List.of(definition), null, null, null,
+ null, null, List.of(), REGION);
+ EcsTask task = service.runTask(null, "failed-volume-launch", 1, LaunchType.FARGATE, null, null,
+ List.of(), null, REGION).getFirst();
+ assertEquals("STOPPED", task.getLastStatus());
+ assertEquals("Failed to start: launch failed", task.getStoppedReason());
+ assertTrue(partial.hasTaskVolumes());
+ service.reconcile();
+ assertFalse(partial.hasTaskVolumes());
+ service.reconcile();
+ verify(manager, times(1)).stopTaskAndCollectExitCodes(partial);
+ service.stopManagedContainers();
+ }
+
+ @Test
+ void shutdownRetriesVolumeRemovalAndRetainsPendingCleanupAfterReset() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(false);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ EcsContainerManager manager = mock(EcsContainerManager.class);
+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"), Map.of(),
+ null, null, REGION, Map.of(), Set.of("task-volume"));
+ when(manager.startTask(any(), any(), any(), anyString())).thenReturn(handle);
+ doAnswer(ignored -> {
+ handle.recordContainerRemoved("app");
+ return null;
+ }).when(manager).stopTask(handle);
+ when(manager.stopTaskAndCollectExitCodes(handle)).thenAnswer(ignored -> {
+ handle.recordTaskVolumeRemoved("task-volume");
+ return Map.of("app", 0);
+ });
+ EcsService service = new EcsService(new RegionResolver(REGION, "000000000000"), manager, config,
+ mock(EcsLoadBalancerRegistrar.class), new SingleUseStorageFactory(), null);
+ service.initializeStorage();
+ ContainerDefinition definition = new ContainerDefinition();
+ definition.setName("app");
+ definition.setImage("busybox:stable");
+ service.registerTaskDefinition("shutdown-volume", List.of(definition), null, null, null,
+ null, null, List.of(), REGION);
+ service.runTask(null, "shutdown-volume", 1, LaunchType.FARGATE, null, null,
+ List.of(), null, REGION);
+ service.stopManagedContainers();
+ verify(manager, times(2)).cleanupStoppedTask(handle);
+ assertTrue(handle.hasTaskVolumes());
+ service.clear();
+ service.afterReset();
+ service.reconcile();
+ assertFalse(handle.hasTaskVolumes());
+ service.stopManagedContainers();
+ }
+
@Test
void startupTransitionWaitsForTheTaskLock() throws Exception {
EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
@@ -175,9 +301,12 @@ class EcsServiceTeardownTest {
task.setContainers(List.of(container));
return original;
});
+ when(containerManager.taskVolumeNames(any(), any())).thenReturn(Set.of("recovered-volume"));
when(containerManager.stopTaskAndCollectExitCodes(any())).thenAnswer(invocation -> {
EcsTaskHandle stoppingHandle = invocation.getArgument(0);
+ assertEquals(Set.of("recovered-volume"), stoppingHandle.getTaskVolumeNames());
stoppingHandle.recordContainerRemoved("app");
+ stoppingHandle.recordTaskVolumeRemoved("recovered-volume");
return Map.of("app", 0);
});
@@ -200,6 +329,7 @@ class EcsServiceTeardownTest {
verify(containerManager).stopTaskAndCollectExitCodes(argThat(
handle -> "docker-id".equals(handle.getContainerIds().get("app"))));
+ verify(containerManager).taskVolumeNames(any(), any());
assertEquals("STOPPED", service.describeTasks(null, List.of(taskArn), REGION).getFirst().getLastStatus());
service.stopManagedContainers();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEphemeralVolumesDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEphemeralVolumesDockerIntegrationTest.java
new file mode 100644
index 0000000000..1c72e96506
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEphemeralVolumesDockerIntegrationTest.java
@@ -0,0 +1,163 @@
+package io.github.hectorvent.floci.services.ecs.container;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.InspectContainerResponse.Mount;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.core.command.WaitContainerResultCallback;
+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.EcsTask;
+import io.github.hectorvent.floci.services.ecs.model.MountPoint;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.ecs.model.Volume;
+import io.github.hectorvent.floci.testing.TestImages;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@QuarkusTest
+@TestProfile(EcsContainerManagerEphemeralVolumesDockerIntegrationTest.VolumeProfile.class)
+class EcsContainerManagerEphemeralVolumesDockerIntegrationTest {
+
+ @Inject
+ EcsContainerManager manager;
+
+ @Inject
+ DockerClient dockerClient;
+
+ private final List<EcsTaskHandle> handles = new ArrayList<>();
+
+ public static class VolumeProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of("floci.docker.resource-namespace", "ecs-ephemeral-volumes-test");
+ }
+ }
+
+ @BeforeEach
+ void requireDocker() {
+ try {
+ dockerClient.pingCmd().exec();
+ } catch (Exception unavailable) {
+ Assumptions.abort("Docker daemon must be available for ephemeral volume integration tests");
+ }
+ }
+
+ @AfterEach
+ void cleanUpTasks() {
+ handles.forEach(manager::stopTask);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void containersShareEphemeralStorageWithinATaskAndRemoveItOnTeardown(boolean emptyHost) throws Exception {
+ EcsTaskHandle first = startTask("first-task", emptyHost, 300);
+ assertEquals(0, waitFor(first, "reader"), "the reader must see the writer's file through the shared mount");
+ Mount firstWriter = dataMount(first, "writer");
+ Mount firstReader = dataMount(first, "reader");
+ assertEquals(firstWriter.getName(), firstReader.getName());
+ assertEquals(Boolean.TRUE, firstWriter.getRW());
+ assertEquals(Boolean.FALSE, firstReader.getRW());
+
+ EcsTaskHandle second = startTask("second-task", emptyHost, 1);
+ assertEquals(0, waitFor(second, "reader"), "a second task must get its own empty volume");
+ Mount secondWriter = dataMount(second, "writer");
+ assertNotEquals(firstWriter.getName(), secondWriter.getName());
+ assertEquals(0, waitFor(second, "writer"));
+
+ manager.stopTask(first);
+ handles.remove(first);
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectVolumeCmd(firstWriter.getName()).exec());
+ dockerClient.inspectVolumeCmd(secondWriter.getName()).exec();
+
+ manager.cleanupStoppedTask(second);
+ handles.remove(second);
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectVolumeCmd(secondWriter.getName()).exec());
+ }
+
+ @Test
+ void nextRunRemovesOrphanVolumesAndContainersWithMatchingOwnership() {
+ EcsTaskHandle task = startTask("orphan-task", false, 300);
+ String volumeName = task.getTaskVolumeNames().iterator().next();
+ Map<String, String> labels = dockerClient.inspectVolumeCmd(volumeName).exec().getLabels();
+ assertEquals("ecs", labels.get(ContainerStorageHelper.SERVICE_LABEL));
+ assertEquals("ecs-ephemeral-volumes-test/4566", labels.get(ContainerStorageHelper.OWNER_LABEL));
+ assertTrue(labels.containsKey(EcsContainerManager.RUN_LABEL));
+ try {
+ String nextRun = "next-run-" + UUID.randomUUID();
+ assertTrue(manager.removeLeftoverContainers(nextRun));
+ dockerClient.inspectVolumeCmd(volumeName).exec();
+ assertTrue(manager.removeLeftoverTaskVolumes(nextRun));
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectVolumeCmd(volumeName).exec());
+ for (String containerId : task.getContainerIds().values()) {
+ assertThrows(NotFoundException.class, () -> dockerClient.inspectContainerCmd(containerId).exec());
+ }
+ } finally {
+ manager.stopTask(task);
+ handles.remove(task);
+ }
+ }
+
+ private EcsTaskHandle startTask(String marker, boolean emptyHost, int writerSleep) {
+ ContainerDefinition writer = definition("writer", false);
+ writer.setCommand(List.of("sh", "-c",
+ "test ! -e /data/marker || exit 41; echo " + marker + " > /data/marker; exec sleep " + writerSleep));
+ ContainerDefinition reader = definition("reader", true);
+ reader.setCommand(List.of("sh", "-c", """
+ for attempt in 1 2 3 4 5; do
+ if [ "$(cat /data/marker 2>/dev/null)" = "%s" ]; then exit 0; fi
+ sleep 1
+ done
+ exit 42
+ """.formatted(marker)));
+ TaskDefinition taskDefinition = new TaskDefinition();
+ taskDefinition.setFamily("ephemeral-volume-test");
+ taskDefinition.setContainerDefinitions(List.of(writer, reader));
+ taskDefinition.setVolumes(List.of(new Volume("shared", null, null,
+ emptyHost ? Map.of("host", Map.of()) : null)));
+ EcsTask task = new EcsTask();
+ task.setTaskArn("arn:aws:ecs:us-east-1:000000000000:task/ephemeral-volume-test/" + UUID.randomUUID());
+ EcsTaskHandle handle = manager.startTask(task, taskDefinition, List.of(), "us-east-1");
+ handles.add(handle);
+ return handle;
+ }
+
+ private static ContainerDefinition definition(String name, boolean readOnly) {
+ ContainerDefinition definition = new ContainerDefinition();
+ definition.setName(name);
+ definition.setImage(TestImages.BUSYBOX);
+ definition.setStopTimeout(1);
+ definition.setMountPoints(List.of(new MountPoint("shared", "/data", readOnly)));
+ return definition;
+ }
+
+ private Mount dataMount(EcsTaskHandle handle, String container) {
+ return dockerClient.inspectContainerCmd(handle.getContainerIds().get(container)).exec().getMounts().stream()
+ .filter(mount -> "/data".equals(mount.getDestination().getPath())).findFirst().orElseThrow();
+ }
+
+ private Integer waitFor(EcsTaskHandle handle, String container) throws Exception {
+ try (WaitContainerResultCallback callback = new WaitContainerResultCallback()) {
+ return dockerClient.waitContainerCmd(handle.getContainerIds().get(container)).exec(callback)
+ .awaitStatusCode(20, TimeUnit.SECONDS);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java
index 9b3cf84c66..9367fa96e1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java
@@ -1,7 +1,10 @@
package io.github.hectorvent.floci.services.ecs.container;
import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
import com.github.dockerjava.api.command.ListContainersCmd;
+import com.github.dockerjava.api.command.ListVolumesCmd;
+import com.github.dockerjava.api.command.ListVolumesResponse;
import com.github.dockerjava.api.command.RemoveContainerCmd;
import com.github.dockerjava.api.command.StopContainerCmd;
import com.github.dockerjava.api.model.Container;
@@ -16,6 +19,7 @@ import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.ssm.SsmService;
+import io.github.hectorvent.floci.testutil.LogCapture;
import org.junit.jupiter.api.Test;
import java.io.Closeable;
@@ -23,6 +27,8 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -33,6 +39,7 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.RETURNS_SELF;
import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -259,10 +266,126 @@ class EcsContainerManagerLifecycleTest {
verify(lifecycleManager, never()).removeIfExistsStrict(anyString());
}
+ @Test
+ void previousRunVolumesAreRemovedAfterContainersAndFailedRemovalIsRetried() {
+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);
+ stubLeftoverListing(lifecycle, Map.of(), Map.of());
+ InspectVolumeResponse old = leftoverVolume("old", "alpha/4566", "previous");
+ InspectVolumeResponse current = leftoverVolume("current", "alpha/4566", "current");
+ InspectVolumeResponse foreign = leftoverVolume("foreign", "beta/4566", "previous");
+ InspectVolumeResponse unowned = leftoverVolume("unowned", "alpha/4566", null);
+ InspectVolumeResponse conflicting = leftoverVolume("conflicting", "alpha/4566", "previous");
+ when(conflicting.getLabels()).thenReturn(Map.of("io.floci.service", "ecs",
+ "io.floci.owner", "alpha/4566", "floci_owner_port", "beta/4566", "io.floci.ecs.run", "previous"));
+ stubVolumeListing(lifecycle.getDockerClient(), List.of(old, current, foreign, unowned, conflicting));
+ doThrow(new IllegalStateException("busy")).doNothing().when(lifecycle).removeVolumeStrict("old");
+ EcsContainerManager manager = manager(lifecycle, namespacedConfig());
+
+ assertTrue(manager.removeLeftoverContainers("current"));
+ assertFalse(manager.removeLeftoverTaskVolumes("current"));
+ assertTrue(manager.removeLeftoverTaskVolumes("current"));
+ verify(lifecycle, times(2)).removeVolumeStrict("old");
+ verify(lifecycle, never()).removeVolumeStrict("current");
+ verify(lifecycle, never()).removeVolumeStrict("foreign");
+ verify(lifecycle, never()).removeVolumeStrict("unowned");
+ verify(lifecycle, never()).removeVolumeStrict("conflicting");
+ }
+
+ @Test
+ void previousRunVolumeSweepWaitsForContainerRemoval() {
+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);
+ stubLeftoverListing(lifecycle, Map.of("old-container", leftover("old-container",
+ Map.of("io.floci.service", "ecs", "io.floci.owner", "alpha/4566", "io.floci.ecs.run", "previous"))),
+ Map.of());
+ doThrow(new IllegalStateException("busy")).when(lifecycle).removeIfExistsStrict("old-container");
+ assertFalse(manager(lifecycle, namespacedConfig()).removeLeftoverContainers("current"));
+ verify(lifecycle.getDockerClient(), never()).listVolumesCmd();
+ }
+
+ @Test
+ void previousRunVolumeSweepRetriesWhenInventoryCannotBeRead() {
+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);
+ stubLeftoverListing(lifecycle, Map.of(), Map.of());
+ when(lifecycle.getDockerClient().listVolumesCmd()).thenThrow(new IllegalStateException("offline"));
+ EcsContainerManager manager = manager(lifecycle, namespacedConfig());
+ assertTrue(manager.removeLeftoverContainers("current"));
+ assertFalse(manager.removeLeftoverTaskVolumes("current"));
+ verify(lifecycle, never()).removeVolume(anyString());
+ }
+
+ @Test
+ void volumeRemovalWarnsOncePerVolumeAndResetsAfterSuccessfulRemoval() {
+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);
+ stubLeftoverListing(lifecycle, Map.of(), Map.of());
+ stubVolumeListing(lifecycle.getDockerClient(), List.of(
+ leftoverVolume("alpha", "alpha/4566", "previous"),
+ leftoverVolume("beta", "alpha/4566", "previous")));
+ doThrow(new IllegalStateException("busy")).when(lifecycle).removeVolumeStrict(anyString());
+ EcsContainerManager manager = manager(lifecycle, namespacedConfig());
+ List<LogRecord> records = LogCapture.capture(EcsContainerManager.class, () -> {
+ assertFalse(manager.removeLeftoverTaskVolumes("current"));
+ assertFalse(manager.removeLeftoverTaskVolumes("current"));
+ doNothing().when(lifecycle).removeVolumeStrict("alpha");
+ assertFalse(manager.removeLeftoverTaskVolumes("current"));
+ doThrow(new IllegalStateException("busy again")).when(lifecycle).removeVolumeStrict("alpha");
+ assertFalse(manager.removeLeftoverTaskVolumes("current"));
+ });
+ assertEquals(List.of(Level.WARNING.intValue(), Level.WARNING.intValue(),
+ Level.FINE.intValue(), Level.FINE.intValue(), Level.FINE.intValue(),
+ Level.WARNING.intValue(), Level.FINE.intValue()),
+ records.stream().map(record -> record.getLevel().intValue()).toList());
+ verify(lifecycle, times(4)).removeVolumeStrict("alpha");
+ verify(lifecycle, times(4)).removeVolumeStrict("beta");
+ verify(lifecycle, never()).removeVolume(anyString());
+ }
+
+ @Test
+ void volumeListingWarnsOnceAndResetsAfterSuccessfulListing() {
+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);
+ stubLeftoverListing(lifecycle, Map.of(), Map.of());
+ ListVolumesCmd command = lifecycle.getDockerClient().listVolumesCmd();
+ ListVolumesResponse empty = mock(ListVolumesResponse.class);
+ when(empty.getVolumes()).thenReturn(List.of());
+ when(command.exec()).thenThrow(new IllegalStateException("offline"))
+ .thenThrow(new IllegalStateException("offline"))
+ .thenReturn(empty, empty).thenThrow(new IllegalStateException("offline again"));
+ EcsContainerManager manager = manager(lifecycle, namespacedConfig());
+ List<LogRecord> records = LogCapture.capture(EcsContainerManager.class, () -> {
+ assertFalse(manager.removeLeftoverTaskVolumes("current"));
+ assertFalse(manager.removeLeftoverTaskVolumes("current"));
+ assertTrue(manager.removeLeftoverTaskVolumes("current"));
+ assertFalse(manager.removeLeftoverTaskVolumes("current"));
+ });
+ assertEquals(List.of(Level.WARNING.intValue(), Level.FINE.intValue(), Level.WARNING.intValue()),
+ records.stream().map(record -> record.getLevel().intValue()).toList());
+ }
+
+ private static InspectVolumeResponse leftoverVolume(String name, String owner, String run) {
+ InspectVolumeResponse volume = mock(InspectVolumeResponse.class);
+ when(volume.getName()).thenReturn(name);
+ Map<String, String> labels = new LinkedHashMap<>();
+ labels.put("io.floci.service", "ecs");
+ labels.put("io.floci.owner", owner);
+ if (run != null) {
+ labels.put("io.floci.ecs.run", run);
+ }
+ when(volume.getLabels()).thenReturn(labels);
+ return volume;
+ }
+
+ private static void stubVolumeListing(DockerClient docker, List<InspectVolumeResponse> volumes) {
+ ListVolumesCmd command = mock(ListVolumesCmd.class, RETURNS_SELF);
+ ListVolumesResponse response = mock(ListVolumesResponse.class);
+ when(docker.listVolumesCmd()).thenReturn(command);
+ when(command.exec()).thenReturn(response);
+ when(response.getVolumes()).thenReturn(volumes);
+ }
+
private static void stubLeftoverListing(ContainerLifecycleManager lifecycleManager,
Map<String, Container> newKeys, Map<String, Container> legacyKeys) {
DockerClient dockerClient = mock(DockerClient.class);
when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ stubVolumeListing(dockerClient, List.of());
ListContainersCmd listCmd = mock(ListContainersCmd.class, RETURNS_SELF);
when(dockerClient.listContainersCmd()).thenReturn(listCmd);
ListContainersCmd byNewKeys = mock(ListContainersCmd.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java
index ea13049f97..a7eba5b702 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.ecs.container;
+import com.github.dockerjava.api.DockerClient;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
@@ -22,21 +23,33 @@ import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.ssm.SsmService;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -61,6 +74,174 @@ class EcsContainerManagerVolumesTest {
private EcrRegistryManager ecrRegistryManager;
private EcsContainerManager manager;
+ private static Stream<Volume> ephemeralVolumes() {
+ return Stream.of(new Volume("shared", null),
+ new Volume("shared", null, null, Map.of("host", Map.of())),
+ new Volume("shared", "", null, Map.of("host", Map.of("sourcePath", ""))));
+ }
+
+ @ParameterizedTest
+ @MethodSource("ephemeralVolumes")
+ void ephemeralVolumeIsCreatedOnceAndSharedByContainersWithTheirAccessModes(Volume volume) {
+ EcsTaskHandle handle = manager.startTask(task("000000000000", "us-east-1"),
+ ephemeralDefinition(volume), List.of(), "us-east-1");
+
+ ArgumentCaptor<String> volumeName = ArgumentCaptor.forClass(String.class);
+ verify(builder).withNamedVolume(volumeName.capture(), eq("/data"), eq(false));
+ verify(builder).withNamedVolume(volumeName.getValue(), "/data", true);
+ verify(lifecycleManager).ensureVolume(eq(volumeName.getValue()), any());
+ verify(lifecycleManager, times(1)).ensureVolume(anyString(), any());
+ assertEquals(List.of(volumeName.getValue()), handle.getTaskVolumeNames().stream().toList());
+ verify(builder, never()).withBind(any(), any());
+ }
+
+ @Test
+ void taskVolumeIdentityIncludesAccountAndRegion() {
+ TaskDefinition definition = ephemeralDefinition(new Volume("shared", null));
+ manager.startTask(task("111111111111", "us-east-1"), definition, List.of(), "us-east-1");
+ manager.startTask(task("222222222222", "us-east-1"), definition, List.of(), "us-east-1");
+ manager.startTask(task("111111111111", "eu-west-1"), definition, List.of(), "eu-west-1");
+
+ ArgumentCaptor<String> names = ArgumentCaptor.forClass(String.class);
+ verify(lifecycleManager, times(3)).ensureVolume(names.capture(), any());
+ assertEquals(3, names.getAllValues().stream().distinct().count());
+ }
+
+ @Test
+ void unmountedAndUnsupportedVolumesAreNotCreatedAsEphemeralStorage() {
+ ContainerDefinition app = new ContainerDefinition();
+ app.setName("app");
+ app.setImage("app:latest");
+ app.setMountPoints(List.of(new MountPoint("external", "/external", false),
+ new MountPoint("ebs", "/ebs", false)));
+ TaskDefinition definition = new TaskDefinition();
+ definition.setFamily("ephemeral-test");
+ definition.setContainerDefinitions(List.of(app));
+ definition.setVolumes(List.of(new Volume("unused", null),
+ new Volume("external", null, null, Map.of("dockerVolumeConfiguration", Map.of("scope", "shared"))),
+ new Volume("ebs", null, null, Map.of("configuredAtLaunch", true))));
+
+ manager.startTask(task("000000000000", "us-east-1"), definition, List.of(), "us-east-1");
+
+ verify(lifecycleManager, never()).ensureVolume(anyString(), any());
+ verify(builder, never()).withNamedVolume(any(), any(), anyBoolean());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void failedLaunchRemovesItsEphemeralVolume(boolean firstContainerStarted) {
+ if (firstContainerStarted) {
+ when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()))
+ .thenThrow(new IllegalStateException("launch failed"));
+ } else {
+ when(lifecycleManager.createAndStart(any())).thenThrow(new IllegalStateException("launch failed"));
+ }
+ assertThrows(IllegalStateException.class, () -> manager.startTask(task("000000000000", "us-east-1"),
+ ephemeralDefinition(new Volume("shared", null)), List.of(), "us-east-1"));
+
+ ArgumentCaptor<String> name = ArgumentCaptor.forClass(String.class);
+ verify(lifecycleManager).ensureVolume(name.capture(), any());
+ verify(lifecycleManager).removeVolume(name.getValue());
+ }
+
+ @Test
+ void volumeRemovalIsRetriedAfterContainersAreGone() {
+ when(lifecycleManager.getDockerClient()).thenReturn(mock(DockerClient.class, RETURNS_DEEP_STUBS));
+ EcsContainerManager stopping = spy(manager);
+ doReturn(0).when(stopping).getExitCodeIfStopped(anyString());
+ doReturn(null).when(stopping).getFinishedAtIfStopped(anyString());
+ EcsTaskHandle handle = stopping.startTask(task("000000000000", "us-east-1"),
+ ephemeralDefinition(new Volume("shared", null)), List.of(), "us-east-1");
+ String volumeName = handle.getTaskVolumeNames().iterator().next();
+ when(lifecycleManager.removeVolume(volumeName)).thenReturn(false, true);
+
+ stopping.stopTask(handle);
+ assertTrue(handle.allContainersRemoved());
+ assertTrue(handle.hasTaskVolumes());
+ stopping.stopTask(handle);
+ assertFalse(handle.hasTaskVolumes());
+ verify(lifecycleManager, times(2)).removeVolume(volumeName);
+ }
+
+ @Test
+ void volumeIsKeptUntilEveryContainerHasBeenRemoved() {
+ EcsTaskHandle handle = manager.startTask(task("000000000000", "us-east-1"),
+ ephemeralDefinition(new Volume("shared", null)), List.of(), "us-east-1");
+ String name = handle.getTaskVolumeNames().iterator().next();
+ doThrow(new IllegalStateException("remove failed")).when(lifecycleManager)
+ .stopAndRemoveStrict("docker-id", null);
+
+ manager.cleanupStoppedTask(handle);
+ assertTrue(handle.hasTaskVolumes());
+ verify(lifecycleManager, never()).removeVolume(name);
+
+ doNothing().when(lifecycleManager).stopAndRemoveStrict("docker-id", null);
+ when(lifecycleManager.removeVolume(name)).thenReturn(true);
+ manager.cleanupStoppedTask(handle);
+ assertFalse(handle.hasTaskVolumes());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void failedLaunchPreservesResourcesWhenCleanupFails(boolean containerRemovalFails) {
+ when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()))
+ .thenThrow(new IllegalStateException("launch failed"));
+ if (containerRemovalFails) {
+ doThrow(new IllegalStateException("remove failed")).when(lifecycleManager)
+ .stopAndRemoveStrict("docker-id", null);
+ } else {
+ when(lifecycleManager.removeVolume(anyString())).thenReturn(false);
+ }
+ EcsTaskLaunchException failure = assertThrows(EcsTaskLaunchException.class,
+ () -> manager.startTask(task("000000000000", "us-east-1"),
+ ephemeralDefinition(new Volume("shared", null)), List.of(), "us-east-1"));
+ EcsTaskHandle handle = failure.getHandle();
+ assertTrue(handle.hasTaskVolumes());
+ assertEquals(!containerRemovalFails, handle.allContainersRemoved());
+ assertEquals("launch failed", failure.getMessage());
+
+ doNothing().when(lifecycleManager).stopAndRemoveStrict("docker-id", null);
+ when(lifecycleManager.removeVolume(anyString())).thenReturn(true);
+ manager.cleanupStoppedTask(handle);
+ assertTrue(handle.allContainersRemoved());
+ assertFalse(handle.hasTaskVolumes());
+ }
+
+ @Test
+ void recoveredVolumeNamesMatchLaunchAndExcludeUnusedAndExternalVolumes() {
+ EcsTask task = task("000000000000", "us-east-1");
+ TaskDefinition definition = ephemeralDefinition(new Volume("shared", null));
+ EcsTaskHandle handle = manager.startTask(task, definition, List.of(), "us-east-1");
+ assertEquals(handle.getTaskVolumeNames(), manager.taskVolumeNames(task, definition));
+ definition.setVolumes(List.of(new Volume("shared", "/host/path")));
+ assertTrue(manager.taskVolumeNames(task, definition).isEmpty());
+ definition.setVolumes(List.of(new Volume("shared", null,
+ new EfsVolumeConfiguration("fs-test", null, null, null, null, null))));
+ assertTrue(manager.taskVolumeNames(task, definition).isEmpty());
+ }
+
+ private static TaskDefinition ephemeralDefinition(Volume volume) {
+ ContainerDefinition writer = new ContainerDefinition();
+ writer.setName("writer");
+ writer.setImage("app:latest");
+ writer.setMountPoints(List.of(new MountPoint("shared", "/data", false)));
+ ContainerDefinition reader = new ContainerDefinition();
+ reader.setName("reader");
+ reader.setImage("app:latest");
+ reader.setMountPoints(List.of(new MountPoint("shared", "/data", true)));
+ TaskDefinition definition = new TaskDefinition();
+ definition.setFamily("ephemeral-test");
+ definition.setContainerDefinitions(List.of(writer, reader));
+ definition.setVolumes(List.of(volume, new Volume("unused", null)));
+ return definition;
+ }
+
+ private static EcsTask task(String account, String region) {
+ EcsTask task = new EcsTask();
+ task.setTaskArn("arn:aws:ecs:" + region + ":" + account + ":task/test-cluster/same-task-id");
+ return task;
+ }
+
@BeforeEach
void setUp() {
builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
@@ -69,6 +250,7 @@ class EcsContainerManagerVolumesTest {
when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.removeVolume(anyString())).thenReturn(true);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any()))