Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): mount and clean up ephemeral task volumes (#5511)
Mohith760c38ef99d2
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsEphemeralVolumeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsEphemeralVolumeTest.javanew file mode 100644index 0000000000..2bcfab9f52--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsEphemeralVolumeTest.java@@ -0,0 +1,112 @@+package com.floci.test;++import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import software.amazon.awssdk.core.waiters.WaiterOverrideConfiguration;+import software.amazon.awssdk.services.ec2.Ec2Client;+import software.amazon.awssdk.services.ecs.EcsClient;+import software.amazon.awssdk.services.ecs.model.Compatibility;+import software.amazon.awssdk.services.ecs.model.ContainerDefinition;+import software.amazon.awssdk.services.ecs.model.DescribeTasksRequest;+import software.amazon.awssdk.services.ecs.model.EFSVolumeConfiguration;+import software.amazon.awssdk.services.ecs.model.HostVolumeProperties;+import software.amazon.awssdk.services.ecs.model.InvalidParameterException;+import software.amazon.awssdk.services.ecs.model.LaunchType;+import software.amazon.awssdk.services.ecs.model.MountPoint;+import software.amazon.awssdk.services.ecs.model.NetworkMode;+import software.amazon.awssdk.services.ecs.model.RunTaskResponse;+import software.amazon.awssdk.services.ecs.model.Task;+import software.amazon.awssdk.services.ecs.model.TaskDefinition;+import software.amazon.awssdk.services.ecs.model.Volume;++import java.time.Duration;+import java.util.UUID;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;+import static org.junit.jupiter.api.Assumptions.assumeTrue;++@DisplayName("ECS ephemeral volumes shared within a task")+class EcsEphemeralVolumeTest {++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void efsVolumeRequiresFileSystemId(boolean blankId) {+ String name = "sdk-invalid-efs-" + UUID.randomUUID().toString().substring(0, 8);+ EFSVolumeConfiguration.Builder efs = EFSVolumeConfiguration.builder();+ if (blankId) {+ efs.fileSystemId("");+ }+ try (EcsClient ecs = TestFixtures.ecsClient()) {+ assertThatThrownBy(() -> ecs.registerTaskDefinition(request -> request.family(name)+ .containerDefinitions(container("reader", true, "true"))+ .volumes(Volume.builder().name("shared").efsVolumeConfiguration(efs.build()).build())))+ .isInstanceOf(InvalidParameterException.class);+ }+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void nameOnlyAndEmptyHostVolumesShareFilesBetweenContainers(boolean emptyHost) {+ String name = "sdk-ephemeral-volume-" + UUID.randomUUID().toString().substring(0, 8);+ Volume.Builder volume = Volume.builder().name("shared");+ if (emptyHost) {+ volume.host(HostVolumeProperties.builder().build());+ }+ try (EcsClient ecs = TestFixtures.ecsClient(); Ec2Client ec2 = TestFixtures.ec2Client()) {+ String subnet = ec2.describeSubnets().subnets().get(0).subnetId();+ ecs.createCluster(request -> request.clusterName(name));+ TaskDefinition definition = ecs.registerTaskDefinition(request -> request.family(name)+ .networkMode(NetworkMode.AWSVPC).requiresCompatibilities(Compatibility.FARGATE)+ .cpu("256").memory("512").volumes(volume.build())+ .containerDefinitions(+ container("writer", false, "echo shared > /data/marker"),+ container("reader", true, """+ for attempt in 1 2 3 4 5; do+ if [ "$(cat /data/marker 2>/dev/null)" = shared ]; then exit 0; fi+ sleep 1+ done+ exit 42+ """))).taskDefinition();+ try {+ assertThat(definition.volumes()).hasSize(1);+ assertThat(definition.volumes().get(0).name()).isEqualTo("shared");+ assertThat(definition.volumes().get(0).host() != null).isEqualTo(emptyHost);+ RunTaskResponse started = ecs.runTask(request -> request.cluster(name)+ .taskDefinition(definition.taskDefinitionArn()).launchType(LaunchType.FARGATE)+ .networkConfiguration(network -> network.awsvpcConfiguration(vpc -> vpc.subnets(subnet))));+ assertThat(started.failures()).isEmpty();+ assertThat(started.tasks()).hasSize(1);+ Task task = started.tasks().get(0);+ try {+ assertThat(task.containers()).hasSize(2);+ assumeTrue(task.containers().stream().allMatch(container -> container.runtimeId() != null),+ "ECS Docker mode is required to check files shared by running containers");+ DescribeTasksRequest describe = DescribeTasksRequest.builder()+ .cluster(name).tasks(task.taskArn()).build();+ assertThat(ecs.waiter().waitUntilTasksStopped(describe,+ WaiterOverrideConfiguration.builder().waitTimeout(Duration.ofSeconds(30)).build())+ .matched().response()).isPresent();+ Task stopped = ecs.describeTasks(describe).tasks().get(0);+ assertThat(stopped.containers()).hasSize(2).allSatisfy(container -> {+ assertThat(container.lastStatus()).isEqualTo("STOPPED");+ assertThat(container.exitCode()).isZero();+ });+ } finally {+ ecs.stopTask(request -> request.cluster(name).task(task.taskArn()));+ }+ } finally {+ ecs.deregisterTaskDefinition(request -> request.taskDefinition(definition.taskDefinitionArn()));+ ecs.deleteCluster(request -> request.cluster(name));+ }+ }+ }++ private static ContainerDefinition container(String name, boolean readOnly, String command) {+ return ContainerDefinition.builder().name(name).image("busybox:stable").essential(true)+ .command("sh", "-c", command).stopTimeout(1)+ .mountPoints(MountPoint.builder().sourceVolume("shared").containerPath("/data")+ .readOnly(readOnly).build()).build();+ }+}diff --git a/docs/services/ecs.md b/docs/services/ecs.mdindex b7e9ad4835..11d011dab0 100644--- a/docs/services/ecs.md+++ b/docs/services/ecs.md@@ -862,6 +862,23 @@ for reasons that have nothing to do with the role's policy.Each of these logs a warning naming the task and the role it asked for.+### Ephemeral task volumes++A volume declared with only a `name`, or with an empty `host` and no `sourcePath`, supplies+ephemeral storage shared by the containers in that task. Each container's `mountPoints` selects+the volume, mount path and read-only mode. Floci creates a Docker volume for each mounted task+volume; different tasks get separate storage even when they use the same task definition.++Task teardown removes this storage after the containers have been removed. Cleanup also runs+when a task exits naturally or fails to launch. The task reconciler retries failed removals,+including cleanup left pending by a failed launch or a state reset.+With `reconcile-containers-on-startup` enabled, Floci also removes task volumes left by a previous+run after their containers are gone. Failed volume cleanup retries on reconciler ticks without+blocking new task launches. The sweep matches the emulator owner and run labels; it+leaves current-run volumes and other emulator instances alone. Unlike AWS's configurable delayed+EC2 agent cleanup, Floci attempts removal during teardown. EFS-backed volumes remain shared and persistent, and the+unsupported external volume configurations listed above remain unimplemented.+### Host volume safetyA task definition's `volumes[].host.sourcePath` is a caller-controlled filesystem path that Floci bind-mounts straight into the launched container, so both `RegisterTaskDefinition` and the actual bind mount at `RunTask` time validate it (the second check narrows the window between validation and mount, and also covers task definitions registered before this policy existed).@@ -886,7 +903,7 @@ Always rejected, regardless of configuration:- `FLOCI_SERVICES_ECS_ALLOW_UNSAFE_HOST_VOLUMES=true`: allow any host path (for local development where any host path should be mountable). The traversal, bare-root, and Docker socket blocks above are never bypassed by this flag.-A rejected `sourcePath` fails `RegisterTaskDefinition` with `InvalidParameterException`; a rejection caught again at `RunTask` time (e.g. a task definition registered before this policy existed) stops the task with that message as its `stoppedReason`. Named Docker volumes, EFS volumes, and host volumes with no `sourcePath` (ephemeral, container-local storage) are unaffected by these checks.+A rejected `sourcePath` fails `RegisterTaskDefinition` with `InvalidParameterException`; a rejection caught again at `RunTask` time (e.g. a task definition registered before this policy existed) stops the task with that message as its `stoppedReason`. Named Docker volumes, EFS volumes, and host volumes with no `sourcePath` (ephemeral storage shared within a task) are unaffected by these checks.### EFS volume ownershipdiff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 818b437407..605d323bfe 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -583,14 +583,23 @@ public class ContainerLifecycleManager {createVolumeIfAbsent(volumeName);}- /** Creates the named volume when it is missing; returns whether it was created. */+ /** Creates a volume with resource ownership labels for recoverable cleanup. */+ public void ensureVolume(String volumeName, Map<String, String> labels) {+ createVolumeIfAbsent(volumeName, labels);+ }+private boolean createVolumeIfAbsent(String volumeName) {+ return createVolumeIfAbsent(volumeName, Map.of());+ }++ /** Creates the named volume when it is missing; returns whether it was created. */+ private boolean createVolumeIfAbsent(String volumeName, Map<String, String> labels) {if (volumeExists(volumeName)) {return false;}dockerClient.createVolumeCmd().withName(volumeName)- .withLabels(ContainerStorageHelper.defaultLabels(config))+ .withLabels(mergedLabels(labels)).exec();LOG.debugv("Created volume {0}", volumeName);return true;diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandler.javaindex 61cbe91905..24e87852ab 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandler.java@@ -1517,12 +1517,16 @@ public class EcsJsonHandler {private static final Set<String> VOLUME_CONSUMED = Set.of("name", "efsVolumeConfiguration");private EfsVolumeConfiguration parseEfsVolumeConfiguration(JsonNode node) {- if (node == null || !node.isObject()) {+ if (node == null || node.isMissingNode() || node.isNull()) {return null;}+ if (!node.isObject()) {+ throw new AwsException("InvalidParameterException", "EFS volume configuration must be an object.", 400);+ }String fileSystemId = node.path("fileSystemId").asText(null);- if (fileSystemId == null) {- return null;+ if (fileSystemId == null || fileSystemId.isBlank()) {+ throw new AwsException("InvalidParameterException",+ "EFS volume configuration requires a fileSystemId.", 400);}Integer transitEncryptionPort = node.path("transitEncryptionPort").isNumber()? node.path("transitEncryptionPort").asInt() : null;diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex 71704a8242..bfb24d09b2 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -19,6 +19,7 @@ import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsSimport io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;+import io.github.hectorvent.floci.services.ecs.container.EcsTaskLaunchException;import io.github.hectorvent.floci.services.ecs.exec.EcsExecChannelHandler;import io.github.hectorvent.floci.services.ecs.exec.EcsExecSessionRegistry;import io.github.hectorvent.floci.services.ecs.exec.ExecSession;@@ -128,6 +129,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabprivate final Object reconcilerLock = new Object();// False until a previous run's task containers are confirmed gone; no task starts before.private volatile boolean leftoverContainersRemoved = true;+ private volatile boolean leftoverTaskVolumesRemoved;private volatile boolean leftoverContainersBlockReported;// region::clusterName → EcsCluster@@ -138,8 +140,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabprivate Map<String, Integer> latestRevisions = new ConcurrentHashMap<>();// taskArn → EcsTaskprivate final Map<String, EcsTask> tasks = new ConcurrentHashMap<>();- // taskArn → EcsTaskHandle (running containers or unresolved log readers)+ // taskArn → EcsTaskHandle (running containers or pending resource cleanup)private final Map<String, EcsTaskHandle> taskHandles = new ConcurrentHashMap<>();+ private final Set<String> shutdownCleanupTasks = ConcurrentHashMap.newKeySet();// region::clusterName/serviceName → EcsServiceModelprivate Map<String, EcsServiceModel> services = new ConcurrentHashMap<>();private AccountAwareStorageBackend<EcsServiceModel> servicesStore;@@ -239,6 +242,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabthis.containerManager = containerManager;this.dockerMode = !config.services().ecs().mock();this.sweepLeftoverContainers = dockerMode && config.services().ecs().reconcileContainersOnStartup();+ this.leftoverTaskVolumesRemoved = !sweepLeftoverContainers;this.baseUrl = config.effectiveBaseUrl();this.lbRegistrar = lbRegistrar;this.storageFactory = storageFactory;@@ -287,6 +291,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabvoid releasePreviousRunLeftovers() {if (sweepLeftoverContainers) {leftoverContainersRemoved = containerManager.removeLeftoverContainers();+ retryPreviousRunTaskVolumes();}if (lbRegistrar != null) {lbRegistrar.releaseRecordedTargets();@@ -384,8 +389,16 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}try {containerManager.stopTask(claimed);+ for (int attempt = 0; attempt < 2 && claimed.hasTaskVolumes(); attempt++) {+ containerManager.cleanupStoppedTask(claimed);+ }} catch (Exception e) {LOG.warnv("Failed to stop ECS task {0} on shutdown: {1}", taskArn, e.getMessage());+ } finally {+ if (claimed.hasTaskVolumes()) {+ taskHandles.put(taskArn, claimed);+ shutdownCleanupTasks.add(taskArn);+ }}}// Task state is memory-only, so an ENI left behind here would outlive every task that@@ -1551,6 +1564,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, ResettabstopTask(cluster.getClusterName(), taskArn, task.getStoppedReason(), task.getStopCode(), region);}} catch (Exception e) {+ if (e instanceof EcsTaskLaunchException failedLaunch) {+ retainPendingCleanupHandle(taskArn, failedLaunch.getHandle());+ }LOG.errorv("Failed to start ECS task {0}: {1}", taskArn, e.getMessage());task.setLastStatus(TaskStatus.STOPPED.name());task.setDesiredStatus(TaskStatus.STOPPED.name());@@ -1983,7 +1999,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabreturn task;}taskHandles.remove(task.getTaskArn(), handle);- retainUnresolvedLogHandle(task.getTaskArn(), handle);+ retainPendingCleanupHandle(task.getTaskArn(), handle);finishedAt = handle.getFinishedAt();}@@ -4716,6 +4732,11 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab// ── Service Reconciliation ────────────────────────────────────────────────void reconcile() {+ try {+ retryPreviousRunTaskVolumes();+ } catch (Exception e) {+ LOG.warnv(e, "ECS previous-run task volume cleanup failed: {0}", e.getMessage());+ }try {reconcileTasks();} catch (Exception e) {@@ -4733,6 +4754,12 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}}+ private void retryPreviousRunTaskVolumes() {+ if (sweepLeftoverContainers && leftoverContainersRemoved && !leftoverTaskVolumesRemoved) {+ leftoverTaskVolumesRemoved = containerManager.removeLeftoverTaskVolumes();+ }+ }+/*** Whether a task may start: true once the containers a previous run left behind are gone,* retrying the sweep otherwise. It runs before every task launch, RunTask and StartTask@@ -4798,8 +4825,13 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}private void reconcileTask(String taskArn) {+ if (shutdownCleanupTasks.contains(taskArn)) {+ reconcileStoppedTask(taskArn);+ return;+ }EcsTask task = tasks.get(taskArn);if (task == null) {+ reconcileStoppedTask(taskArn);return;}synchronized (task) {@@ -4854,6 +4886,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab// Close log streams and remove the stopped Docker containers without re-inspecting.containerManager.cleanupStoppedTask(claimed);+ retainPendingCleanupHandle(taskArn, claimed);if (task.getContainers() != null) {task.getContainers().forEach(c -> {@@ -4961,14 +4994,17 @@ public class EcsService implements ContainerTeardown, ResourceProvider, ResettabcontainerManager.stopTaskAndCollectExitCodes(claimed);} finally {// A task can be reported STOPPED even if Docker rejected both teardown attempts.- // Keep only the affected log readers and retry their teardown on later reconciliation ticks.- retainUnresolvedLogHandle(taskArn, claimed);+ // Keep pending log readers and task volumes for later cleanup attempts.+ retainPendingCleanupHandle(taskArn, claimed);}}- private void retainUnresolvedLogHandle(String taskArn, EcsTaskHandle handle) {- if (handle != null && handle.hasOpenLogStreams()) {+ private void retainPendingCleanupHandle(String taskArn, EcsTaskHandle handle) {+ if (handle != null && (handle.hasOpenLogStreams() || handle.hasTaskVolumes()+ || !handle.allContainersRemoved())) {taskHandles.put(taskArn, handle);+ } else {+ shutdownCleanupTasks.remove(taskArn);}}@@ -4986,8 +5022,10 @@ public class EcsService implements ContainerTeardown, ResourceProvider, ResettabcontainerIds.put(container.getName(), dockerId);}Map<String, Integer> stopTimeouts = new LinkedHashMap<>();+ Set<String> taskVolumeNames = Set.of();try {TaskDefinition definition = resolveTaskDefinitionOrThrow(task.getTaskDefinitionArn(), region);+ taskVolumeNames = containerManager.taskVolumeNames(task, definition);for (ContainerDefinition container : definition.getContainerDefinitions()) {if (container.getStopTimeout() != null) {stopTimeouts.put(container.getName(), container.getStopTimeout());@@ -4998,7 +5036,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabtask.getTaskArn());}EcsTaskHandle recovered = new EcsTaskHandle(task.getTaskArn(), containerIds, Map.of(),- null, task.getNetworkInterfaceId(), region, stopTimeouts);+ null, task.getNetworkInterfaceId(), region, stopTimeouts, taskVolumeNames);EcsTaskHandle existing = taskHandles.putIfAbsent(task.getTaskArn(), recovered);return existing != null ? existing : recovered;}diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.javaindex 7eba3b4e75..de674b2c49 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java@@ -3,6 +3,8 @@ package io.github.hectorvent.floci.services.ecs.container;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.async.ResultCallback;import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.command.InspectVolumeResponse;+import com.github.dockerjava.api.command.ListVolumesResponse;import com.github.dockerjava.api.command.StatsCmd;import com.github.dockerjava.api.exception.NotFoundException;import com.github.dockerjava.api.model.ContainerNetwork;@@ -75,6 +77,7 @@ import java.time.format.DateTimeParseException;import java.util.ArrayList;import java.util.HashSet;import java.util.LinkedHashMap;+import java.util.LinkedHashSet;import java.util.List;import java.util.Map;import java.util.Optional;@@ -144,6 +147,8 @@ public class EcsContainerManager {// A sweep failure is retried before each task launch; each distinct failure is a WARN once.private volatile boolean leftoverListFailureReported;private final Set<String> leftoverRemovalFailuresReported = ConcurrentHashMap.newKeySet();+ private volatile boolean leftoverTaskVolumeListFailureReported;+ private final Set<String> leftoverTaskVolumeRemovalFailuresReported = ConcurrentHashMap.newKeySet();@Injectpublic EcsContainerManager(ContainerBuilder containerBuilder,@@ -230,15 +235,19 @@ public class EcsContainerManager {// name -> EFS configuration (materialised below as a shared local Docker volume).Map<String, String> volumeSourcePaths = new LinkedHashMap<>();Map<String, EfsVolumeConfiguration> efsVolumes = new LinkedHashMap<>();+ Map<String, String> ephemeralVolumes = new LinkedHashMap<>();+ Set<String> taskVolumeNames = new LinkedHashSet<>();if (taskDef.getVolumes() != null) {for (Volume v : taskDef.getVolumes()) {if (v.name() == null) {continue;}- if (v.hostSourcePath() != null) {+ if (v.hostSourcePath() != null && !v.hostSourcePath().isBlank()) {volumeSourcePaths.put(v.name(), v.hostSourcePath());} else if (v.efs() != null) {efsVolumes.put(v.name(), v.efs());+ } else if (isEphemeralVolume(v)) {+ ephemeralVolumes.put(v.name(), taskVolumeName(task, v));}}}@@ -299,6 +308,13 @@ public class EcsContainerManager {throw e;}+ Map<String, Integer> stopTimeouts = new LinkedHashMap<>();+ for (ContainerDefinition def : taskDef.getContainerDefinitions()) {+ if (def.getStopTimeout() != null) {+ stopTimeouts.put(def.getName(), def.getStopTimeout());+ }+ }+String fluentHost = null;String networkModeName = taskDef.getNetworkMode() != null? taskDef.getNetworkMode().name()@@ -445,6 +461,7 @@ public class EcsContainerManager {}String sourcePath = volumeSourcePaths.get(mp.sourceVolume());EfsVolumeConfiguration efs = efsVolumes.get(mp.sourceVolume());+ String ephemeralVolume = ephemeralVolumes.get(mp.sourceVolume());if (sourcePath != null) {// Host volume: bind-mount an absolute path on the Docker host. Re-validate// here (not just at RegisterTaskDefinition time): this narrows the window@@ -460,6 +477,13 @@ public class EcsContainerManager {}} else if (efs != null) {mountEfsVolume(specBuilder, efs, mp);+ } else if (ephemeralVolume != null) {+ if (taskVolumeNames.add(ephemeralVolume)) {+ Map<String, String> labels = new LinkedHashMap<>(ownerLabels());+ labels.put(ContainerStorageHelper.SERVICE_LABEL, "ecs");+ lifecycleManager.ensureVolume(ephemeralVolume, labels);+ }+ specBuilder.withNamedVolume(ephemeralVolume, mp.containerPath(), mp.readOnly());} else {LOG.warnv("Skipping mountPoint with unresolved volume {0} on container {1}",mp.sourceVolume(), def.getName());@@ -497,6 +521,8 @@ public class EcsContainerManager {dockerId = info.containerId();}+ containerIds.put(def.getName(), dockerId);+LOG.infov("Created ECS container {0} for task {1} container {2}", dockerId, taskId, def.getName());// Resolve network bindings for ECS-specific model@@ -508,7 +534,6 @@ public class EcsContainerManager {metadataIdsByContainer.get(def.getName()));container.setImageDigest(imageDigestsByContainer.get(def.getName()));runtimeContainers.add(container);- containerIds.put(def.getName(), dockerId);// awsfirelens containers are shipped to Fluent Bit by Docker; don't also scrape json-file.// Only awslogs containers go to CloudWatch Logs; the others stay on the console.@@ -528,23 +553,16 @@ public class EcsContainerManager {}}} catch (Exception e) {- boolean allRemoved = true;- for (String dockerId : containerIds.values()) {- if (!stopAndConfirmRemoved(dockerId)) {- allRemoved = false;- }- }- if (firelensVolumeName != null) {- lifecycleManager.removeVolume(firelensVolumeName);- }+ EcsTaskHandle partial = new EcsTaskHandle(task.getTaskArn(), containerIds, logStreamsByContainerId,+ firelensVolumeName,+ protectedNetwork == null ? null : protectedNetwork.eni().getNetworkInterfaceId(), region,+ stopTimeouts, taskVolumeNames);+ cleanupStoppedTask(partial);if (protectedNetwork != null) {releaseTaskNetwork(task, region);}- revokeTaskRoleCredentials(task.getTaskArn());- // A container left behind by a failed removal is still on the network holding its- // address, exactly as on the stop path.- if (allRemoved) {- releaseTaskLinkLocalAddresses(task.getTaskArn());+ if (!partial.allContainersRemoved() || partial.hasTaskVolumes() || partial.hasOpenLogStreams()) {+ throw new EcsTaskLaunchException(e, partial);}throw e;}@@ -577,16 +595,44 @@ public class EcsContainerManager {task.setDesiredStatus(TaskStatus.RUNNING.name());task.setStartedAt(Instant.now());- Map<String, Integer> stopTimeouts = new LinkedHashMap<>();- for (ContainerDefinition def : taskDef.getContainerDefinitions()) {- if (def.getStopTimeout() != null) {- stopTimeouts.put(def.getName(), def.getStopTimeout());- }- }return new EcsTaskHandle(task.getTaskArn(), containerIds, logStreamsByContainerId,firelensVolumeName,protectedNetwork == null ? null : protectedNetwork.eni().getNetworkInterfaceId(), region,- stopTimeouts);+ stopTimeouts, taskVolumeNames);+ }++ private String taskVolumeName(EcsTask task, Volume volume) {+ return ContainerStorageHelper.dockerName(config,+ "ecs-task-volume-" + Digests.sha256Hex(task.getTaskArn() + "|" + volume.name()));+ }++ /** Reconstructs only task-owned volumes referenced by a container, without creating them. */+ public Set<String> taskVolumeNames(EcsTask task, TaskDefinition definition) {+ Set<String> referenced = new HashSet<>();+ for (ContainerDefinition container : definition.getContainerDefinitions()) {+ if (container.getMountPoints() != null) {+ container.getMountPoints().forEach(mount -> referenced.add(mount.sourceVolume()));+ }+ }+ Set<String> names = new LinkedHashSet<>();+ if (definition.getVolumes() != null) {+ for (Volume volume : definition.getVolumes()) {+ if (referenced.contains(volume.name()) && isEphemeralVolume(volume)) {+ names.add(taskVolumeName(task, volume));+ }+ }+ }+ return names;+ }++ private static boolean isEphemeralVolume(Volume volume) {+ Map<String, Object> properties = volume.unparsed();+ return (volume.hostSourcePath() == null || volume.hostSourcePath().isBlank()) && volume.efs() == null+ && (properties == null || (properties.get("efsVolumeConfiguration") == null+ && properties.get("dockerVolumeConfiguration") == null+ && properties.get("fsxWindowsFileServerVolumeConfiguration") == null+ && properties.get("s3filesVolumeConfiguration") == null+ && !Boolean.TRUE.equals(properties.get("configuredAtLaunch"))));}/** A task-level {@code memory} is MiB as a plain number, or a {@code "1GB"}-style string. */@@ -1163,6 +1209,61 @@ public class EcsContainerManager {return allRemoved;}+ /** Removes previous-run task volumes independently of the container launch gate. */+ public boolean removeLeftoverTaskVolumes() {+ return removeLeftoverTaskVolumes(runId);+ }++ /** {@link #removeLeftoverTaskVolumes()} keeping volumes of {@code currentRunId}. */+ public boolean removeLeftoverTaskVolumes(String currentRunId) {+ String owner = ContainerStorageHelper.ownerIdentity(config);+ ContainerStorageHelper.LabelAliases aliases = ContainerStorageHelper.CONTAINER_LABEL_ALIASES;+ try {+ List<InspectVolumeResponse> volumes = aliases.listByLabels(+ Map.of(ContainerStorageHelper.SERVICE_LABEL, "ecs", ContainerStorageHelper.OWNER_LABEL, owner),+ filter -> {+ ListVolumesResponse response = lifecycleManager.getDockerClient().listVolumesCmd()+ .withFilter("label", filter.entrySet().stream()+ .map(entry -> entry.getKey() + "=" + entry.getValue()).toList()).exec();+ if (response == null || response.getVolumes() == null) {+ throw new IllegalStateException("Docker returned no volume inventory");+ }+ return response.getVolumes();+ }, InspectVolumeResponse::getName);+ leftoverTaskVolumeListFailureReported = false;+ boolean allRemoved = true;+ for (InspectVolumeResponse volume : volumes) {+ Map<String, String> labels = volume.getLabels();+ String previousRun = aliases.labelValue(labels, RUN_LABEL);+ if (!aliases.consistent(volume.getName(), labels)+ || !aliases.matches(labels, ContainerStorageHelper.SERVICE_LABEL, "ecs")+ || !aliases.matches(labels, ContainerStorageHelper.OWNER_LABEL, owner)+ || previousRun == null || currentRunId.equals(previousRun)) {+ continue;+ }+ try {+ lifecycleManager.removeVolumeStrict(volume.getName());+ leftoverTaskVolumeRemovalFailuresReported.remove(volume.getName());+ } catch (RuntimeException e) {+ allRemoved = false;+ Logger.Level level = leftoverTaskVolumeRemovalFailuresReported.add(volume.getName())+ ? Logger.Level.WARN : Logger.Level.DEBUG;+ LOG.logv(level, "Could not remove ECS task volume {0} left by a previous run: {1}",+ volume.getName(), e.getMessage());+ }+ }+ if (allRemoved) {+ leftoverTaskVolumeRemovalFailuresReported.clear();+ }+ return allRemoved;+ } catch (RuntimeException e) {+ LOG.logv(leftoverTaskVolumeListFailureReported ? Logger.Level.DEBUG : Logger.Level.WARN,+ "Could not list ECS task volumes from a previous run: {0}", e.getMessage());+ leftoverTaskVolumeListFailureReported = true;+ return false;+ }+ }+/** The ENI {@link #attachTaskNetwork} allocated for this task, or null if it is already gone. */private NetworkInterface existingTaskEni(EcsTask task, String region) {if (task.getNetworkInterfaceId() == null) {@@ -1274,8 +1375,10 @@ public class EcsContainerManager {return;}boolean allRemoved = true;- for (String dockerId : handle.getContainerIds().values()) {- if (!stopAndConfirmRemoved(dockerId)) {+ for (Map.Entry<String, String> entry : handle.getContainerIds().entrySet()) {+ if (stopAndConfirmRemoved(entry.getValue())) {+ handle.recordContainerRemoved(entry.getKey());+ } else {allRemoved = false;}}@@ -1285,6 +1388,7 @@ public class EcsContainerManager {removeFirelensVolume(handle);revokeTaskRoleCredentials(handle.getTaskArn());if (allRemoved) {+ removeTaskVolumes(handle);releaseTaskLinkLocalAddresses(handle.getTaskArn());}}@@ -1373,6 +1477,7 @@ public class EcsContainerManager {if (handle.allContainersRemoved()) {cleanupProtectedNetwork(handle);removeFirelensVolume(handle);+ removeTaskVolumes(handle);// Only now: a container that survived a failed removal still holds its address.releaseTaskLinkLocalAddresses(handle.getTaskArn());}@@ -1423,6 +1528,14 @@ public class EcsContainerManager {lifecycleManager.removeVolume(handle.getFirelensVolumeName());}+ private void removeTaskVolumes(EcsTaskHandle handle) {+ for (String volumeName : handle.getTaskVolumeNames()) {+ if (lifecycleManager.removeVolume(volumeName)) {+ handle.recordTaskVolumeRemoved(volumeName);+ }+ }+ }+private static ContainerDefinition findFirelensRouter(List<ContainerDefinition> defs) {if (defs == null) {return null;diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.javaindex b36da8baf3..19b0f47a26 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java@@ -4,6 +4,7 @@ import java.io.Closeable;import java.time.Instant;import java.util.HashSet;import java.util.LinkedHashMap;+import java.util.LinkedHashSet;import java.util.Map;import java.util.Set;@@ -29,6 +30,7 @@ public class EcsTaskHandle {private final String firelensVolumeName;private final String networkInterfaceId;private final String region;+ private final Set<String> taskVolumeNames;/** Per-container {@code stopTimeout}, so teardown waits as long as the task definition asked. */private final Map<String, Integer> stopTimeouts;/**@@ -57,12 +59,21 @@ public class EcsTaskHandle {Map<String, Closeable> logStreamsByContainerId,String firelensVolumeName, String networkInterfaceId, String region,Map<String, Integer> stopTimeouts) {+ this(taskArn, containerIds, logStreamsByContainerId, firelensVolumeName,+ networkInterfaceId, region, stopTimeouts, Set.of());+ }++ public EcsTaskHandle(String taskArn, Map<String, String> containerIds,+ Map<String, Closeable> logStreamsByContainerId,+ String firelensVolumeName, String networkInterfaceId, String region,+ Map<String, Integer> stopTimeouts, Set<String> taskVolumeNames) {this.taskArn = taskArn;this.containerIds = new LinkedHashMap<>(containerIds);this.logStreamsByContainerId = new LinkedHashMap<>(logStreamsByContainerId);this.firelensVolumeName = firelensVolumeName;this.networkInterfaceId = networkInterfaceId;this.region = region;+ this.taskVolumeNames = new LinkedHashSet<>(taskVolumeNames);this.stopTimeouts = new LinkedHashMap<>(stopTimeouts);}@@ -72,6 +83,15 @@ public class EcsTaskHandle {public String getFirelensVolumeName() { return firelensVolumeName; }public String getNetworkInterfaceId() { return networkInterfaceId; }public String getRegion() { return region; }+ public Set<String> getTaskVolumeNames() { return Set.copyOf(taskVolumeNames); }++ public void recordTaskVolumeRemoved(String volumeName) {+ taskVolumeNames.remove(volumeName);+ }++ public boolean hasTaskVolumes() {+ return !taskVolumeNames.isEmpty();+ }/** The container's {@code stopTimeout} in seconds, or {@link #DEFAULT_STOP_TIMEOUT_SECONDS}. */public int stopTimeoutFor(String containerName) {diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskLaunchException.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskLaunchException.javanew file mode 100644index 0000000000..34b424cc8c--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskLaunchException.java@@ -0,0 +1,20 @@+package io.github.hectorvent.floci.services.ecs.container;++import io.github.hectorvent.floci.core.common.AwsException;++/** A failed launch whose remaining resources still need teardown. */+public class EcsTaskLaunchException extends AwsException {++ private final EcsTaskHandle handle;++ public EcsTaskLaunchException(Exception cause, EcsTaskHandle handle) {+ super(cause instanceof AwsException aws ? aws.getErrorCode() : "ClientException",+ cause.getMessage(), cause instanceof AwsException aws ? aws.getHttpStatus() : 400);+ initCause(cause);+ this.handle = handle;+ }++ public EcsTaskHandle getHandle() {+ return handle;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/model/Volume.java b/src/main/java/io/github/hectorvent/floci/services/ecs/model/Volume.javaindex 601dc7ca9e..778e47a2bb 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/model/Volume.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/model/Volume.java@@ -5,14 +5,16 @@ import io.quarkus.runtime.annotations.RegisterForReflection;import java.util.Map;/**- * A task-level volume in an ECS task definition. Two shapes are modelled:+ * A task-level volume in an ECS task definition. Three shapes are modelled:* <ul>+ * <li>A volume without a host source path or external configuration supplies ephemeral+ * storage shared by the containers in one task.</li>* <li>EC2-launch-type {@code host} volume: {@code {"name": ..., "host": {"sourcePath": ...}}};* {@code hostSourcePath} is the absolute path on the Docker host that the volume binds to.</li>* <li>{@code efsVolumeConfiguration} volume: {@code {"name": ..., "efsVolumeConfiguration": {...}}};* see {@link EfsVolumeConfiguration}.</li>* </ul>- * The two are mutually exclusive. A container references the volume by {@code name} via a+ * The configurations are mutually exclusive. A container references the volume by {@code name} via a* {@link MountPoint}.** <p>{@code unparsed} keeps the volume members Floci backs with nothing, which is the rest ofdiff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.javaindex ff971a2f5f..e2ee42810e 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java@@ -29,6 +29,7 @@ import java.io.Closeable;import java.io.IOException;import java.util.ArrayList;import java.util.List;+import java.util.Map;import java.util.Optional;import java.util.OptionalInt;import java.util.Set;@@ -65,6 +66,22 @@ class ContainerLifecycleManagerVolumeTest {dockerClient, imageCacheService, containerDetector, portAllocator, config);}+ @Test+ void ensureVolumePersistsOwnershipLabelsAndTheirLegacyAliases() {+ InspectVolumeCmd inspect = mock(InspectVolumeCmd.class);+ when(dockerClient.inspectVolumeCmd("task-volume")).thenReturn(inspect);+ when(inspect.exec()).thenThrow(new NotFoundException("missing"));+ CreateVolumeCmd create = mock(CreateVolumeCmd.class, RETURNS_SELF);+ when(dockerClient.createVolumeCmd()).thenReturn(create);+ manager.ensureVolume("task-volume", Map.of("io.floci.service", "ecs",+ "io.floci.owner", "alpha/4566", "io.floci.ecs.run", "run-id"));+ verify(create).withLabels(argThat(labels -> "ecs".equals(labels.get("io.floci.service"))+ && "alpha/4566".equals(labels.get("io.floci.owner"))+ && "alpha/4566".equals(labels.get("floci_owner_port"))+ && "run-id".equals(labels.get("io.floci.ecs.run"))+ && "run-id".equals(labels.get("floci.ecs-run"))));+ }+@Testvoid stopAndRemoveStopsContainerBeforeClosingItsLogStream() {StopContainerCmd stop = mock(StopContainerCmd.class);diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandlerVolumesTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandlerVolumesTest.javaindex c3a6a8d68f..d012bf9b20 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandlerVolumesTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandlerVolumesTest.java@@ -13,6 +13,8 @@ import jakarta.ws.rs.core.Response;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.io.TempDir;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;import java.io.IOException;import java.nio.file.Files;@@ -49,6 +51,19 @@ class EcsJsonHandlerVolumesTest {private EmulatorConfig config;private EcsJsonHandler handler;+ @ParameterizedTest+ @ValueSource(strings = {"{}", "{\"fileSystemId\":\"\"}", "{\"fileSystemId\":\" \"}", "\"invalid\""})+ void registerTaskDefinitionRejectsMalformedEfsVolume(String efs) throws Exception {+ JsonNode request = objectMapper.readTree("""+ {"family":"invalid-efs","containerDefinitions":[{"name":"app","image":"busybox:stable",+ "mountPoints":[{"sourceVolume":"shared","containerPath":"/data"}]}],+ "volumes":[{"name":"shared","efsVolumeConfiguration":%s}]}+ """.formatted(efs));+ AwsException error = assertThrows(AwsException.class,+ () -> handler.handle("RegisterTaskDefinition", request, "us-east-1"));+ assertEquals("InvalidParameterException", error.getErrorCode());+ }+@BeforeEachvoid setUp() {objectMapper = new ObjectMapper();diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.javaindex 4327722b81..8a65628ec6 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java@@ -49,6 +49,63 @@ class EcsServiceRestartLeftoversTest {private static final String TARGET_GROUP_ARN ="arn:aws:elasticloadbalancing:us-east-1:000000000000:targetgroup/web/0123456789abcdef";+ @Test+ void pendingPreviousRunVolumesDoNotBlockRunTaskOrServiceTasksAndRetryOnTicks() {+ SharedStorageFactory storage = new SharedStorageFactory();+ persistServiceWithLoadBalancer(storage);+ EcsContainerManager manager = mock(EcsContainerManager.class);+ when(manager.removeLeftoverContainers()).thenReturn(true);+ when(manager.removeLeftoverTaskVolumes()).thenReturn(false, false, true);+ when(manager.getExitCodeIfStopped(anyString())).thenReturn(null);+ when(manager.startTask(any(), any(), any(), anyString())).thenAnswer(invocation -> {+ EcsTask task = invocation.getArgument(0);+ return new EcsTaskHandle(task.getTaskArn(), Map.of("web", "docker-id"), Map.of());+ });+ EcsService restarted = service(storage, false, manager, mock(EcsLoadBalancerRegistrar.class), null);+ restarted.releasePreviousRunLeftovers();+ EcsTask task = restarted.runTask("app-cluster", "web", 1, LaunchType.EC2, null, null,+ null, null, REGION).getFirst();+ assertEquals("RUNNING", task.getLastStatus());+ restarted.reconcileServices();+ verify(manager, times(2)).startTask(any(), any(), any(), anyString());+ verify(manager).removeLeftoverTaskVolumes();++ restarted.reconcile();+ restarted.reconcile();+ restarted.reconcile();+ verify(manager, times(3)).removeLeftoverTaskVolumes();+ verify(manager).removeLeftoverContainers();+ verify(manager, times(2)).startTask(any(), any(), any(), anyString());+ }++ @Test+ void volumeCleanupWaitsForContainersAndRetriesWithoutAnyTaskLaunch() {+ EcsContainerManager manager = mock(EcsContainerManager.class);+ when(manager.removeLeftoverContainers()).thenReturn(false, true);+ when(manager.removeLeftoverTaskVolumes()).thenReturn(false, true);+ when(manager.getExitCodeIfStopped(anyString())).thenReturn(null);+ when(manager.startTask(any(), any(), any(), anyString())).thenAnswer(invocation -> {+ EcsTask task = invocation.getArgument(0);+ return new EcsTaskHandle(task.getTaskArn(), Map.of("web", "docker-id"), Map.of());+ });+ EcsService restarted = service(new SharedStorageFactory(), false, manager,+ mock(EcsLoadBalancerRegistrar.class), null);+ restarted.releasePreviousRunLeftovers();+ restarted.reconcile();+ verify(manager, never()).removeLeftoverTaskVolumes();+ ContainerDefinition definition = new ContainerDefinition();+ definition.setName("web");+ definition.setImage("busybox:stable");+ restarted.registerTaskDefinition("web", List.of(definition), NetworkMode.bridge, null, null,+ null, null, List.of(), REGION);+ restarted.runTask(null, "web", 1, LaunchType.EC2, null, null, null, null, REGION);+ restarted.reconcile();+ restarted.reconcile();+ restarted.reconcile();+ verify(manager, times(2)).removeLeftoverTaskVolumes();+ verify(manager, times(2)).removeLeftoverContainers();+ }+@Testvoid releasesTheRegistrationsThePreviousRunRecorded() {EcsLoadBalancerRegistrar lbRegistrar = mock(EcsLoadBalancerRegistrar.class);@@ -138,6 +195,7 @@ class EcsServiceRestartLeftoversTest {mock(EcsLoadBalancerRegistrar.class), null).releasePreviousRunLeftovers();verify(containerManager, never()).removeLeftoverContainers();+ verify(containerManager, never()).removeLeftoverTaskVolumes();}@Test@@ -158,6 +216,7 @@ class EcsServiceRestartLeftoversTest {restarted.runTask("app-cluster", "web", 1, LaunchType.EC2, null, null, null, null, REGION);verify(containerManager, never()).removeLeftoverContainers();+ verify(containerManager, never()).removeLeftoverTaskVolumes();verify(containerManager).startTask(any(), any(), any(), anyString());verify(lbRegistrar).releaseRecordedTargets();verify(discoveryRegistrar).releaseRecordedInstances();diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.javaindex 3f41617aaf..0baa89c38a 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java@@ -9,16 +9,20 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;+import io.github.hectorvent.floci.services.ecs.container.EcsTaskLaunchException;import io.github.hectorvent.floci.services.ecs.model.Container;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.EcsTask;import io.github.hectorvent.floci.services.ecs.model.LaunchType;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;import java.io.Closeable;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.Set;import java.util.concurrent.CompletableFuture;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;@@ -33,6 +37,7 @@ import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.argThat;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;@@ -47,6 +52,127 @@ class EcsServiceTeardownTest {private static final String REGION = "us-east-1";+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void stoppedTaskKeepsItsHandleUntilEphemeralVolumeCleanupSucceeds(boolean exitsNaturally) {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(false);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ EcsContainerManager manager = mock(EcsContainerManager.class);+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"), Map.of(),+ null, null, REGION, Map.of(), Set.of("task-volume"));+ when(manager.startTask(any(), any(), any(), anyString())).thenReturn(handle);+ when(manager.getExitCodeIfStopped("docker-id")).thenReturn(0);+ AtomicInteger attempts = new AtomicInteger();+ doAnswer(ignored -> {+ attempts.incrementAndGet();+ handle.recordContainerRemoved("app");+ return null;+ }).when(manager).cleanupStoppedTask(handle);+ when(manager.stopTaskAndCollectExitCodes(handle)).thenAnswer(ignored -> {+ handle.recordContainerRemoved("app");+ if (attempts.incrementAndGet() > 1) {+ handle.recordTaskVolumeRemoved("task-volume");+ }+ return Map.of("app", 0);+ });+ EcsService service = new EcsService(new RegionResolver(REGION, "000000000000"), manager, config,+ mock(EcsLoadBalancerRegistrar.class), new SingleUseStorageFactory(), null);+ service.initializeStorage();+ ContainerDefinition definition = new ContainerDefinition();+ definition.setName("app");+ definition.setImage("app:latest");+ service.registerTaskDefinition("volume-retry", List.of(definition), null, null, null,+ null, null, List.of(), REGION);+ String taskArn = service.runTask(null, "volume-retry", 1, LaunchType.FARGATE, null, null,+ List.of(), null, REGION).getFirst().getTaskArn();++ if (exitsNaturally) {+ service.reconcile();+ } else {+ service.stopTask(null, taskArn, null, REGION);+ }+ assertEquals("STOPPED", service.describeTasks(null, List.of(taskArn), REGION).getFirst().getLastStatus());+ assertTrue(handle.hasTaskVolumes());++ service.reconcile();+ assertFalse(handle.hasTaskVolumes());+ service.reconcile();+ assertEquals(2, attempts.get(), "cleanup stops retrying after the volume is removed");+ }++ @Test+ void failedLaunchCleanupIsRetriedByStoppedTaskReconciliation() {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(false);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ EcsContainerManager manager = mock(EcsContainerManager.class);+ EcsTaskHandle partial = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"), Map.of(),+ null, null, REGION, Map.of(), Set.of("task-volume"));+ when(manager.startTask(any(), any(), any(), anyString()))+ .thenThrow(new EcsTaskLaunchException(new IllegalStateException("launch failed"), partial));+ when(manager.stopTaskAndCollectExitCodes(partial)).thenAnswer(ignored -> {+ partial.recordContainerRemoved("app");+ partial.recordTaskVolumeRemoved("task-volume");+ return Map.of("app", 137);+ });+ EcsService service = new EcsService(new RegionResolver(REGION, "000000000000"), manager, config,+ mock(EcsLoadBalancerRegistrar.class), new SingleUseStorageFactory(), null);+ service.initializeStorage();+ ContainerDefinition definition = new ContainerDefinition();+ definition.setName("app");+ definition.setImage("busybox:stable");+ service.registerTaskDefinition("failed-volume-launch", List.of(definition), null, null, null,+ null, null, List.of(), REGION);+ EcsTask task = service.runTask(null, "failed-volume-launch", 1, LaunchType.FARGATE, null, null,+ List.of(), null, REGION).getFirst();+ assertEquals("STOPPED", task.getLastStatus());+ assertEquals("Failed to start: launch failed", task.getStoppedReason());+ assertTrue(partial.hasTaskVolumes());+ service.reconcile();+ assertFalse(partial.hasTaskVolumes());+ service.reconcile();+ verify(manager, times(1)).stopTaskAndCollectExitCodes(partial);+ service.stopManagedContainers();+ }++ @Test+ void shutdownRetriesVolumeRemovalAndRetainsPendingCleanupAfterReset() {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(false);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ EcsContainerManager manager = mock(EcsContainerManager.class);+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"), Map.of(),+ null, null, REGION, Map.of(), Set.of("task-volume"));+ when(manager.startTask(any(), any(), any(), anyString())).thenReturn(handle);+ doAnswer(ignored -> {+ handle.recordContainerRemoved("app");+ return null;+ }).when(manager).stopTask(handle);+ when(manager.stopTaskAndCollectExitCodes(handle)).thenAnswer(ignored -> {+ handle.recordTaskVolumeRemoved("task-volume");+ return Map.of("app", 0);+ });+ EcsService service = new EcsService(new RegionResolver(REGION, "000000000000"), manager, config,+ mock(EcsLoadBalancerRegistrar.class), new SingleUseStorageFactory(), null);+ service.initializeStorage();+ ContainerDefinition definition = new ContainerDefinition();+ definition.setName("app");+ definition.setImage("busybox:stable");+ service.registerTaskDefinition("shutdown-volume", List.of(definition), null, null, null,+ null, null, List.of(), REGION);+ service.runTask(null, "shutdown-volume", 1, LaunchType.FARGATE, null, null,+ List.of(), null, REGION);+ service.stopManagedContainers();+ verify(manager, times(2)).cleanupStoppedTask(handle);+ assertTrue(handle.hasTaskVolumes());+ service.clear();+ service.afterReset();+ service.reconcile();+ assertFalse(handle.hasTaskVolumes());+ service.stopManagedContainers();+ }+@Testvoid startupTransitionWaitsForTheTaskLock() throws Exception {EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);@@ -175,9 +301,12 @@ class EcsServiceTeardownTest {task.setContainers(List.of(container));return original;});+ when(containerManager.taskVolumeNames(any(), any())).thenReturn(Set.of("recovered-volume"));when(containerManager.stopTaskAndCollectExitCodes(any())).thenAnswer(invocation -> {EcsTaskHandle stoppingHandle = invocation.getArgument(0);+ assertEquals(Set.of("recovered-volume"), stoppingHandle.getTaskVolumeNames());stoppingHandle.recordContainerRemoved("app");+ stoppingHandle.recordTaskVolumeRemoved("recovered-volume");return Map.of("app", 0);});@@ -200,6 +329,7 @@ class EcsServiceTeardownTest {verify(containerManager).stopTaskAndCollectExitCodes(argThat(handle -> "docker-id".equals(handle.getContainerIds().get("app"))));+ verify(containerManager).taskVolumeNames(any(), any());assertEquals("STOPPED", service.describeTasks(null, List.of(taskArn), REGION).getFirst().getLastStatus());service.stopManagedContainers();}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEphemeralVolumesDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEphemeralVolumesDockerIntegrationTest.javanew file mode 100644index 0000000000..1c72e96506--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEphemeralVolumesDockerIntegrationTest.java@@ -0,0 +1,163 @@+package io.github.hectorvent.floci.services.ecs.container;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.InspectContainerResponse.Mount;+import com.github.dockerjava.api.exception.NotFoundException;+import com.github.dockerjava.core.command.WaitContainerResultCallback;+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;+import io.github.hectorvent.floci.services.ecs.model.EcsTask;+import io.github.hectorvent.floci.services.ecs.model.MountPoint;+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;+import io.github.hectorvent.floci.services.ecs.model.Volume;+import io.github.hectorvent.floci.testing.TestImages;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import jakarta.inject.Inject;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.Assumptions;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.UUID;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++@QuarkusTest+@TestProfile(EcsContainerManagerEphemeralVolumesDockerIntegrationTest.VolumeProfile.class)+class EcsContainerManagerEphemeralVolumesDockerIntegrationTest {++ @Inject+ EcsContainerManager manager;++ @Inject+ DockerClient dockerClient;++ private final List<EcsTaskHandle> handles = new ArrayList<>();++ public static class VolumeProfile implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.of("floci.docker.resource-namespace", "ecs-ephemeral-volumes-test");+ }+ }++ @BeforeEach+ void requireDocker() {+ try {+ dockerClient.pingCmd().exec();+ } catch (Exception unavailable) {+ Assumptions.abort("Docker daemon must be available for ephemeral volume integration tests");+ }+ }++ @AfterEach+ void cleanUpTasks() {+ handles.forEach(manager::stopTask);+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void containersShareEphemeralStorageWithinATaskAndRemoveItOnTeardown(boolean emptyHost) throws Exception {+ EcsTaskHandle first = startTask("first-task", emptyHost, 300);+ assertEquals(0, waitFor(first, "reader"), "the reader must see the writer's file through the shared mount");+ Mount firstWriter = dataMount(first, "writer");+ Mount firstReader = dataMount(first, "reader");+ assertEquals(firstWriter.getName(), firstReader.getName());+ assertEquals(Boolean.TRUE, firstWriter.getRW());+ assertEquals(Boolean.FALSE, firstReader.getRW());++ EcsTaskHandle second = startTask("second-task", emptyHost, 1);+ assertEquals(0, waitFor(second, "reader"), "a second task must get its own empty volume");+ Mount secondWriter = dataMount(second, "writer");+ assertNotEquals(firstWriter.getName(), secondWriter.getName());+ assertEquals(0, waitFor(second, "writer"));++ manager.stopTask(first);+ handles.remove(first);+ assertThrows(NotFoundException.class, () -> dockerClient.inspectVolumeCmd(firstWriter.getName()).exec());+ dockerClient.inspectVolumeCmd(secondWriter.getName()).exec();++ manager.cleanupStoppedTask(second);+ handles.remove(second);+ assertThrows(NotFoundException.class, () -> dockerClient.inspectVolumeCmd(secondWriter.getName()).exec());+ }++ @Test+ void nextRunRemovesOrphanVolumesAndContainersWithMatchingOwnership() {+ EcsTaskHandle task = startTask("orphan-task", false, 300);+ String volumeName = task.getTaskVolumeNames().iterator().next();+ Map<String, String> labels = dockerClient.inspectVolumeCmd(volumeName).exec().getLabels();+ assertEquals("ecs", labels.get(ContainerStorageHelper.SERVICE_LABEL));+ assertEquals("ecs-ephemeral-volumes-test/4566", labels.get(ContainerStorageHelper.OWNER_LABEL));+ assertTrue(labels.containsKey(EcsContainerManager.RUN_LABEL));+ try {+ String nextRun = "next-run-" + UUID.randomUUID();+ assertTrue(manager.removeLeftoverContainers(nextRun));+ dockerClient.inspectVolumeCmd(volumeName).exec();+ assertTrue(manager.removeLeftoverTaskVolumes(nextRun));+ assertThrows(NotFoundException.class, () -> dockerClient.inspectVolumeCmd(volumeName).exec());+ for (String containerId : task.getContainerIds().values()) {+ assertThrows(NotFoundException.class, () -> dockerClient.inspectContainerCmd(containerId).exec());+ }+ } finally {+ manager.stopTask(task);+ handles.remove(task);+ }+ }++ private EcsTaskHandle startTask(String marker, boolean emptyHost, int writerSleep) {+ ContainerDefinition writer = definition("writer", false);+ writer.setCommand(List.of("sh", "-c",+ "test ! -e /data/marker || exit 41; echo " + marker + " > /data/marker; exec sleep " + writerSleep));+ ContainerDefinition reader = definition("reader", true);+ reader.setCommand(List.of("sh", "-c", """+ for attempt in 1 2 3 4 5; do+ if [ "$(cat /data/marker 2>/dev/null)" = "%s" ]; then exit 0; fi+ sleep 1+ done+ exit 42+ """.formatted(marker)));+ TaskDefinition taskDefinition = new TaskDefinition();+ taskDefinition.setFamily("ephemeral-volume-test");+ taskDefinition.setContainerDefinitions(List.of(writer, reader));+ taskDefinition.setVolumes(List.of(new Volume("shared", null, null,+ emptyHost ? Map.of("host", Map.of()) : null)));+ EcsTask task = new EcsTask();+ task.setTaskArn("arn:aws:ecs:us-east-1:000000000000:task/ephemeral-volume-test/" + UUID.randomUUID());+ EcsTaskHandle handle = manager.startTask(task, taskDefinition, List.of(), "us-east-1");+ handles.add(handle);+ return handle;+ }++ private static ContainerDefinition definition(String name, boolean readOnly) {+ ContainerDefinition definition = new ContainerDefinition();+ definition.setName(name);+ definition.setImage(TestImages.BUSYBOX);+ definition.setStopTimeout(1);+ definition.setMountPoints(List.of(new MountPoint("shared", "/data", readOnly)));+ return definition;+ }++ private Mount dataMount(EcsTaskHandle handle, String container) {+ return dockerClient.inspectContainerCmd(handle.getContainerIds().get(container)).exec().getMounts().stream()+ .filter(mount -> "/data".equals(mount.getDestination().getPath())).findFirst().orElseThrow();+ }++ private Integer waitFor(EcsTaskHandle handle, String container) throws Exception {+ try (WaitContainerResultCallback callback = new WaitContainerResultCallback()) {+ return dockerClient.waitContainerCmd(handle.getContainerIds().get(container)).exec(callback)+ .awaitStatusCode(20, TimeUnit.SECONDS);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.javaindex 9b3cf84c66..9367fa96e1 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java@@ -1,7 +1,10 @@package io.github.hectorvent.floci.services.ecs.container;import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.InspectVolumeResponse;import com.github.dockerjava.api.command.ListContainersCmd;+import com.github.dockerjava.api.command.ListVolumesCmd;+import com.github.dockerjava.api.command.ListVolumesResponse;import com.github.dockerjava.api.command.RemoveContainerCmd;import com.github.dockerjava.api.command.StopContainerCmd;import com.github.dockerjava.api.model.Container;@@ -16,6 +19,7 @@ import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;import io.github.hectorvent.floci.services.ssm.SsmService;+import io.github.hectorvent.floci.testutil.LogCapture;import org.junit.jupiter.api.Test;import java.io.Closeable;@@ -23,6 +27,8 @@ import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.logging.Level;+import java.util.logging.LogRecord;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -33,6 +39,7 @@ import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;import static org.mockito.Mockito.RETURNS_SELF;import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.doNothing;import static org.mockito.Mockito.doReturn;import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;@@ -259,10 +266,126 @@ class EcsContainerManagerLifecycleTest {verify(lifecycleManager, never()).removeIfExistsStrict(anyString());}+ @Test+ void previousRunVolumesAreRemovedAfterContainersAndFailedRemovalIsRetried() {+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);+ stubLeftoverListing(lifecycle, Map.of(), Map.of());+ InspectVolumeResponse old = leftoverVolume("old", "alpha/4566", "previous");+ InspectVolumeResponse current = leftoverVolume("current", "alpha/4566", "current");+ InspectVolumeResponse foreign = leftoverVolume("foreign", "beta/4566", "previous");+ InspectVolumeResponse unowned = leftoverVolume("unowned", "alpha/4566", null);+ InspectVolumeResponse conflicting = leftoverVolume("conflicting", "alpha/4566", "previous");+ when(conflicting.getLabels()).thenReturn(Map.of("io.floci.service", "ecs",+ "io.floci.owner", "alpha/4566", "floci_owner_port", "beta/4566", "io.floci.ecs.run", "previous"));+ stubVolumeListing(lifecycle.getDockerClient(), List.of(old, current, foreign, unowned, conflicting));+ doThrow(new IllegalStateException("busy")).doNothing().when(lifecycle).removeVolumeStrict("old");+ EcsContainerManager manager = manager(lifecycle, namespacedConfig());++ assertTrue(manager.removeLeftoverContainers("current"));+ assertFalse(manager.removeLeftoverTaskVolumes("current"));+ assertTrue(manager.removeLeftoverTaskVolumes("current"));+ verify(lifecycle, times(2)).removeVolumeStrict("old");+ verify(lifecycle, never()).removeVolumeStrict("current");+ verify(lifecycle, never()).removeVolumeStrict("foreign");+ verify(lifecycle, never()).removeVolumeStrict("unowned");+ verify(lifecycle, never()).removeVolumeStrict("conflicting");+ }++ @Test+ void previousRunVolumeSweepWaitsForContainerRemoval() {+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);+ stubLeftoverListing(lifecycle, Map.of("old-container", leftover("old-container",+ Map.of("io.floci.service", "ecs", "io.floci.owner", "alpha/4566", "io.floci.ecs.run", "previous"))),+ Map.of());+ doThrow(new IllegalStateException("busy")).when(lifecycle).removeIfExistsStrict("old-container");+ assertFalse(manager(lifecycle, namespacedConfig()).removeLeftoverContainers("current"));+ verify(lifecycle.getDockerClient(), never()).listVolumesCmd();+ }++ @Test+ void previousRunVolumeSweepRetriesWhenInventoryCannotBeRead() {+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);+ stubLeftoverListing(lifecycle, Map.of(), Map.of());+ when(lifecycle.getDockerClient().listVolumesCmd()).thenThrow(new IllegalStateException("offline"));+ EcsContainerManager manager = manager(lifecycle, namespacedConfig());+ assertTrue(manager.removeLeftoverContainers("current"));+ assertFalse(manager.removeLeftoverTaskVolumes("current"));+ verify(lifecycle, never()).removeVolume(anyString());+ }++ @Test+ void volumeRemovalWarnsOncePerVolumeAndResetsAfterSuccessfulRemoval() {+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);+ stubLeftoverListing(lifecycle, Map.of(), Map.of());+ stubVolumeListing(lifecycle.getDockerClient(), List.of(+ leftoverVolume("alpha", "alpha/4566", "previous"),+ leftoverVolume("beta", "alpha/4566", "previous")));+ doThrow(new IllegalStateException("busy")).when(lifecycle).removeVolumeStrict(anyString());+ EcsContainerManager manager = manager(lifecycle, namespacedConfig());+ List<LogRecord> records = LogCapture.capture(EcsContainerManager.class, () -> {+ assertFalse(manager.removeLeftoverTaskVolumes("current"));+ assertFalse(manager.removeLeftoverTaskVolumes("current"));+ doNothing().when(lifecycle).removeVolumeStrict("alpha");+ assertFalse(manager.removeLeftoverTaskVolumes("current"));+ doThrow(new IllegalStateException("busy again")).when(lifecycle).removeVolumeStrict("alpha");+ assertFalse(manager.removeLeftoverTaskVolumes("current"));+ });+ assertEquals(List.of(Level.WARNING.intValue(), Level.WARNING.intValue(),+ Level.FINE.intValue(), Level.FINE.intValue(), Level.FINE.intValue(),+ Level.WARNING.intValue(), Level.FINE.intValue()),+ records.stream().map(record -> record.getLevel().intValue()).toList());+ verify(lifecycle, times(4)).removeVolumeStrict("alpha");+ verify(lifecycle, times(4)).removeVolumeStrict("beta");+ verify(lifecycle, never()).removeVolume(anyString());+ }++ @Test+ void volumeListingWarnsOnceAndResetsAfterSuccessfulListing() {+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);+ stubLeftoverListing(lifecycle, Map.of(), Map.of());+ ListVolumesCmd command = lifecycle.getDockerClient().listVolumesCmd();+ ListVolumesResponse empty = mock(ListVolumesResponse.class);+ when(empty.getVolumes()).thenReturn(List.of());+ when(command.exec()).thenThrow(new IllegalStateException("offline"))+ .thenThrow(new IllegalStateException("offline"))+ .thenReturn(empty, empty).thenThrow(new IllegalStateException("offline again"));+ EcsContainerManager manager = manager(lifecycle, namespacedConfig());+ List<LogRecord> records = LogCapture.capture(EcsContainerManager.class, () -> {+ assertFalse(manager.removeLeftoverTaskVolumes("current"));+ assertFalse(manager.removeLeftoverTaskVolumes("current"));+ assertTrue(manager.removeLeftoverTaskVolumes("current"));+ assertFalse(manager.removeLeftoverTaskVolumes("current"));+ });+ assertEquals(List.of(Level.WARNING.intValue(), Level.FINE.intValue(), Level.WARNING.intValue()),+ records.stream().map(record -> record.getLevel().intValue()).toList());+ }++ private static InspectVolumeResponse leftoverVolume(String name, String owner, String run) {+ InspectVolumeResponse volume = mock(InspectVolumeResponse.class);+ when(volume.getName()).thenReturn(name);+ Map<String, String> labels = new LinkedHashMap<>();+ labels.put("io.floci.service", "ecs");+ labels.put("io.floci.owner", owner);+ if (run != null) {+ labels.put("io.floci.ecs.run", run);+ }+ when(volume.getLabels()).thenReturn(labels);+ return volume;+ }++ private static void stubVolumeListing(DockerClient docker, List<InspectVolumeResponse> volumes) {+ ListVolumesCmd command = mock(ListVolumesCmd.class, RETURNS_SELF);+ ListVolumesResponse response = mock(ListVolumesResponse.class);+ when(docker.listVolumesCmd()).thenReturn(command);+ when(command.exec()).thenReturn(response);+ when(response.getVolumes()).thenReturn(volumes);+ }+private static void stubLeftoverListing(ContainerLifecycleManager lifecycleManager,Map<String, Container> newKeys, Map<String, Container> legacyKeys) {DockerClient dockerClient = mock(DockerClient.class);when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ stubVolumeListing(dockerClient, List.of());ListContainersCmd listCmd = mock(ListContainersCmd.class, RETURNS_SELF);when(dockerClient.listContainersCmd()).thenReturn(listCmd);ListContainersCmd byNewKeys = mock(ListContainersCmd.class);diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.javaindex ea13049f97..a7eba5b702 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.ecs.container;+import com.github.dockerjava.api.DockerClient;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;@@ -22,21 +23,33 @@ import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;import io.github.hectorvent.floci.services.ssm.SsmService;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.MethodSource;+import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;import java.util.List;import java.util.Map;import java.util.Optional;import java.util.OptionalInt;+import java.util.stream.Stream;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyBoolean;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;import static org.mockito.Mockito.RETURNS_SELF;+import static org.mockito.Mockito.doNothing;+import static org.mockito.Mockito.doReturn;+import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;+import static org.mockito.Mockito.spy;import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@@ -61,6 +74,174 @@ class EcsContainerManagerVolumesTest {private EcrRegistryManager ecrRegistryManager;private EcsContainerManager manager;+ private static Stream<Volume> ephemeralVolumes() {+ return Stream.of(new Volume("shared", null),+ new Volume("shared", null, null, Map.of("host", Map.of())),+ new Volume("shared", "", null, Map.of("host", Map.of("sourcePath", ""))));+ }++ @ParameterizedTest+ @MethodSource("ephemeralVolumes")+ void ephemeralVolumeIsCreatedOnceAndSharedByContainersWithTheirAccessModes(Volume volume) {+ EcsTaskHandle handle = manager.startTask(task("000000000000", "us-east-1"),+ ephemeralDefinition(volume), List.of(), "us-east-1");++ ArgumentCaptor<String> volumeName = ArgumentCaptor.forClass(String.class);+ verify(builder).withNamedVolume(volumeName.capture(), eq("/data"), eq(false));+ verify(builder).withNamedVolume(volumeName.getValue(), "/data", true);+ verify(lifecycleManager).ensureVolume(eq(volumeName.getValue()), any());+ verify(lifecycleManager, times(1)).ensureVolume(anyString(), any());+ assertEquals(List.of(volumeName.getValue()), handle.getTaskVolumeNames().stream().toList());+ verify(builder, never()).withBind(any(), any());+ }++ @Test+ void taskVolumeIdentityIncludesAccountAndRegion() {+ TaskDefinition definition = ephemeralDefinition(new Volume("shared", null));+ manager.startTask(task("111111111111", "us-east-1"), definition, List.of(), "us-east-1");+ manager.startTask(task("222222222222", "us-east-1"), definition, List.of(), "us-east-1");+ manager.startTask(task("111111111111", "eu-west-1"), definition, List.of(), "eu-west-1");++ ArgumentCaptor<String> names = ArgumentCaptor.forClass(String.class);+ verify(lifecycleManager, times(3)).ensureVolume(names.capture(), any());+ assertEquals(3, names.getAllValues().stream().distinct().count());+ }++ @Test+ void unmountedAndUnsupportedVolumesAreNotCreatedAsEphemeralStorage() {+ ContainerDefinition app = new ContainerDefinition();+ app.setName("app");+ app.setImage("app:latest");+ app.setMountPoints(List.of(new MountPoint("external", "/external", false),+ new MountPoint("ebs", "/ebs", false)));+ TaskDefinition definition = new TaskDefinition();+ definition.setFamily("ephemeral-test");+ definition.setContainerDefinitions(List.of(app));+ definition.setVolumes(List.of(new Volume("unused", null),+ new Volume("external", null, null, Map.of("dockerVolumeConfiguration", Map.of("scope", "shared"))),+ new Volume("ebs", null, null, Map.of("configuredAtLaunch", true))));++ manager.startTask(task("000000000000", "us-east-1"), definition, List.of(), "us-east-1");++ verify(lifecycleManager, never()).ensureVolume(anyString(), any());+ verify(builder, never()).withNamedVolume(any(), any(), anyBoolean());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void failedLaunchRemovesItsEphemeralVolume(boolean firstContainerStarted) {+ if (firstContainerStarted) {+ when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()))+ .thenThrow(new IllegalStateException("launch failed"));+ } else {+ when(lifecycleManager.createAndStart(any())).thenThrow(new IllegalStateException("launch failed"));+ }+ assertThrows(IllegalStateException.class, () -> manager.startTask(task("000000000000", "us-east-1"),+ ephemeralDefinition(new Volume("shared", null)), List.of(), "us-east-1"));++ ArgumentCaptor<String> name = ArgumentCaptor.forClass(String.class);+ verify(lifecycleManager).ensureVolume(name.capture(), any());+ verify(lifecycleManager).removeVolume(name.getValue());+ }++ @Test+ void volumeRemovalIsRetriedAfterContainersAreGone() {+ when(lifecycleManager.getDockerClient()).thenReturn(mock(DockerClient.class, RETURNS_DEEP_STUBS));+ EcsContainerManager stopping = spy(manager);+ doReturn(0).when(stopping).getExitCodeIfStopped(anyString());+ doReturn(null).when(stopping).getFinishedAtIfStopped(anyString());+ EcsTaskHandle handle = stopping.startTask(task("000000000000", "us-east-1"),+ ephemeralDefinition(new Volume("shared", null)), List.of(), "us-east-1");+ String volumeName = handle.getTaskVolumeNames().iterator().next();+ when(lifecycleManager.removeVolume(volumeName)).thenReturn(false, true);++ stopping.stopTask(handle);+ assertTrue(handle.allContainersRemoved());+ assertTrue(handle.hasTaskVolumes());+ stopping.stopTask(handle);+ assertFalse(handle.hasTaskVolumes());+ verify(lifecycleManager, times(2)).removeVolume(volumeName);+ }++ @Test+ void volumeIsKeptUntilEveryContainerHasBeenRemoved() {+ EcsTaskHandle handle = manager.startTask(task("000000000000", "us-east-1"),+ ephemeralDefinition(new Volume("shared", null)), List.of(), "us-east-1");+ String name = handle.getTaskVolumeNames().iterator().next();+ doThrow(new IllegalStateException("remove failed")).when(lifecycleManager)+ .stopAndRemoveStrict("docker-id", null);++ manager.cleanupStoppedTask(handle);+ assertTrue(handle.hasTaskVolumes());+ verify(lifecycleManager, never()).removeVolume(name);++ doNothing().when(lifecycleManager).stopAndRemoveStrict("docker-id", null);+ when(lifecycleManager.removeVolume(name)).thenReturn(true);+ manager.cleanupStoppedTask(handle);+ assertFalse(handle.hasTaskVolumes());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void failedLaunchPreservesResourcesWhenCleanupFails(boolean containerRemovalFails) {+ when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()))+ .thenThrow(new IllegalStateException("launch failed"));+ if (containerRemovalFails) {+ doThrow(new IllegalStateException("remove failed")).when(lifecycleManager)+ .stopAndRemoveStrict("docker-id", null);+ } else {+ when(lifecycleManager.removeVolume(anyString())).thenReturn(false);+ }+ EcsTaskLaunchException failure = assertThrows(EcsTaskLaunchException.class,+ () -> manager.startTask(task("000000000000", "us-east-1"),+ ephemeralDefinition(new Volume("shared", null)), List.of(), "us-east-1"));+ EcsTaskHandle handle = failure.getHandle();+ assertTrue(handle.hasTaskVolumes());+ assertEquals(!containerRemovalFails, handle.allContainersRemoved());+ assertEquals("launch failed", failure.getMessage());++ doNothing().when(lifecycleManager).stopAndRemoveStrict("docker-id", null);+ when(lifecycleManager.removeVolume(anyString())).thenReturn(true);+ manager.cleanupStoppedTask(handle);+ assertTrue(handle.allContainersRemoved());+ assertFalse(handle.hasTaskVolumes());+ }++ @Test+ void recoveredVolumeNamesMatchLaunchAndExcludeUnusedAndExternalVolumes() {+ EcsTask task = task("000000000000", "us-east-1");+ TaskDefinition definition = ephemeralDefinition(new Volume("shared", null));+ EcsTaskHandle handle = manager.startTask(task, definition, List.of(), "us-east-1");+ assertEquals(handle.getTaskVolumeNames(), manager.taskVolumeNames(task, definition));+ definition.setVolumes(List.of(new Volume("shared", "/host/path")));+ assertTrue(manager.taskVolumeNames(task, definition).isEmpty());+ definition.setVolumes(List.of(new Volume("shared", null,+ new EfsVolumeConfiguration("fs-test", null, null, null, null, null))));+ assertTrue(manager.taskVolumeNames(task, definition).isEmpty());+ }++ private static TaskDefinition ephemeralDefinition(Volume volume) {+ ContainerDefinition writer = new ContainerDefinition();+ writer.setName("writer");+ writer.setImage("app:latest");+ writer.setMountPoints(List.of(new MountPoint("shared", "/data", false)));+ ContainerDefinition reader = new ContainerDefinition();+ reader.setName("reader");+ reader.setImage("app:latest");+ reader.setMountPoints(List.of(new MountPoint("shared", "/data", true)));+ TaskDefinition definition = new TaskDefinition();+ definition.setFamily("ephemeral-test");+ definition.setContainerDefinitions(List.of(writer, reader));+ definition.setVolumes(List.of(volume, new Volume("unused", null)));+ return definition;+ }++ private static EcsTask task(String account, String region) {+ EcsTask task = new EcsTask();+ task.setTaskArn("arn:aws:ecs:" + region + ":" + account + ":task/test-cluster/same-task-id");+ return task;+ }+@BeforeEachvoid setUp() {builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);@@ -69,6 +250,7 @@ class EcsContainerManagerVolumesTest {when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));lifecycleManager = mock(ContainerLifecycleManager.class);+ when(lifecycleManager.removeVolume(anyString())).thenReturn(true);when(lifecycleManager.resolveImageForLaunch(any(), any())).thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));when(lifecycleManager.createAndStart(any()))