Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): reconcile task lifecycle when container exits naturally (#1078)

Hector Ventura0655be9d237a
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsOneShotTaskLifecycleTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsOneShotTaskLifecycleTest.java
new file mode 100644
index 0000000000..c42e17c7e7
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsOneShotTaskLifecycleTest.java
@@ -0,0 +1,249 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.*;
+import software.amazon.awssdk.core.waiters.WaiterOverrideConfiguration;
+import software.amazon.awssdk.core.waiters.WaiterResponse;
+import software.amazon.awssdk.services.ecs.EcsClient;
+import software.amazon.awssdk.services.ecs.model.*;
+import software.amazon.awssdk.services.ecs.waiters.EcsWaiter;
+import software.amazon.awssdk.services.iam.IamClient;
+import software.amazon.awssdk.services.iam.model.CreateRoleRequest;
+
+import java.time.Duration;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.*;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+/**
+ * Verifies that a one-shot ECS task whose container exits naturally transitions
+ * to STOPPED, populates exitCode, and unblocks aws ecs wait tasks-stopped.
+ *
+ * Requires Docker mode (FLOCI_ECS_MOCK=false or default) and a reachable Docker
+ * socket. The test is skipped automatically when Lambda dispatch is unavailable,
+ * which is a reliable proxy for "Docker-in-Docker works in this environment".
+ */
+@DisplayName("ECS one-shot task lifecycle (container exits naturally)")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class EcsOneShotTaskLifecycleTest {
+
+ private static EcsClient ecs;
+ private static String clusterName;
+ private static String family;
+ private static String taskArn;
+ private static String clusterArn;
+
+ @BeforeAll
+ static void setup() {
+ assumeTrue(TestFixtures.isLambdaDispatchAvailable(),
+ "Skipping ECS one-shot lifecycle test: Docker dispatch not available in this environment");
+
+ ecs = TestFixtures.ecsClient();
+
+ String suffix = String.valueOf(System.currentTimeMillis() % 100000);
+ clusterName = "oneshot-cluster-" + suffix;
+ family = "oneshot-task-" + suffix;
+
+ // IAM role (required by the API; not enforced in Floci)
+ try (IamClient iam = TestFixtures.iamClient()) {
+ try {
+ iam.createRole(CreateRoleRequest.builder()
+ .roleName("ecs-oneshot-role-" + suffix)
+ .assumeRolePolicyDocument("""
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Principal":{"Service":"ecs-tasks.amazonaws.com"},
+ "Action":"sts:AssumeRole"}]}""")
+ .build());
+ } catch (Exception ignored) {
+ }
+ }
+
+ // Cluster
+ Cluster cluster = ecs.createCluster(CreateClusterRequest.builder()
+ .clusterName(clusterName)
+ .build()).cluster();
+ clusterArn = cluster.clusterArn();
+
+ // Task definition: busybox exits cleanly after ~2 s
+ ecs.registerTaskDefinition(RegisterTaskDefinitionRequest.builder()
+ .family(family)
+ .networkMode(NetworkMode.BRIDGE)
+ .cpu("256")
+ .memory("64")
+ .containerDefinitions(ContainerDefinition.builder()
+ .name("main")
+ .image("busybox:latest")
+ .command("sh", "-c", "echo hello && sleep 2 && echo done")
+ .essential(true)
+ .cpu(256)
+ .memory(64)
+ .build())
+ .build());
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (ecs == null) {
+ return;
+ }
+ try {
+ List<String> running = ecs.listTasks(ListTasksRequest.builder()
+ .cluster(clusterName)
+ .desiredStatus(DesiredStatus.RUNNING)
+ .build()).taskArns();
+ for (String arn : running) {
+ ecs.stopTask(StopTaskRequest.builder().cluster(clusterName).task(arn).build());
+ }
+ } catch (Exception ignored) {}
+ try {
+ ecs.deleteCluster(DeleteClusterRequest.builder().cluster(clusterName).build());
+ } catch (Exception ignored) {}
+ ecs.close();
+ }
+
+ // ── Tests ────────────────────────────────────────────────────────────────
+
+ @Test
+ @Order(1)
+ @DisplayName("RunTask - task starts RUNNING")
+ void runTask() {
+ List<Task> launched = ecs.runTask(RunTaskRequest.builder()
+ .cluster(clusterName)
+ .taskDefinition(family)
+ .launchType(LaunchType.FARGATE)
+ .count(1)
+ .build()).tasks();
+
+ assertThat(launched).hasSize(1);
+ taskArn = launched.get(0).taskArn();
+ assertThat(launched.get(0).lastStatus()).isEqualTo("RUNNING");
+ }
+
+ @Test
+ @Order(2)
+ @DisplayName("WaitTasksStopped - transitions to STOPPED within 60 s")
+ void waitForTaskStopped() {
+ // The container runs for ~2 s; the reconciler fires every 5 s; the waiter
+ // polls every 6 s. Allow 60 s total to be CI-safe.
+ WaiterOverrideConfiguration overrides = WaiterOverrideConfiguration.builder()
+ .waitTimeout(Duration.ofSeconds(60))
+ .build();
+
+ try (EcsWaiter waiter = EcsWaiter.builder().client(ecs).build()) {
+ WaiterResponse<DescribeTasksResponse> response = waiter.waitUntilTasksStopped(
+ DescribeTasksRequest.builder()
+ .cluster(clusterName)
+ .tasks(taskArn)
+ .build(),
+ overrides);
+
+ assertThat(response.matched().response()).isPresent();
+ Task task = response.matched().response().get().tasks().get(0);
+ assertThat(task.lastStatus()).isEqualTo("STOPPED");
+ }
+ }
+
+ @Test
+ @Order(3)
+ @DisplayName("DescribeTasks - stoppedAt is populated after natural exit")
+ void stoppedAtPopulated() {
+ Task task = describeSingle();
+ assertThat(task.stoppedAt())
+ .as("stoppedAt must be set after a natural container exit")
+ .isNotNull();
+ }
+
+ @Test
+ @Order(4)
+ @DisplayName("DescribeTasks - exitCode is 0 for clean container exit")
+ void exitCodePopulated() {
+ Task task = describeSingle();
+ assertThat(task.containers()).isNotEmpty();
+ Container main = task.containers().stream()
+ .filter(c -> "main".equals(c.name()))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("container 'main' not found"));
+
+ assertThat(main.exitCode())
+ .as("exitCode must be 0 for a container that exited cleanly")
+ .isEqualTo(0);
+ }
+
+ @Test
+ @Order(5)
+ @DisplayName("DescribeTasks - container lastStatus is STOPPED")
+ void containerStatusStopped() {
+ Task task = describeSingle();
+ task.containers().forEach(c ->
+ assertThat(c.lastStatus())
+ .as("container %s lastStatus", c.name())
+ .isEqualTo("STOPPED"));
+ }
+
+ @Test
+ @Order(6)
+ @DisplayName("DescribeClusters - runningTasksCount decremented after exit")
+ void runningCountDecremented() {
+ Cluster cluster = ecs.describeClusters(DescribeClustersRequest.builder()
+ .clusters(clusterName)
+ .build()).clusters().get(0);
+
+ assertThat(cluster.runningTasksCount())
+ .as("runningTasksCount should be 0 after the one-shot task stops")
+ .isEqualTo(0);
+ }
+
+ @Test
+ @Order(7)
+ @DisplayName("StopTask - explicit stop populates exitCode when already stopped")
+ void explicitStopExitCode() {
+ // Register and run a second task; stop it explicitly before it exits naturally.
+ String family2 = family + "-explicit";
+ ecs.registerTaskDefinition(RegisterTaskDefinitionRequest.builder()
+ .family(family2)
+ .networkMode(NetworkMode.BRIDGE)
+ .cpu("256")
+ .memory("64")
+ .containerDefinitions(ContainerDefinition.builder()
+ .name("main")
+ .image("busybox:latest")
+ .command("sh", "-c", "sleep 300")
+ .essential(true)
+ .cpu(256)
+ .memory(64)
+ .build())
+ .build());
+
+ String explicitTaskArn = ecs.runTask(RunTaskRequest.builder()
+ .cluster(clusterName)
+ .taskDefinition(family2)
+ .launchType(LaunchType.FARGATE)
+ .count(1)
+ .build()).tasks().get(0).taskArn();
+
+ Task stopped = ecs.stopTask(StopTaskRequest.builder()
+ .cluster(clusterName)
+ .task(explicitTaskArn)
+ .reason("test-explicit-stop")
+ .build()).task();
+
+ assertThat(stopped.lastStatus()).isEqualTo("STOPPED");
+ assertThat(stopped.stoppedAt()).isNotNull();
+
+ // exitCode should be present (Docker assigns 137 for SIGKILL on explicit stop)
+ assertThat(stopped.containers()).isNotEmpty();
+ stopped.containers().forEach(c ->
+ assertThat(c.exitCode())
+ .as("explicit stopTask must populate exitCode on container %s", c.name())
+ .isNotNull());
+ }
+
+ // ── Helpers ──────────────────────────────────────────────────────────────
+
+ private Task describeSingle() {
+ return ecs.describeTasks(DescribeTasksRequest.builder()
+ .cluster(clusterName)
+ .tasks(taskArn)
+ .build()).tasks().get(0);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index 3803825a76..266ef9f69b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -30,6 +30,7 @@ import org.jboss.logging.Logger;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -92,7 +93,7 @@ public class EcsService {
@PostConstruct
void init() {
- reconciler.scheduleAtFixedRate(this::reconcileServices, 5, 5, TimeUnit.SECONDS);
+ reconciler.scheduleAtFixedRate(this::reconcile, 5, 5, TimeUnit.SECONDS);
}
@PreDestroy
@@ -316,21 +317,32 @@ public class EcsService {
public EcsTask stopTask(String clusterRef, String taskRef, String reason, String region) {
EcsTask task = resolveTaskOrThrow(taskRef, region);
+ if (TaskStatus.STOPPED.name().equals(task.getLastStatus())) {
+ return task;
+ }
task.setDesiredStatus(TaskStatus.STOPPED.name());
task.setLastStatus(TaskStatus.STOPPING.name());
task.setStoppedReason(reason != null ? reason : "Stopped by user");
deregisterTaskFromLoadBalancers(task, region);
+ Map<String, Integer> exitCodes = Map.of();
if (dockerMode) {
EcsTaskHandle handle = taskHandles.remove(task.getTaskArn());
- containerManager.stopTask(handle);
+ exitCodes = containerManager.stopTaskAndCollectExitCodes(handle);
}
task.setLastStatus(TaskStatus.STOPPED.name());
task.setStoppedAt(Instant.now());
if (task.getContainers() != null) {
- task.getContainers().forEach(c -> c.setLastStatus("STOPPED"));
+ final Map<String, Integer> codes = exitCodes;
+ task.getContainers().forEach(c -> {
+ c.setLastStatus("STOPPED");
+ Integer code = codes.get(c.getName());
+ if (code != null) {
+ c.setExitCode(code);
+ }
+ });
}
EcsCluster cluster = resolveClusterByArn(task.getClusterArn());
@@ -1000,6 +1012,75 @@ public class EcsService {
// ── Service Reconciliation ────────────────────────────────────────────────
+ void reconcile() {
+ reconcileTasks();
+ reconcileServices();
+ }
+
+ private void reconcileTasks() {
+ for (String taskArn : taskHandles.keySet()) {
+ try {
+ reconcileTask(taskArn);
+ } catch (Exception e) {
+ LOG.debugv("Error reconciling ECS task {0}: {1}", taskArn, e.getMessage());
+ }
+ }
+ }
+
+ private void reconcileTask(String taskArn) {
+ EcsTask task = tasks.get(taskArn);
+ if (task == null || !TaskStatus.RUNNING.name().equals(task.getLastStatus())) {
+ return;
+ }
+
+ EcsTaskHandle handle = taskHandles.get(taskArn);
+ if (handle == null) {
+ return;
+ }
+
+ // Inspect every container; abort if any are still running.
+ Map<String, Integer> exitCodes = new LinkedHashMap<>();
+ for (Map.Entry<String, String> entry : handle.getContainerIds().entrySet()) {
+ Integer code = containerManager.getExitCodeIfStopped(entry.getValue());
+ if (code == null) {
+ return;
+ }
+ exitCodes.put(entry.getKey(), code);
+ }
+
+ // All containers have exited. Atomically claim the handle to avoid
+ // racing with an explicit stopTask() call.
+ EcsTaskHandle claimed = taskHandles.remove(taskArn);
+ if (claimed == null) {
+ return;
+ }
+
+ // Close log streams and remove the stopped Docker containers without re-inspecting.
+ containerManager.cleanupStoppedTask(claimed);
+
+ if (task.getContainers() != null) {
+ task.getContainers().forEach(c -> {
+ c.setLastStatus("STOPPED");
+ Integer code = exitCodes.get(c.getName());
+ if (code != null) {
+ c.setExitCode(code);
+ }
+ });
+ }
+
+ task.setLastStatus(TaskStatus.STOPPED.name());
+ task.setDesiredStatus(TaskStatus.STOPPED.name());
+ task.setStoppedAt(Instant.now());
+ task.setStoppedReason("Essential container in task exited");
+
+ EcsCluster cluster = resolveClusterByArn(task.getClusterArn());
+ if (cluster != null && cluster.getRunningTasksCount() > 0) {
+ cluster.setRunningTasksCount(cluster.getRunningTasksCount() - 1);
+ }
+
+ LOG.infov("ECS task {0} reconciled to STOPPED (all containers exited)", taskArn);
+ }
+
void reconcileServices() {
for (Map.Entry<String, EcsServiceModel> entry : services.entrySet()) {
try {
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
index 65ea65d54f..eb7edeb040 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.services.ecs.model.NetworkBinding;
import io.github.hectorvent.floci.services.ecs.model.PortMapping;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.ExposedPort;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -145,11 +146,39 @@ public class EcsContainerManager {
* Stops and removes all Docker containers for a task.
*/
public void stopTask(EcsTaskHandle handle) {
+ stopTaskAndCollectExitCodes(handle);
+ }
+
+ /**
+ * Closes log streams and removes already-stopped containers without re-inspecting exit codes.
+ * Use this from the reconciler when exit codes have already been collected.
+ */
+ public void cleanupStoppedTask(EcsTaskHandle handle) {
if (handle == null) {
return;
}
+ for (Closeable logStream : handle.getLogStreams()) {
+ try {
+ logStream.close();
+ } catch (Exception ignored) {
+ }
+ }
+ for (String dockerId : handle.getContainerIds().values()) {
+ lifecycleManager.stopAndRemove(dockerId, null);
+ }
+ }
+
+ /**
+ * Stops all containers, captures their exit codes, then removes them.
+ * Stop happens before inspect so the exit code reflects the actual stop
+ * signal (e.g. 137 for SIGKILL), not a stale pre-stop value.
+ */
+ public Map<String, Integer> stopTaskAndCollectExitCodes(EcsTaskHandle handle) {
+ Map<String, Integer> exitCodes = new LinkedHashMap<>();
+ if (handle == null) {
+ return exitCodes;
+ }
- // Close all log streams first
for (Closeable logStream : handle.getLogStreams()) {
try {
logStream.close();
@@ -157,9 +186,49 @@ public class EcsContainerManager {
}
}
- // Stop and remove all containers
+ // Phase 1: stop all containers (no-op for those already exited).
+ for (String dockerId : handle.getContainerIds().values()) {
+ try {
+ lifecycleManager.getDockerClient().stopContainerCmd(dockerId).withTimeout(5).exec();
+ } catch (NotFoundException ignored) {
+ } catch (Exception e) {
+ LOG.warnv("Error stopping ECS container {0}: {1}", dockerId, e.getMessage());
+ }
+ }
+
+ // Phase 2: inspect exit codes, then remove.
for (Map.Entry<String, String> entry : handle.getContainerIds().entrySet()) {
- lifecycleManager.stopAndRemove(entry.getValue(), null);
+ String name = entry.getKey();
+ String dockerId = entry.getValue();
+ exitCodes.put(name, getExitCodeIfStopped(dockerId));
+ try {
+ lifecycleManager.getDockerClient().removeContainerCmd(dockerId).withForce(true).exec();
+ } catch (NotFoundException ignored) {
+ } catch (Exception e) {
+ LOG.warnv("Error removing ECS container {0}: {1}", dockerId, e.getMessage());
+ }
+ }
+ return exitCodes;
+ }
+
+ /**
+ * Returns the exit code of a container that has already stopped, or {@code null}
+ * if the container is still running or its state cannot be determined.
+ * A missing container (externally removed) is treated as a clean exit (code 0).
+ */
+ public Integer getExitCodeIfStopped(String dockerId) {
+ try {
+ var inspect = lifecycleManager.getDockerClient().inspectContainerCmd(dockerId).exec();
+ if (Boolean.TRUE.equals(inspect.getState().getRunning())) {
+ return null;
+ }
+ Long code = inspect.getState().getExitCodeLong();
+ return code != null ? code.intValue() : null;
+ } catch (NotFoundException e) {
+ return 0;
+ } catch (Exception e) {
+ LOG.debugv("Could not inspect container {0}: {1}", dockerId, e.getMessage());
+ return null;
}
}