Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): reconcile task lifecycle when container exits naturally (#1078)
Hector Ventura0655be9d237a
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsOneShotTaskLifecycleTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsOneShotTaskLifecycleTest.javanew file mode 100644index 0000000000..c42e17c7e7--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsOneShotTaskLifecycleTest.java@@ -0,0 +1,249 @@+package com.floci.test;++import org.junit.jupiter.api.*;+import software.amazon.awssdk.core.waiters.WaiterOverrideConfiguration;+import software.amazon.awssdk.core.waiters.WaiterResponse;+import software.amazon.awssdk.services.ecs.EcsClient;+import software.amazon.awssdk.services.ecs.model.*;+import software.amazon.awssdk.services.ecs.waiters.EcsWaiter;+import software.amazon.awssdk.services.iam.IamClient;+import software.amazon.awssdk.services.iam.model.CreateRoleRequest;++import java.time.Duration;+import java.util.List;++import static org.assertj.core.api.Assertions.*;+import static org.junit.jupiter.api.Assumptions.assumeTrue;++/**+ * Verifies that a one-shot ECS task whose container exits naturally transitions+ * to STOPPED, populates exitCode, and unblocks aws ecs wait tasks-stopped.+ *+ * Requires Docker mode (FLOCI_ECS_MOCK=false or default) and a reachable Docker+ * socket. The test is skipped automatically when Lambda dispatch is unavailable,+ * which is a reliable proxy for "Docker-in-Docker works in this environment".+ */+@DisplayName("ECS one-shot task lifecycle (container exits naturally)")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class EcsOneShotTaskLifecycleTest {++ private static EcsClient ecs;+ private static String clusterName;+ private static String family;+ private static String taskArn;+ private static String clusterArn;++ @BeforeAll+ static void setup() {+ assumeTrue(TestFixtures.isLambdaDispatchAvailable(),+ "Skipping ECS one-shot lifecycle test: Docker dispatch not available in this environment");++ ecs = TestFixtures.ecsClient();++ String suffix = String.valueOf(System.currentTimeMillis() % 100000);+ clusterName = "oneshot-cluster-" + suffix;+ family = "oneshot-task-" + suffix;++ // IAM role (required by the API; not enforced in Floci)+ try (IamClient iam = TestFixtures.iamClient()) {+ try {+ iam.createRole(CreateRoleRequest.builder()+ .roleName("ecs-oneshot-role-" + suffix)+ .assumeRolePolicyDocument("""+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",+ "Principal":{"Service":"ecs-tasks.amazonaws.com"},+ "Action":"sts:AssumeRole"}]}""")+ .build());+ } catch (Exception ignored) {+ }+ }++ // Cluster+ Cluster cluster = ecs.createCluster(CreateClusterRequest.builder()+ .clusterName(clusterName)+ .build()).cluster();+ clusterArn = cluster.clusterArn();++ // Task definition: busybox exits cleanly after ~2 s+ ecs.registerTaskDefinition(RegisterTaskDefinitionRequest.builder()+ .family(family)+ .networkMode(NetworkMode.BRIDGE)+ .cpu("256")+ .memory("64")+ .containerDefinitions(ContainerDefinition.builder()+ .name("main")+ .image("busybox:latest")+ .command("sh", "-c", "echo hello && sleep 2 && echo done")+ .essential(true)+ .cpu(256)+ .memory(64)+ .build())+ .build());+ }++ @AfterAll+ static void cleanup() {+ if (ecs == null) {+ return;+ }+ try {+ List<String> running = ecs.listTasks(ListTasksRequest.builder()+ .cluster(clusterName)+ .desiredStatus(DesiredStatus.RUNNING)+ .build()).taskArns();+ for (String arn : running) {+ ecs.stopTask(StopTaskRequest.builder().cluster(clusterName).task(arn).build());+ }+ } catch (Exception ignored) {}+ try {+ ecs.deleteCluster(DeleteClusterRequest.builder().cluster(clusterName).build());+ } catch (Exception ignored) {}+ ecs.close();+ }++ // ── Tests ────────────────────────────────────────────────────────────────++ @Test+ @Order(1)+ @DisplayName("RunTask - task starts RUNNING")+ void runTask() {+ List<Task> launched = ecs.runTask(RunTaskRequest.builder()+ .cluster(clusterName)+ .taskDefinition(family)+ .launchType(LaunchType.FARGATE)+ .count(1)+ .build()).tasks();++ assertThat(launched).hasSize(1);+ taskArn = launched.get(0).taskArn();+ assertThat(launched.get(0).lastStatus()).isEqualTo("RUNNING");+ }++ @Test+ @Order(2)+ @DisplayName("WaitTasksStopped - transitions to STOPPED within 60 s")+ void waitForTaskStopped() {+ // The container runs for ~2 s; the reconciler fires every 5 s; the waiter+ // polls every 6 s. Allow 60 s total to be CI-safe.+ WaiterOverrideConfiguration overrides = WaiterOverrideConfiguration.builder()+ .waitTimeout(Duration.ofSeconds(60))+ .build();++ try (EcsWaiter waiter = EcsWaiter.builder().client(ecs).build()) {+ WaiterResponse<DescribeTasksResponse> response = waiter.waitUntilTasksStopped(+ DescribeTasksRequest.builder()+ .cluster(clusterName)+ .tasks(taskArn)+ .build(),+ overrides);++ assertThat(response.matched().response()).isPresent();+ Task task = response.matched().response().get().tasks().get(0);+ assertThat(task.lastStatus()).isEqualTo("STOPPED");+ }+ }++ @Test+ @Order(3)+ @DisplayName("DescribeTasks - stoppedAt is populated after natural exit")+ void stoppedAtPopulated() {+ Task task = describeSingle();+ assertThat(task.stoppedAt())+ .as("stoppedAt must be set after a natural container exit")+ .isNotNull();+ }++ @Test+ @Order(4)+ @DisplayName("DescribeTasks - exitCode is 0 for clean container exit")+ void exitCodePopulated() {+ Task task = describeSingle();+ assertThat(task.containers()).isNotEmpty();+ Container main = task.containers().stream()+ .filter(c -> "main".equals(c.name()))+ .findFirst()+ .orElseThrow(() -> new AssertionError("container 'main' not found"));++ assertThat(main.exitCode())+ .as("exitCode must be 0 for a container that exited cleanly")+ .isEqualTo(0);+ }++ @Test+ @Order(5)+ @DisplayName("DescribeTasks - container lastStatus is STOPPED")+ void containerStatusStopped() {+ Task task = describeSingle();+ task.containers().forEach(c ->+ assertThat(c.lastStatus())+ .as("container %s lastStatus", c.name())+ .isEqualTo("STOPPED"));+ }++ @Test+ @Order(6)+ @DisplayName("DescribeClusters - runningTasksCount decremented after exit")+ void runningCountDecremented() {+ Cluster cluster = ecs.describeClusters(DescribeClustersRequest.builder()+ .clusters(clusterName)+ .build()).clusters().get(0);++ assertThat(cluster.runningTasksCount())+ .as("runningTasksCount should be 0 after the one-shot task stops")+ .isEqualTo(0);+ }++ @Test+ @Order(7)+ @DisplayName("StopTask - explicit stop populates exitCode when already stopped")+ void explicitStopExitCode() {+ // Register and run a second task; stop it explicitly before it exits naturally.+ String family2 = family + "-explicit";+ ecs.registerTaskDefinition(RegisterTaskDefinitionRequest.builder()+ .family(family2)+ .networkMode(NetworkMode.BRIDGE)+ .cpu("256")+ .memory("64")+ .containerDefinitions(ContainerDefinition.builder()+ .name("main")+ .image("busybox:latest")+ .command("sh", "-c", "sleep 300")+ .essential(true)+ .cpu(256)+ .memory(64)+ .build())+ .build());++ String explicitTaskArn = ecs.runTask(RunTaskRequest.builder()+ .cluster(clusterName)+ .taskDefinition(family2)+ .launchType(LaunchType.FARGATE)+ .count(1)+ .build()).tasks().get(0).taskArn();++ Task stopped = ecs.stopTask(StopTaskRequest.builder()+ .cluster(clusterName)+ .task(explicitTaskArn)+ .reason("test-explicit-stop")+ .build()).task();++ assertThat(stopped.lastStatus()).isEqualTo("STOPPED");+ assertThat(stopped.stoppedAt()).isNotNull();++ // exitCode should be present (Docker assigns 137 for SIGKILL on explicit stop)+ assertThat(stopped.containers()).isNotEmpty();+ stopped.containers().forEach(c ->+ assertThat(c.exitCode())+ .as("explicit stopTask must populate exitCode on container %s", c.name())+ .isNotNull());+ }++ // ── Helpers ──────────────────────────────────────────────────────────────++ private Task describeSingle() {+ return ecs.describeTasks(DescribeTasksRequest.builder()+ .cluster(clusterName)+ .tasks(taskArn)+ .build()).tasks().get(0);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex 3803825a76..266ef9f69b 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -30,6 +30,7 @@ import org.jboss.logging.Logger;import java.time.Instant;import java.util.ArrayList;+import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.UUID;@@ -92,7 +93,7 @@ public class EcsService {@PostConstructvoid init() {- reconciler.scheduleAtFixedRate(this::reconcileServices, 5, 5, TimeUnit.SECONDS);+ reconciler.scheduleAtFixedRate(this::reconcile, 5, 5, TimeUnit.SECONDS);}@PreDestroy@@ -316,21 +317,32 @@ public class EcsService {public EcsTask stopTask(String clusterRef, String taskRef, String reason, String region) {EcsTask task = resolveTaskOrThrow(taskRef, region);+ if (TaskStatus.STOPPED.name().equals(task.getLastStatus())) {+ return task;+ }task.setDesiredStatus(TaskStatus.STOPPED.name());task.setLastStatus(TaskStatus.STOPPING.name());task.setStoppedReason(reason != null ? reason : "Stopped by user");deregisterTaskFromLoadBalancers(task, region);+ Map<String, Integer> exitCodes = Map.of();if (dockerMode) {EcsTaskHandle handle = taskHandles.remove(task.getTaskArn());- containerManager.stopTask(handle);+ exitCodes = containerManager.stopTaskAndCollectExitCodes(handle);}task.setLastStatus(TaskStatus.STOPPED.name());task.setStoppedAt(Instant.now());if (task.getContainers() != null) {- task.getContainers().forEach(c -> c.setLastStatus("STOPPED"));+ final Map<String, Integer> codes = exitCodes;+ task.getContainers().forEach(c -> {+ c.setLastStatus("STOPPED");+ Integer code = codes.get(c.getName());+ if (code != null) {+ c.setExitCode(code);+ }+ });}EcsCluster cluster = resolveClusterByArn(task.getClusterArn());@@ -1000,6 +1012,75 @@ public class EcsService {// ── Service Reconciliation ────────────────────────────────────────────────+ void reconcile() {+ reconcileTasks();+ reconcileServices();+ }++ private void reconcileTasks() {+ for (String taskArn : taskHandles.keySet()) {+ try {+ reconcileTask(taskArn);+ } catch (Exception e) {+ LOG.debugv("Error reconciling ECS task {0}: {1}", taskArn, e.getMessage());+ }+ }+ }++ private void reconcileTask(String taskArn) {+ EcsTask task = tasks.get(taskArn);+ if (task == null || !TaskStatus.RUNNING.name().equals(task.getLastStatus())) {+ return;+ }++ EcsTaskHandle handle = taskHandles.get(taskArn);+ if (handle == null) {+ return;+ }++ // Inspect every container; abort if any are still running.+ Map<String, Integer> exitCodes = new LinkedHashMap<>();+ for (Map.Entry<String, String> entry : handle.getContainerIds().entrySet()) {+ Integer code = containerManager.getExitCodeIfStopped(entry.getValue());+ if (code == null) {+ return;+ }+ exitCodes.put(entry.getKey(), code);+ }++ // All containers have exited. Atomically claim the handle to avoid+ // racing with an explicit stopTask() call.+ EcsTaskHandle claimed = taskHandles.remove(taskArn);+ if (claimed == null) {+ return;+ }++ // Close log streams and remove the stopped Docker containers without re-inspecting.+ containerManager.cleanupStoppedTask(claimed);++ if (task.getContainers() != null) {+ task.getContainers().forEach(c -> {+ c.setLastStatus("STOPPED");+ Integer code = exitCodes.get(c.getName());+ if (code != null) {+ c.setExitCode(code);+ }+ });+ }++ task.setLastStatus(TaskStatus.STOPPED.name());+ task.setDesiredStatus(TaskStatus.STOPPED.name());+ task.setStoppedAt(Instant.now());+ task.setStoppedReason("Essential container in task exited");++ EcsCluster cluster = resolveClusterByArn(task.getClusterArn());+ if (cluster != null && cluster.getRunningTasksCount() > 0) {+ cluster.setRunningTasksCount(cluster.getRunningTasksCount() - 1);+ }++ LOG.infov("ECS task {0} reconciled to STOPPED (all containers exited)", taskArn);+ }+void reconcileServices() {for (Map.Entry<String, EcsServiceModel> entry : services.entrySet()) {try {diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.javaindex 65ea65d54f..eb7edeb040 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.services.ecs.model.NetworkBinding;import io.github.hectorvent.floci.services.ecs.model.PortMapping;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.exception.NotFoundException;import com.github.dockerjava.api.model.ExposedPort;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -145,11 +146,39 @@ public class EcsContainerManager {* Stops and removes all Docker containers for a task.*/public void stopTask(EcsTaskHandle handle) {+ stopTaskAndCollectExitCodes(handle);+ }++ /**+ * Closes log streams and removes already-stopped containers without re-inspecting exit codes.+ * Use this from the reconciler when exit codes have already been collected.+ */+ public void cleanupStoppedTask(EcsTaskHandle handle) {if (handle == null) {return;}+ for (Closeable logStream : handle.getLogStreams()) {+ try {+ logStream.close();+ } catch (Exception ignored) {+ }+ }+ for (String dockerId : handle.getContainerIds().values()) {+ lifecycleManager.stopAndRemove(dockerId, null);+ }+ }++ /**+ * Stops all containers, captures their exit codes, then removes them.+ * Stop happens before inspect so the exit code reflects the actual stop+ * signal (e.g. 137 for SIGKILL), not a stale pre-stop value.+ */+ public Map<String, Integer> stopTaskAndCollectExitCodes(EcsTaskHandle handle) {+ Map<String, Integer> exitCodes = new LinkedHashMap<>();+ if (handle == null) {+ return exitCodes;+ }- // Close all log streams firstfor (Closeable logStream : handle.getLogStreams()) {try {logStream.close();@@ -157,9 +186,49 @@ public class EcsContainerManager {}}- // Stop and remove all containers+ // Phase 1: stop all containers (no-op for those already exited).+ for (String dockerId : handle.getContainerIds().values()) {+ try {+ lifecycleManager.getDockerClient().stopContainerCmd(dockerId).withTimeout(5).exec();+ } catch (NotFoundException ignored) {+ } catch (Exception e) {+ LOG.warnv("Error stopping ECS container {0}: {1}", dockerId, e.getMessage());+ }+ }++ // Phase 2: inspect exit codes, then remove.for (Map.Entry<String, String> entry : handle.getContainerIds().entrySet()) {- lifecycleManager.stopAndRemove(entry.getValue(), null);+ String name = entry.getKey();+ String dockerId = entry.getValue();+ exitCodes.put(name, getExitCodeIfStopped(dockerId));+ try {+ lifecycleManager.getDockerClient().removeContainerCmd(dockerId).withForce(true).exec();+ } catch (NotFoundException ignored) {+ } catch (Exception e) {+ LOG.warnv("Error removing ECS container {0}: {1}", dockerId, e.getMessage());+ }+ }+ return exitCodes;+ }++ /**+ * Returns the exit code of a container that has already stopped, or {@code null}+ * if the container is still running or its state cannot be determined.+ * A missing container (externally removed) is treated as a clean exit (code 0).+ */+ public Integer getExitCodeIfStopped(String dockerId) {+ try {+ var inspect = lifecycleManager.getDockerClient().inspectContainerCmd(dockerId).exec();+ if (Boolean.TRUE.equals(inspect.getState().getRunning())) {+ return null;+ }+ Long code = inspect.getState().getExitCodeLong();+ return code != null ? code.intValue() : null;+ } catch (NotFoundException e) {+ return 0;+ } catch (Exception e) {+ LOG.debugv("Could not inspect container {0}: {1}", dockerId, e.getMessage());+ return null;}}