Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2): retry Docker port collisions (#2029)
Aniket Shuklab27b32f044da
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.javaindex 1bdcd3431b..d282887a84 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java@@ -1452,6 +1452,9 @@ public class CloudFormationResourceProvisioner {var instance = reservation.getInstances().get(0);r.setPhysicalId(instance.getInstanceId());r.getAttributes().put("InstanceId", instance.getInstanceId());+ r.getAttributes().put(CfnRollback.ROLLBACK_OWNED_ATTR, "true");+ ec2Service.awaitContainerLaunch(instance);+ r.getAttributes().remove(CfnRollback.ROLLBACK_OWNED_ATTR);if (instance.getPrivateIpAddress() != null) {r.getAttributes().put("PrivateIp", instance.getPrivateIpAddress());}diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.javaindex e7fefefaf9..e110877d84 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java@@ -34,6 +34,7 @@ import java.util.ArrayList;import java.util.List;import java.util.Locale;import java.util.Map;+import java.util.Objects;import java.util.Optional;import java.util.Set;import java.util.concurrent.CountDownLatch;@@ -47,6 +48,8 @@ import java.util.regex.Pattern;* Manages Docker container lifecycle for EC2 instances.* Handles launch, stop, start, terminate, and reboot operations.* SSH key injection and UserData execution are performed asynchronously after launch.+ * A running instance has a reachable container address, while best-effort link-local IMDS setup,+ * SSH initialization, and UserData may still be completing in the launch worker.*/@ApplicationScopedpublic class Ec2ContainerManager {@@ -134,55 +137,28 @@ public class Ec2ContainerManager {executor.submit(() -> {try {String instanceId = instance.getInstanceId();- String containerName = ContainerStorageHelper.resourceName(config, "ec2", null, instanceId);-- // Allocate SSH host port- int sshHostPort = portAllocator.allocate(- config.services().ec2().sshPortRangeStart(),- config.services().ec2().sshPortRangeEnd());- instance.setSshHostPort(sshHostPort);-// IMDS endpoint that this container should useString flociHost = dockerHostResolver.resolve();int imdsPort = config.services().ec2().imdsPort();- String imdsEndpoint = "http://" + flociHost + ":" + imdsPort;- String serviceEndpoint = "http://" + flociHost + ":4566";-- // Build container spec — minimal images keep the historic tail- // command, while cloud-image AMI guests can boot their init.- ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image.dockerImage())- .withName(containerName)- .withEmbeddedDns()- .withDockerNetwork(Optional.empty())- .withEnv(localAwsEnvironment(region, serviceEndpoint, imdsEndpoint))- .withEnv("AWS_EC2_INSTANCE_ID", instanceId)- .withPortBinding(22, sshHostPort)- .withHostDockerInternalOnLinux()- .withLogRotation()- // EC2 instances expose IMDS on 169.254.169.254. Floci- // needs network administration privileges in the local- // container to attach that link-local address.- .withPrivileged(true)- .withCmd(image.systemd() ? List.of("/sbin/init") : List.of("tail", "-f", "/dev/null"));- if (image.systemd()) {- specBuilder- .withCgroupnsMode("host")- .withMount(new Mount().withType(MountType.TMPFS).withTarget("/run"))- .withMount(new Mount().withType(MountType.TMPFS).withTarget("/run/lock"))- .withBind("/sys/fs/cgroup", "/sys/fs/cgroup");+ StartedContainer started = createAndStartContainer(instance, image, region, flociHost, imdsPort);+ if (started == null) {+ return;}- ContainerSpec spec = specBuilder.build();+ int sshHostPort = started.sshHostPort();+ String containerId = started.containerId();- // Create container without starting it- String containerId = lifecycleManager.create(spec);- instance.setDockerContainerId(containerId);-- // Start the container- lifecycleManager.startCreated(containerId, spec);+ if (isLaunchCancelled(instance)) {+ failLaunch(instance);+ return;+ }// Poll until Docker confirms the container is runningboolean running = false;for (int i = 0; i < 30 && !running; i++) {+ if (isLaunchCancelled(instance)) {+ failLaunch(instance);+ return;+ }running = lifecycleManager.isContainerRunning(containerId);if (!running) {Thread.sleep(500);@@ -191,7 +167,12 @@ public class Ec2ContainerManager {if (!running) {LOG.warnv("EC2 instance {0} container {1} did not reach running state", instanceId, containerId);- instance.setState(InstanceState.terminated());+ failLaunch(instance);+ return;+ }++ if (isLaunchCancelled(instance)) {+ failLaunch(instance);return;}@@ -199,7 +180,7 @@ public class Ec2ContainerManager {// Docker can report the container as running before network// settings are populated; wait here so IMDS is registered// before link-local metadata validation and UserData run.- String containerIp = waitForContainerBridgeIp(containerId, instanceId);+ String containerIp = waitForContainerBridgeIp(containerId, instanceId, instance);if (containerIp != null && !containerIp.isBlank()) {instance.setContainerBridgeIp(containerIp);exposeReachablePrivateAddress(instance, containerIp, config.services().ec2().awsFaithfulPrivateIp());@@ -208,11 +189,14 @@ public class Ec2ContainerManager {else {LOG.warnv("EC2 instance {0} container {1} did not receive a usable bridge IP for IMDS",instanceId, containerId);- instance.setState(InstanceState.terminated());+ failLaunch(instance);return;}- configureLinkLocalMetadataEndpoint(containerId, instanceId, flociHost, imdsPort);+ if (!markRunning(instance)) {+ failLaunch(instance);+ return;+ }// Set public-facing addresses only for instances whose subnet// opts in via MapPublicIpOnLaunch (#1984). Private-subnet@@ -223,10 +207,13 @@ public class Ec2ContainerManager {instance.setPublicDnsName("localhost");}- instance.setState(InstanceState.running());LOG.infov("EC2 instance {0} running in container {1} (SSH host port {2})",instanceId, containerId, String.valueOf(sshHostPort));+ // IMDS proxy setup is best effort and has its own bounded commands. The instance+ // is already running once Docker has assigned its reachable network address.+ configureLinkLocalMetadataEndpoint(containerId, instanceId, flociHost, imdsPort);+// Publish security-group TCP ingress ports on the host via socat sidecars.if (appPorts != null && !appPorts.isEmpty()) {portForwardManager.reconcile(instance, appPorts);@@ -251,14 +238,217 @@ public class Ec2ContainerManager {} catch (InterruptedException e) {Thread.currentThread().interrupt();- instance.setState(InstanceState.terminated());+ failLaunch(instance);} catch (Exception e) {LOG.warnv("Failed to launch EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());- instance.setState(InstanceState.terminated());+ failLaunch(instance);}});}+ private StartedContainer createAndStartContainer(Instance instance, ResolvedAmiImage image, String region,+ String flociHost, int imdsPort) {+ String instanceId = instance.getInstanceId();+ String containerName = ContainerStorageHelper.resourceName(config, "ec2", null, instanceId);+ String imdsEndpoint = "http://" + flociHost + ":" + imdsPort;+ String serviceEndpoint = "http://" + flociHost + ":4566";++ while (true) {+ if (isLaunchCancelled(instance)) {+ return null;+ }+ int sshHostPort = portAllocator.allocate(+ config.services().ec2().sshPortRangeStart(),+ config.services().ec2().sshPortRangeEnd());+ ContainerSpec spec = buildContainerSpec(containerName, image, region, serviceEndpoint, imdsEndpoint,+ instanceId, sshHostPort);+ String containerId = null;+ boolean recorded = false;+ try {+ containerId = lifecycleManager.create(spec);+ if (!recordCreatedContainer(instance, containerId, sshHostPort)) {+ lifecycleManager.removeIfExists(containerId);+ portAllocator.release(sshHostPort);+ return null;+ }+ recorded = true;+ lifecycleManager.startCreated(containerId, spec);+ return new StartedContainer(containerId, sshHostPort);+ } catch (Exception e) {+ boolean ownsCleanup = !recorded || clearRecordedContainer(instance, containerId, sshHostPort);+ if (!ownsCleanup) {+ return null;+ }+ if (containerId != null) {+ lifecycleManager.removeIfExists(containerId);+ }+ if (isHostPortCollision(e)) {+ // Docker Desktop can own a published port without exposing it to a host-side+ // ServerSocket probe. Keep it unavailable for this process and try the next port.+ portAllocator.markReserved(sshHostPort);+ LOG.warnv("EC2 instance {0} could not use SSH host port {1}; trying another port",+ instanceId, String.valueOf(sshHostPort));+ continue;+ }+ portAllocator.release(sshHostPort);+ throw e;+ }+ }+ }++ private ContainerSpec buildContainerSpec(String containerName, ResolvedAmiImage image, String region,+ String serviceEndpoint, String imdsEndpoint, String instanceId,+ int sshHostPort) {+ // Minimal images keep the historic tail command, while cloud-image AMI guests can boot their init.+ ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image.dockerImage())+ .withName(containerName)+ .withEmbeddedDns()+ .withDockerNetwork(Optional.empty())+ .withEnv(localAwsEnvironment(region, serviceEndpoint, imdsEndpoint))+ .withEnv("AWS_EC2_INSTANCE_ID", instanceId)+ .withPortBinding(22, sshHostPort)+ .withHostDockerInternalOnLinux()+ .withLogRotation()+ // EC2 instances expose IMDS on 169.254.169.254. Floci needs network administration+ // privileges in the local container to attach that link-local address.+ .withPrivileged(true)+ .withCmd(image.systemd() ? List.of("/sbin/init") : List.of("tail", "-f", "/dev/null"));+ if (image.systemd()) {+ specBuilder+ .withCgroupnsMode("host")+ .withMount(new Mount().withType(MountType.TMPFS).withTarget("/run"))+ .withMount(new Mount().withType(MountType.TMPFS).withTarget("/run/lock"))+ .withBind("/sys/fs/cgroup", "/sys/fs/cgroup");+ }+ return specBuilder.build();+ }++ private void failLaunch(Instance instance) {+ String containerId;+ int sshHostPort;+ String containerIp;+ boolean alreadyCleaned;+ synchronized (instance) {+ alreadyCleaned = isLaunchCancelledState(instance)+ && instance.getDockerContainerId() == null+ && instance.getSshHostPort() <= 0+ && (instance.getContainerBridgeIp() == null || instance.getContainerBridgeIp().isBlank());+ instance.setState(InstanceState.terminated());+ containerId = instance.getDockerContainerId();+ sshHostPort = instance.getSshHostPort();+ containerIp = instance.getContainerBridgeIp();+ instance.setDockerContainerId(null);+ instance.setSshHostPort(0);+ instance.setContainerBridgeIp(null);+ }+ if (alreadyCleaned) {+ return;+ }+ try {+ portForwardManager.unpublishAll(instance);+ } catch (Exception e) {+ LOG.warnv("Error removing EC2 port forwards during failed launch of {0}: {1}",+ instance.getInstanceId(), e.getMessage());+ }+ if (containerId != null) {+ try {+ lifecycleManager.removeIfExists(containerId);+ } catch (Exception e) {+ LOG.warnv("Error removing failed EC2 container {0}: {1}", containerId, e.getMessage());+ }+ }+ if (sshHostPort > 0) {+ portAllocator.release(sshHostPort);+ }+ if (containerIp != null && !containerIp.isBlank()) {+ try {+ metadataServer.unregisterContainer(containerIp, instance);+ } catch (Exception e) {+ LOG.warnv("Error unregistering failed EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());+ }+ }+ }++ /**+ * Cancels a pending asynchronous launch. Returns {@code false} only when the instance reached+ * running state while the caller was waiting, in which case it must not be torn down as a timeout.+ *+ * <p>The terminal state is established atomically before cleanup. The launch worker checks that+ * state between phases, and {@link #recordCreatedContainer(Instance, String, int)} rejects and+ * removes a container whose blocking Docker create call returns after cancellation. A timed-out+ * CloudFormation waiter therefore cannot leave a late worker able to transition the instance to+ * running.</p>+ */+ boolean cancelLaunch(Instance instance) {+ synchronized (instance) {+ String state = instance.getState() != null ? instance.getState().getName() : null;+ if ("running".equals(state)) {+ return false;+ }+ instance.setState(InstanceState.terminated());+ }+ failLaunch(instance);+ return true;+ }++ private static boolean isLaunchCancelled(Instance instance) {+ synchronized (instance) {+ return isLaunchCancelledState(instance);+ }+ }++ private static boolean markRunning(Instance instance) {+ synchronized (instance) {+ if (isLaunchCancelledState(instance)) {+ return false;+ }+ instance.setState(InstanceState.running());+ return true;+ }+ }++ private static boolean recordCreatedContainer(Instance instance, String containerId, int sshHostPort) {+ synchronized (instance) {+ if (isLaunchCancelledState(instance)) {+ return false;+ }+ instance.setSshHostPort(sshHostPort);+ instance.setDockerContainerId(containerId);+ return true;+ }+ }++ private static boolean clearRecordedContainer(Instance instance, String containerId, int sshHostPort) {+ synchronized (instance) {+ if (!Objects.equals(containerId, instance.getDockerContainerId())+ || sshHostPort != instance.getSshHostPort()) {+ return false;+ }+ instance.setDockerContainerId(null);+ instance.setSshHostPort(0);+ return true;+ }+ }++ private static boolean isLaunchCancelledState(Instance instance) {+ String state = instance.getState() != null ? instance.getState().getName() : null;+ return "shutting-down".equals(state) || "terminated".equals(state);+ }++ private static boolean isHostPortCollision(Exception exception) {+ for (Throwable cause = exception; cause != null; cause = cause.getCause()) {+ String message = cause.getMessage();+ if (message != null && (message.toLowerCase(Locale.ROOT).contains("port is already allocated")+ || message.toLowerCase(Locale.ROOT).contains("address already in use"))) {+ return true;+ }+ }+ return false;+ }++ private record StartedContainer(String containerId, int sshHostPort) {+ }+/*** Synchronous stop for emulator shutdown: tears down the port-forward sidecars and* stops the container with a short timeout so N instances cannot exhaust the SIGTERM@@ -412,10 +602,15 @@ public class Ec2ContainerManager {* Sets terminatedAt for TTL pruning.*/public void terminate(Instance instance) {- String containerId = instance.getDockerContainerId();- String containerIp = instance.getContainerBridgeIp();- int sshHostPort = instance.getSshHostPort();- instance.setState(InstanceState.shuttingDown());+ String containerId;+ String containerIp;+ int sshHostPort;+ synchronized (instance) {+ containerId = instance.getDockerContainerId();+ containerIp = instance.getContainerBridgeIp();+ sshHostPort = instance.getSshHostPort();+ instance.setState(InstanceState.shuttingDown());+ }executor.submit(() -> {portForwardManager.unpublishAll(instance);if (containerId != null) {@@ -833,7 +1028,15 @@ public class Ec2ContainerManager {}private String waitForContainerBridgeIp(String containerId, String instanceId) throws InterruptedException {+ return waitForContainerBridgeIp(containerId, instanceId, null);+ }++ private String waitForContainerBridgeIp(String containerId, String instanceId, Instance instance)+ throws InterruptedException {for (int i = 0; i < containerBridgeIpAttempts; i++) {+ if (instance != null && isLaunchCancelled(instance)) {+ return null;+ }String containerIp = getContainerBridgeIp(containerId);if (containerIp != null && !containerIp.isBlank()) {return containerIp;diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.javaindex 476bf8090b..f48e0796b7 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.ec2;import java.nio.charset.StandardCharsets;+import java.time.Duration;import java.time.Instant;import java.time.ZoneOffset;import java.time.format.DateTimeFormatter;@@ -111,6 +112,10 @@ public class Ec2Service implements ContainerTeardown {Pattern.compile("^tgw-rtb-[0-9a-f]{8}([0-9a-f]{9})?$");private static final Pattern TRANSIT_GATEWAY_ATTACHMENT_ID_PATTERN =Pattern.compile("^tgw-attach-[0-9a-f]{8}([0-9a-f]{9})?$");+ // A first launch may need to pull a large AMI-backed image. Keep a finite CloudFormation+ // bound, but allow enough time for that legitimate cold-start path before cancellation.+ private static final Duration CONTAINER_LAUNCH_TIMEOUT = Duration.ofMinutes(5);+ private static final long CONTAINER_LAUNCH_POLL_MILLIS = 50;private final String accountId;private final EmulatorConfig config;@@ -2231,6 +2236,54 @@ public class Ec2Service implements ContainerTeardown {return reservation;}+ /**+ * Waits for a container-backed EC2 instance to reach a terminal launch state.+ * CloudFormation uses this to avoid reporting a stack success when the asynchronous+ * Docker launch has already failed. Mock-mode instances do not launch containers.+ * On timeout, cancellation marks the launch terminal and the container manager prevents any+ * in-flight Docker phase from later publishing a running instance.+ *+ * @param instance the instance returned by {@link #runInstances}+ * @throws AwsException if the container terminates or does not launch before the timeout+ */+ public void awaitContainerLaunch(Instance instance) {+ awaitContainerLaunch(instance, CONTAINER_LAUNCH_TIMEOUT);+ }++ void awaitContainerLaunch(Instance instance, Duration timeout) {+ if (config.services().ec2().mock()) {+ return;+ }++ long deadline = System.nanoTime() + timeout.toNanos();+ while (true) {+ String state = instance.getState() != null ? instance.getState().getName() : null;+ if ("running".equals(state)) {+ return;+ }+ if ("terminated".equals(state)) {+ throw launchFailure(instance, "its container terminated during launch");+ }+ if (System.nanoTime() - deadline >= 0) {+ if (containerManager.cancelLaunch(instance)) {+ throw launchFailure(instance, "it did not reach running state before the launch timeout");+ }+ }+ try {+ Thread.sleep(CONTAINER_LAUNCH_POLL_MILLIS);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AwsException("InternalError", "Interrupted while waiting for EC2 instance "+ + instance.getInstanceId() + " to launch", 500);+ }+ }+ }++ private static AwsException launchFailure(Instance instance, String reason) {+ return new AwsException("InternalError", "EC2 instance " + instance.getInstanceId() + " failed to launch because "+ + reason, 500);+ }+/*** Resolves the TCP ingress ports Floci should publish on the host for an instance, aggregated* across its attached security groups. Empty when publishing is disabled or nothing is opened.diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java b/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.javaindex 3d38066159..e0e786e58d 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java@@ -15,7 +15,7 @@ public class Instance {private String instanceId;private String imageId;- private InstanceState state;+ private volatile InstanceState state;private String stateTransitionReason;private String instanceType;private Placement placement;diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEc2ProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEc2ProvisionerTest.javanew file mode 100644index 0000000000..81f8c90ee4--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEc2ProvisionerTest.java@@ -0,0 +1,121 @@+package io.github.hectorvent.floci.services.cloudformation;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.cloudformation.model.StackResource;+import io.github.hectorvent.floci.services.cloudformation.provisioners.CfnRollback;+import io.github.hectorvent.floci.services.cloudformation.provisioners.CloudFormationResourceRegistry;+import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.services.ec2.model.Instance;+import io.github.hectorvent.floci.services.ec2.model.InstanceState;+import io.github.hectorvent.floci.services.ec2.model.Reservation;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Map;+import java.util.function.Function;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyInt;+import static org.mockito.ArgumentMatchers.anyList;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class CloudFormationEc2ProvisionerTest {++ private final ObjectMapper mapper = new ObjectMapper();+ private Ec2Service ec2Service;+ private CloudFormationResourceProvisioner provisioner;++ @BeforeEach+ void setUp() {+ ec2Service = mock(Ec2Service.class);+ provisioner = new CloudFormationResourceProvisioner(+ null, null, null, null, null, null, null, null, null, null,+ null, null, null, null, null, null, mapper, null, null, null, null, null, null,+ ec2Service, null, null, null, null, null, null, null,+ null, null, new CloudFormationResourceRegistry(List.of()));+ }++ @Test+ void terminatedContainerLaunchMarksEc2ResourceCreateFailed() throws Exception {+ Instance instance = new Instance();+ instance.setInstanceId("i-launch-failed");+ instance.setState(InstanceState.terminated());+ Reservation reservation = new Reservation();+ reservation.getInstances().add(instance);+ when(ec2Service.runInstances(anyString(), anyString(), anyString(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any())).thenReturn(reservation);+ doThrow(new AwsException("InternalError", "EC2 instance i-launch-failed failed to launch because its container terminated during launch", 500))+ .when(ec2Service).awaitContainerLaunch(instance);++ StackResource resource = provisioner.provision("Server", "AWS::EC2::Instance", properties(), engine(),+ "us-east-1", "000000000000", "test-stack");++ assertEquals("CREATE_FAILED", resource.getStatus());+ assertEquals("i-launch-failed", resource.getPhysicalId());+ assertEquals("true", resource.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));+ assertTrue(resource.getStatusReason().contains("container terminated during launch"));+ verify(ec2Service).awaitContainerLaunch(instance);+ }++ @Test+ void timedOutContainerLaunchMarksEc2ResourceCreateFailed() throws Exception {+ Instance instance = new Instance();+ instance.setInstanceId("i-launch-timed-out");+ instance.setState(InstanceState.pending());+ Reservation reservation = new Reservation();+ reservation.getInstances().add(instance);+ when(ec2Service.runInstances(anyString(), anyString(), anyString(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any())).thenReturn(reservation);+ doThrow(new AwsException("InternalError",+ "EC2 instance i-launch-timed-out failed to launch because it did not reach running state before the launch timeout",+ 500)).when(ec2Service).awaitContainerLaunch(instance);++ StackResource resource = provisioner.provision("Server", "AWS::EC2::Instance", properties(), engine(),+ "us-east-1", "000000000000", "test-stack");++ assertEquals("CREATE_FAILED", resource.getStatus());+ assertEquals("i-launch-timed-out", resource.getPhysicalId());+ assertEquals("true", resource.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));+ assertTrue(resource.getStatusReason().contains("launch timeout"));+ verify(ec2Service).awaitContainerLaunch(instance);+ }++ @Test+ void successfulContainerLaunchDoesNotKeepRollbackOwnershipMarker() throws Exception {+ Instance instance = new Instance();+ instance.setInstanceId("i-launch-succeeded");+ instance.setState(InstanceState.running());+ Reservation reservation = new Reservation();+ reservation.getInstances().add(instance);+ when(ec2Service.runInstances(anyString(), anyString(), anyString(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any())).thenReturn(reservation);++ StackResource resource = provisioner.provision("Server", "AWS::EC2::Instance", properties(), engine(),+ "us-east-1", "000000000000", "test-stack");++ assertEquals("CREATE_COMPLETE", resource.getStatus());+ assertNull(resource.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));+ verify(ec2Service).awaitContainerLaunch(instance);+ }++ private JsonNode properties() throws Exception {+ return mapper.readTree("""+ {"ImageId":"ami-12345678","InstanceType":"t3.micro"}+ """);+ }++ private CloudFormationTemplateEngine engine() {+ return new CloudFormationTemplateEngine("000000000000", "us-east-1", "test-stack", "stack/id",+ Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), mapper, (Function<String, String>) name -> null);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.javaindex 179e568e20..b42cdd352b 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java@@ -42,6 +42,7 @@ import java.util.function.BooleanSupplier;import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyInt;@@ -49,11 +50,12 @@ import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Answers.RETURNS_SELF;import static org.mockito.Answers.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.after;+import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.timeout;import static org.mockito.Mockito.times;-import static org.mockito.Mockito.timeout;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;import static org.mockito.Mockito.withSettings;@@ -389,10 +391,118 @@ class Ec2ContainerManagerTest {harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));- assertEquals(TEST_CONTAINER_ID, instance.getDockerContainerId());+ verify(harness.lifecycleManager, timeout(2_000)).removeIfExists(TEST_CONTAINER_ID);+ verify(harness.portAllocator, timeout(2_000)).release(2201);+ assertNull(instance.getDockerContainerId());verify(harness.metadataServer, never()).registerContainer(anyString(), anyString(), any());}+ @Test+ void launchRetriesWithNextPortWhenDockerReportsHostPortCollision() throws Exception {+ Ec2ContainerManager.containerBridgeIpAttempts = 1;+ Ec2ContainerManager.containerBridgeIpPollMillis = 1;+ LaunchHarness harness = launchHarness();+ when(harness.portAllocator.allocate(anyInt(), anyInt())).thenReturn(2201, 2202);+ when(harness.lifecycleManager.create(any(ContainerSpec.class)))+ .thenReturn("container-conflict", TEST_CONTAINER_ID);+ when(harness.lifecycleManager.startCreated(anyString(), any(ContainerSpec.class)))+ .thenThrow(new RuntimeException("Bind for 0.0.0.0:2201 failed: port is already allocated"))+ .thenReturn(null);++ InspectContainerCmd inspect = mock(InspectContainerCmd.class);+ InspectContainerResponse withIp = inspectResponse("172.18.0.12");+ when(harness.dockerClient.inspectContainerCmd(TEST_CONTAINER_ID)).thenReturn(inspect);+ when(inspect.exec()).thenReturn(withIp);+ harness.stubSuccessfulExecs(new CountDownLatch(0), new CountDownLatch(0));++ Instance instance = instance("i-port-collision");+ harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");++ awaitUntil(() -> "running".equals(instance.getState().getName()), Duration.ofSeconds(2));+ assertEquals(2202, instance.getSshHostPort());+ verify(harness.lifecycleManager).removeIfExists("container-conflict");+ verify(harness.portAllocator).markReserved(2201);+ verify(harness.builder).withPortBinding(22, 2201);+ verify(harness.builder).withPortBinding(22, 2202);+ }++ @Test+ void launchReleasesPortAndCleansUpContainerAfterNonRetryableStartFailure() throws Exception {+ LaunchHarness harness = launchHarness();+ when(harness.lifecycleManager.startCreated(eq(TEST_CONTAINER_ID), any(ContainerSpec.class)))+ .thenThrow(new RuntimeException("Docker daemon unavailable"));++ Instance instance = instance("i-start-failure");+ harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");++ awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));+ verify(harness.lifecycleManager).removeIfExists(TEST_CONTAINER_ID);+ verify(harness.portAllocator).release(2201);+ }++ @Test+ void launchTerminatesBeforeCleanupFailure() throws Exception {+ LaunchHarness harness = launchHarness();+ when(harness.lifecycleManager.startCreated(eq(TEST_CONTAINER_ID), any(ContainerSpec.class)))+ .thenThrow(new RuntimeException("Docker daemon unavailable"));+ doThrow(new RuntimeException("port-forward cleanup failed"))+ .when(harness.portForwardManager).unpublishAll(any(Instance.class));++ Instance instance = instance("i-cleanup-failure");+ harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");++ awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));+ verify(harness.portForwardManager).unpublishAll(instance);+ }++ @Test+ void cancelledLaunchRemovesContainerThatStartsAfterCancellation() throws Exception {+ LaunchHarness harness = launchHarness();+ CountDownLatch startEntered = new CountDownLatch(1);+ CountDownLatch allowStart = new CountDownLatch(1);+ when(harness.lifecycleManager.startCreated(eq(TEST_CONTAINER_ID), any(ContainerSpec.class))).thenAnswer(invocation -> {+ startEntered.countDown();+ assertTrue(allowStart.await(2, TimeUnit.SECONDS));+ return null;+ });++ Instance instance = instance("i-cancelled-launch");+ harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");++ assertTrue(startEntered.await(2, TimeUnit.SECONDS), "container startup should begin");+ assertTrue(harness.manager.cancelLaunch(instance));+ verify(harness.lifecycleManager).removeIfExists(TEST_CONTAINER_ID);+ verify(harness.portAllocator).release(2201);++ allowStart.countDown();+ verify(harness.portAllocator, after(500).times(1)).release(2201);+ }++ @Test+ void cancelledLaunchRemovesContainerCreatedAfterCancellation() throws Exception {+ LaunchHarness harness = launchHarness();+ CountDownLatch createEntered = new CountDownLatch(1);+ CountDownLatch allowCreate = new CountDownLatch(1);+ when(harness.lifecycleManager.create(any(ContainerSpec.class))).thenAnswer(invocation -> {+ createEntered.countDown();+ assertTrue(allowCreate.await(2, TimeUnit.SECONDS));+ return TEST_CONTAINER_ID;+ });++ Instance instance = instance("i-cancelled-during-create");+ harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");++ assertTrue(createEntered.await(2, TimeUnit.SECONDS), "container creation should begin");+ assertTrue(harness.manager.cancelLaunch(instance));+ assertEquals("terminated", instance.getState().getName());++ allowCreate.countDown();+ verify(harness.lifecycleManager, timeout(2_000)).removeIfExists(TEST_CONTAINER_ID);+ verify(harness.portAllocator, timeout(2_000)).release(2201);+ verify(harness.lifecycleManager, never()).startCreated(eq(TEST_CONTAINER_ID), any(ContainerSpec.class));+ assertEquals("terminated", instance.getState().getName());+ }+@Testvoid launchMarksInstanceRunningBeforeUserDataCompletes() throws Exception {Ec2ContainerManager.containerBridgeIpAttempts = 2;@@ -593,6 +703,7 @@ class Ec2ContainerManagerTest {DockerClient dockerClient = mock(DockerClient.class);Ec2MetadataServer metadataServer = mock(Ec2MetadataServer.class);ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);+ Ec2PortForwardManager portForwardManager = mock(Ec2PortForwardManager.class);Ec2ContainerManager manager = new Ec2ContainerManager(containerBuilder,lifecycleManager,@@ -603,9 +714,9 @@ class Ec2ContainerManagerTest {portAllocator,config,metadataServer,- mock(Ec2PortForwardManager.class));- return new LaunchHarness(manager, dockerClient, metadataServer, logStreamer, builder,- new CopyOnWriteArrayList<>());+ portForwardManager);+ return new LaunchHarness(manager, lifecycleManager, dockerClient, metadataServer, logStreamer, builder,+ portAllocator, portForwardManager, new CopyOnWriteArrayList<>());}/**@@ -649,10 +760,13 @@ class Ec2ContainerManagerTest {}private record LaunchHarness(Ec2ContainerManager manager,+ ContainerLifecycleManager lifecycleManager,DockerClient dockerClient,Ec2MetadataServer metadataServer,ContainerLogStreamer logStreamer,ContainerBuilder.Builder builder,+ PortAllocator portAllocator,+ Ec2PortForwardManager portForwardManager,List<String[]> executedCommands) {void stubSuccessfulExecs(CountDownLatch userDataStarted, CountDownLatch finishUserData) throws Exception {AtomicReference<String[]> currentCommand = new AtomicReference<>();diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.javaindex abfcec7ee9..ff612c3ea0 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.services.ec2.model.IpPermission;import io.github.hectorvent.floci.services.ec2.model.Ipv6Range;import io.github.hectorvent.floci.services.ec2.model.SecurityGroupRule;import io.github.hectorvent.floci.services.ec2.model.UserIdGroupPair;+import io.github.hectorvent.floci.services.ec2.model.InstanceState;import io.github.hectorvent.floci.services.ec2.model.LaunchTemplate;import io.github.hectorvent.floci.services.ec2.model.ManagedPrefixList;import io.github.hectorvent.floci.services.ec2.model.SecurityGroupRule;@@ -39,9 +40,9 @@ import io.github.hectorvent.floci.services.ec2.model.Vpc;import io.github.hectorvent.floci.services.ec2.model.VpcEndpoint;import io.github.hectorvent.floci.services.ec2.model.Volume;import io.github.hectorvent.floci.services.ec2.model.VolumeAttachment;-import io.github.hectorvent.floci.services.ec2.model.InstanceState;import org.junit.jupiter.api.Test;+import java.time.Duration;import java.util.List;import java.util.Map;import java.util.Optional;@@ -80,6 +81,40 @@ class Ec2ServiceTest {verifyNoInteractions(containerManager);}+ @Test+ void awaitContainerLaunchReportsTerminatedContainer() {+ Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());+ Instance instance = new Instance();+ instance.setInstanceId("i-terminated");+ instance.setState(InstanceState.terminated());++ AwsException error = assertThrows(AwsException.class, () -> service.awaitContainerLaunch(instance));++ assertEquals("InternalError", error.getErrorCode());+ assertTrue(error.getMessage().contains("container terminated during launch"));+ }++ @Test+ void awaitContainerLaunchTimesOutWhileInstanceIsPending() {+ Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);+ Ec2Service service = new Ec2Service(mockConfig(false), containerManager,+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());+ Instance instance = new Instance();+ instance.setInstanceId("i-pending");+ instance.setState(InstanceState.pending());+ when(containerManager.cancelLaunch(instance)).thenReturn(true);++ AwsException error = assertThrows(AwsException.class,+ () -> service.awaitContainerLaunch(instance, Duration.ZERO));++ assertEquals("InternalError", error.getErrorCode());+ assertTrue(error.getMessage().contains("did not reach running state before the launch timeout"));+ verify(containerManager).cancelLaunch(instance);+ }+@Testvoid runInstancesRequiresImageIdInsteadOfDefaulting() {Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),