Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ec2): retry Docker port collisions (#2029)

Aniket Shuklab27b32f044da
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java
index 1bdcd3431b..d282887a84 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java
@@ -1452,6 +1452,9 @@ public class CloudFormationResourceProvisioner {
var instance = reservation.getInstances().get(0);
r.setPhysicalId(instance.getInstanceId());
r.getAttributes().put("InstanceId", instance.getInstanceId());
+ r.getAttributes().put(CfnRollback.ROLLBACK_OWNED_ATTR, "true");
+ ec2Service.awaitContainerLaunch(instance);
+ r.getAttributes().remove(CfnRollback.ROLLBACK_OWNED_ATTR);
if (instance.getPrivateIpAddress() != null) {
r.getAttributes().put("PrivateIp", instance.getPrivateIpAddress());
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
index e7fefefaf9..e110877d84 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
@@ -34,6 +34,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
@@ -47,6 +48,8 @@ import java.util.regex.Pattern;
* Manages Docker container lifecycle for EC2 instances.
* Handles launch, stop, start, terminate, and reboot operations.
* SSH key injection and UserData execution are performed asynchronously after launch.
+ * A running instance has a reachable container address, while best-effort link-local IMDS setup,
+ * SSH initialization, and UserData may still be completing in the launch worker.
*/
@ApplicationScoped
public class Ec2ContainerManager {
@@ -134,55 +137,28 @@ public class Ec2ContainerManager {
executor.submit(() -> {
try {
String instanceId = instance.getInstanceId();
- String containerName = ContainerStorageHelper.resourceName(config, "ec2", null, instanceId);
-
- // Allocate SSH host port
- int sshHostPort = portAllocator.allocate(
- config.services().ec2().sshPortRangeStart(),
- config.services().ec2().sshPortRangeEnd());
- instance.setSshHostPort(sshHostPort);
-
// IMDS endpoint that this container should use
String flociHost = dockerHostResolver.resolve();
int imdsPort = config.services().ec2().imdsPort();
- String imdsEndpoint = "http://" + flociHost + ":" + imdsPort;
- String serviceEndpoint = "http://" + flociHost + ":4566";
-
- // Build container spec — minimal images keep the historic tail
- // command, while cloud-image AMI guests can boot their init.
- ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image.dockerImage())
- .withName(containerName)
- .withEmbeddedDns()
- .withDockerNetwork(Optional.empty())
- .withEnv(localAwsEnvironment(region, serviceEndpoint, imdsEndpoint))
- .withEnv("AWS_EC2_INSTANCE_ID", instanceId)
- .withPortBinding(22, sshHostPort)
- .withHostDockerInternalOnLinux()
- .withLogRotation()
- // EC2 instances expose IMDS on 169.254.169.254. Floci
- // needs network administration privileges in the local
- // container to attach that link-local address.
- .withPrivileged(true)
- .withCmd(image.systemd() ? List.of("/sbin/init") : List.of("tail", "-f", "/dev/null"));
- if (image.systemd()) {
- specBuilder
- .withCgroupnsMode("host")
- .withMount(new Mount().withType(MountType.TMPFS).withTarget("/run"))
- .withMount(new Mount().withType(MountType.TMPFS).withTarget("/run/lock"))
- .withBind("/sys/fs/cgroup", "/sys/fs/cgroup");
+ StartedContainer started = createAndStartContainer(instance, image, region, flociHost, imdsPort);
+ if (started == null) {
+ return;
}
- ContainerSpec spec = specBuilder.build();
+ int sshHostPort = started.sshHostPort();
+ String containerId = started.containerId();
- // Create container without starting it
- String containerId = lifecycleManager.create(spec);
- instance.setDockerContainerId(containerId);
-
- // Start the container
- lifecycleManager.startCreated(containerId, spec);
+ if (isLaunchCancelled(instance)) {
+ failLaunch(instance);
+ return;
+ }
// Poll until Docker confirms the container is running
boolean running = false;
for (int i = 0; i < 30 && !running; i++) {
+ if (isLaunchCancelled(instance)) {
+ failLaunch(instance);
+ return;
+ }
running = lifecycleManager.isContainerRunning(containerId);
if (!running) {
Thread.sleep(500);
@@ -191,7 +167,12 @@ public class Ec2ContainerManager {
if (!running) {
LOG.warnv("EC2 instance {0} container {1} did not reach running state", instanceId, containerId);
- instance.setState(InstanceState.terminated());
+ failLaunch(instance);
+ return;
+ }
+
+ if (isLaunchCancelled(instance)) {
+ failLaunch(instance);
return;
}
@@ -199,7 +180,7 @@ public class Ec2ContainerManager {
// Docker can report the container as running before network
// settings are populated; wait here so IMDS is registered
// before link-local metadata validation and UserData run.
- String containerIp = waitForContainerBridgeIp(containerId, instanceId);
+ String containerIp = waitForContainerBridgeIp(containerId, instanceId, instance);
if (containerIp != null && !containerIp.isBlank()) {
instance.setContainerBridgeIp(containerIp);
exposeReachablePrivateAddress(instance, containerIp, config.services().ec2().awsFaithfulPrivateIp());
@@ -208,11 +189,14 @@ public class Ec2ContainerManager {
else {
LOG.warnv("EC2 instance {0} container {1} did not receive a usable bridge IP for IMDS",
instanceId, containerId);
- instance.setState(InstanceState.terminated());
+ failLaunch(instance);
return;
}
- configureLinkLocalMetadataEndpoint(containerId, instanceId, flociHost, imdsPort);
+ if (!markRunning(instance)) {
+ failLaunch(instance);
+ return;
+ }
// Set public-facing addresses only for instances whose subnet
// opts in via MapPublicIpOnLaunch (#1984). Private-subnet
@@ -223,10 +207,13 @@ public class Ec2ContainerManager {
instance.setPublicDnsName("localhost");
}
- instance.setState(InstanceState.running());
LOG.infov("EC2 instance {0} running in container {1} (SSH host port {2})",
instanceId, containerId, String.valueOf(sshHostPort));
+ // IMDS proxy setup is best effort and has its own bounded commands. The instance
+ // is already running once Docker has assigned its reachable network address.
+ configureLinkLocalMetadataEndpoint(containerId, instanceId, flociHost, imdsPort);
+
// Publish security-group TCP ingress ports on the host via socat sidecars.
if (appPorts != null && !appPorts.isEmpty()) {
portForwardManager.reconcile(instance, appPorts);
@@ -251,14 +238,217 @@ public class Ec2ContainerManager {
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
- instance.setState(InstanceState.terminated());
+ failLaunch(instance);
} catch (Exception e) {
LOG.warnv("Failed to launch EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());
- instance.setState(InstanceState.terminated());
+ failLaunch(instance);
}
});
}
+ private StartedContainer createAndStartContainer(Instance instance, ResolvedAmiImage image, String region,
+ String flociHost, int imdsPort) {
+ String instanceId = instance.getInstanceId();
+ String containerName = ContainerStorageHelper.resourceName(config, "ec2", null, instanceId);
+ String imdsEndpoint = "http://" + flociHost + ":" + imdsPort;
+ String serviceEndpoint = "http://" + flociHost + ":4566";
+
+ while (true) {
+ if (isLaunchCancelled(instance)) {
+ return null;
+ }
+ int sshHostPort = portAllocator.allocate(
+ config.services().ec2().sshPortRangeStart(),
+ config.services().ec2().sshPortRangeEnd());
+ ContainerSpec spec = buildContainerSpec(containerName, image, region, serviceEndpoint, imdsEndpoint,
+ instanceId, sshHostPort);
+ String containerId = null;
+ boolean recorded = false;
+ try {
+ containerId = lifecycleManager.create(spec);
+ if (!recordCreatedContainer(instance, containerId, sshHostPort)) {
+ lifecycleManager.removeIfExists(containerId);
+ portAllocator.release(sshHostPort);
+ return null;
+ }
+ recorded = true;
+ lifecycleManager.startCreated(containerId, spec);
+ return new StartedContainer(containerId, sshHostPort);
+ } catch (Exception e) {
+ boolean ownsCleanup = !recorded || clearRecordedContainer(instance, containerId, sshHostPort);
+ if (!ownsCleanup) {
+ return null;
+ }
+ if (containerId != null) {
+ lifecycleManager.removeIfExists(containerId);
+ }
+ if (isHostPortCollision(e)) {
+ // Docker Desktop can own a published port without exposing it to a host-side
+ // ServerSocket probe. Keep it unavailable for this process and try the next port.
+ portAllocator.markReserved(sshHostPort);
+ LOG.warnv("EC2 instance {0} could not use SSH host port {1}; trying another port",
+ instanceId, String.valueOf(sshHostPort));
+ continue;
+ }
+ portAllocator.release(sshHostPort);
+ throw e;
+ }
+ }
+ }
+
+ private ContainerSpec buildContainerSpec(String containerName, ResolvedAmiImage image, String region,
+ String serviceEndpoint, String imdsEndpoint, String instanceId,
+ int sshHostPort) {
+ // Minimal images keep the historic tail command, while cloud-image AMI guests can boot their init.
+ ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image.dockerImage())
+ .withName(containerName)
+ .withEmbeddedDns()
+ .withDockerNetwork(Optional.empty())
+ .withEnv(localAwsEnvironment(region, serviceEndpoint, imdsEndpoint))
+ .withEnv("AWS_EC2_INSTANCE_ID", instanceId)
+ .withPortBinding(22, sshHostPort)
+ .withHostDockerInternalOnLinux()
+ .withLogRotation()
+ // EC2 instances expose IMDS on 169.254.169.254. Floci needs network administration
+ // privileges in the local container to attach that link-local address.
+ .withPrivileged(true)
+ .withCmd(image.systemd() ? List.of("/sbin/init") : List.of("tail", "-f", "/dev/null"));
+ if (image.systemd()) {
+ specBuilder
+ .withCgroupnsMode("host")
+ .withMount(new Mount().withType(MountType.TMPFS).withTarget("/run"))
+ .withMount(new Mount().withType(MountType.TMPFS).withTarget("/run/lock"))
+ .withBind("/sys/fs/cgroup", "/sys/fs/cgroup");
+ }
+ return specBuilder.build();
+ }
+
+ private void failLaunch(Instance instance) {
+ String containerId;
+ int sshHostPort;
+ String containerIp;
+ boolean alreadyCleaned;
+ synchronized (instance) {
+ alreadyCleaned = isLaunchCancelledState(instance)
+ && instance.getDockerContainerId() == null
+ && instance.getSshHostPort() <= 0
+ && (instance.getContainerBridgeIp() == null || instance.getContainerBridgeIp().isBlank());
+ instance.setState(InstanceState.terminated());
+ containerId = instance.getDockerContainerId();
+ sshHostPort = instance.getSshHostPort();
+ containerIp = instance.getContainerBridgeIp();
+ instance.setDockerContainerId(null);
+ instance.setSshHostPort(0);
+ instance.setContainerBridgeIp(null);
+ }
+ if (alreadyCleaned) {
+ return;
+ }
+ try {
+ portForwardManager.unpublishAll(instance);
+ } catch (Exception e) {
+ LOG.warnv("Error removing EC2 port forwards during failed launch of {0}: {1}",
+ instance.getInstanceId(), e.getMessage());
+ }
+ if (containerId != null) {
+ try {
+ lifecycleManager.removeIfExists(containerId);
+ } catch (Exception e) {
+ LOG.warnv("Error removing failed EC2 container {0}: {1}", containerId, e.getMessage());
+ }
+ }
+ if (sshHostPort > 0) {
+ portAllocator.release(sshHostPort);
+ }
+ if (containerIp != null && !containerIp.isBlank()) {
+ try {
+ metadataServer.unregisterContainer(containerIp, instance);
+ } catch (Exception e) {
+ LOG.warnv("Error unregistering failed EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Cancels a pending asynchronous launch. Returns {@code false} only when the instance reached
+ * running state while the caller was waiting, in which case it must not be torn down as a timeout.
+ *
+ * <p>The terminal state is established atomically before cleanup. The launch worker checks that
+ * state between phases, and {@link #recordCreatedContainer(Instance, String, int)} rejects and
+ * removes a container whose blocking Docker create call returns after cancellation. A timed-out
+ * CloudFormation waiter therefore cannot leave a late worker able to transition the instance to
+ * running.</p>
+ */
+ boolean cancelLaunch(Instance instance) {
+ synchronized (instance) {
+ String state = instance.getState() != null ? instance.getState().getName() : null;
+ if ("running".equals(state)) {
+ return false;
+ }
+ instance.setState(InstanceState.terminated());
+ }
+ failLaunch(instance);
+ return true;
+ }
+
+ private static boolean isLaunchCancelled(Instance instance) {
+ synchronized (instance) {
+ return isLaunchCancelledState(instance);
+ }
+ }
+
+ private static boolean markRunning(Instance instance) {
+ synchronized (instance) {
+ if (isLaunchCancelledState(instance)) {
+ return false;
+ }
+ instance.setState(InstanceState.running());
+ return true;
+ }
+ }
+
+ private static boolean recordCreatedContainer(Instance instance, String containerId, int sshHostPort) {
+ synchronized (instance) {
+ if (isLaunchCancelledState(instance)) {
+ return false;
+ }
+ instance.setSshHostPort(sshHostPort);
+ instance.setDockerContainerId(containerId);
+ return true;
+ }
+ }
+
+ private static boolean clearRecordedContainer(Instance instance, String containerId, int sshHostPort) {
+ synchronized (instance) {
+ if (!Objects.equals(containerId, instance.getDockerContainerId())
+ || sshHostPort != instance.getSshHostPort()) {
+ return false;
+ }
+ instance.setDockerContainerId(null);
+ instance.setSshHostPort(0);
+ return true;
+ }
+ }
+
+ private static boolean isLaunchCancelledState(Instance instance) {
+ String state = instance.getState() != null ? instance.getState().getName() : null;
+ return "shutting-down".equals(state) || "terminated".equals(state);
+ }
+
+ private static boolean isHostPortCollision(Exception exception) {
+ for (Throwable cause = exception; cause != null; cause = cause.getCause()) {
+ String message = cause.getMessage();
+ if (message != null && (message.toLowerCase(Locale.ROOT).contains("port is already allocated")
+ || message.toLowerCase(Locale.ROOT).contains("address already in use"))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private record StartedContainer(String containerId, int sshHostPort) {
+ }
+
/**
* Synchronous stop for emulator shutdown: tears down the port-forward sidecars and
* stops the container with a short timeout so N instances cannot exhaust the SIGTERM
@@ -412,10 +602,15 @@ public class Ec2ContainerManager {
* Sets terminatedAt for TTL pruning.
*/
public void terminate(Instance instance) {
- String containerId = instance.getDockerContainerId();
- String containerIp = instance.getContainerBridgeIp();
- int sshHostPort = instance.getSshHostPort();
- instance.setState(InstanceState.shuttingDown());
+ String containerId;
+ String containerIp;
+ int sshHostPort;
+ synchronized (instance) {
+ containerId = instance.getDockerContainerId();
+ containerIp = instance.getContainerBridgeIp();
+ sshHostPort = instance.getSshHostPort();
+ instance.setState(InstanceState.shuttingDown());
+ }
executor.submit(() -> {
portForwardManager.unpublishAll(instance);
if (containerId != null) {
@@ -833,7 +1028,15 @@ public class Ec2ContainerManager {
}
private String waitForContainerBridgeIp(String containerId, String instanceId) throws InterruptedException {
+ return waitForContainerBridgeIp(containerId, instanceId, null);
+ }
+
+ private String waitForContainerBridgeIp(String containerId, String instanceId, Instance instance)
+ throws InterruptedException {
for (int i = 0; i < containerBridgeIpAttempts; i++) {
+ if (instance != null && isLaunchCancelled(instance)) {
+ return null;
+ }
String containerIp = getContainerBridgeIp(containerId);
if (containerIp != null && !containerIp.isBlank()) {
return containerIp;
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
index 476bf8090b..f48e0796b7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.ec2;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
@@ -111,6 +112,10 @@ public class Ec2Service implements ContainerTeardown {
Pattern.compile("^tgw-rtb-[0-9a-f]{8}([0-9a-f]{9})?$");
private static final Pattern TRANSIT_GATEWAY_ATTACHMENT_ID_PATTERN =
Pattern.compile("^tgw-attach-[0-9a-f]{8}([0-9a-f]{9})?$");
+ // A first launch may need to pull a large AMI-backed image. Keep a finite CloudFormation
+ // bound, but allow enough time for that legitimate cold-start path before cancellation.
+ private static final Duration CONTAINER_LAUNCH_TIMEOUT = Duration.ofMinutes(5);
+ private static final long CONTAINER_LAUNCH_POLL_MILLIS = 50;
private final String accountId;
private final EmulatorConfig config;
@@ -2231,6 +2236,54 @@ public class Ec2Service implements ContainerTeardown {
return reservation;
}
+ /**
+ * Waits for a container-backed EC2 instance to reach a terminal launch state.
+ * CloudFormation uses this to avoid reporting a stack success when the asynchronous
+ * Docker launch has already failed. Mock-mode instances do not launch containers.
+ * On timeout, cancellation marks the launch terminal and the container manager prevents any
+ * in-flight Docker phase from later publishing a running instance.
+ *
+ * @param instance the instance returned by {@link #runInstances}
+ * @throws AwsException if the container terminates or does not launch before the timeout
+ */
+ public void awaitContainerLaunch(Instance instance) {
+ awaitContainerLaunch(instance, CONTAINER_LAUNCH_TIMEOUT);
+ }
+
+ void awaitContainerLaunch(Instance instance, Duration timeout) {
+ if (config.services().ec2().mock()) {
+ return;
+ }
+
+ long deadline = System.nanoTime() + timeout.toNanos();
+ while (true) {
+ String state = instance.getState() != null ? instance.getState().getName() : null;
+ if ("running".equals(state)) {
+ return;
+ }
+ if ("terminated".equals(state)) {
+ throw launchFailure(instance, "its container terminated during launch");
+ }
+ if (System.nanoTime() - deadline >= 0) {
+ if (containerManager.cancelLaunch(instance)) {
+ throw launchFailure(instance, "it did not reach running state before the launch timeout");
+ }
+ }
+ try {
+ Thread.sleep(CONTAINER_LAUNCH_POLL_MILLIS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AwsException("InternalError", "Interrupted while waiting for EC2 instance "
+ + instance.getInstanceId() + " to launch", 500);
+ }
+ }
+ }
+
+ private static AwsException launchFailure(Instance instance, String reason) {
+ return new AwsException("InternalError", "EC2 instance " + instance.getInstanceId() + " failed to launch because "
+ + reason, 500);
+ }
+
/**
* Resolves the TCP ingress ports Floci should publish on the host for an instance, aggregated
* across its attached security groups. Empty when publishing is disabled or nothing is opened.
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java b/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java
index 3d38066159..e0e786e58d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java
@@ -15,7 +15,7 @@ public class Instance {
private String instanceId;
private String imageId;
- private InstanceState state;
+ private volatile InstanceState state;
private String stateTransitionReason;
private String instanceType;
private Placement placement;
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEc2ProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEc2ProvisionerTest.java
new file mode 100644
index 0000000000..81f8c90ee4
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEc2ProvisionerTest.java
@@ -0,0 +1,121 @@
+package io.github.hectorvent.floci.services.cloudformation;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.cloudformation.model.StackResource;
+import io.github.hectorvent.floci.services.cloudformation.provisioners.CfnRollback;
+import io.github.hectorvent.floci.services.cloudformation.provisioners.CloudFormationResourceRegistry;
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.ec2.model.Instance;
+import io.github.hectorvent.floci.services.ec2.model.InstanceState;
+import io.github.hectorvent.floci.services.ec2.model.Reservation;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class CloudFormationEc2ProvisionerTest {
+
+ private final ObjectMapper mapper = new ObjectMapper();
+ private Ec2Service ec2Service;
+ private CloudFormationResourceProvisioner provisioner;
+
+ @BeforeEach
+ void setUp() {
+ ec2Service = mock(Ec2Service.class);
+ provisioner = new CloudFormationResourceProvisioner(
+ null, null, null, null, null, null, null, null, null, null,
+ null, null, null, null, null, null, mapper, null, null, null, null, null, null,
+ ec2Service, null, null, null, null, null, null, null,
+ null, null, new CloudFormationResourceRegistry(List.of()));
+ }
+
+ @Test
+ void terminatedContainerLaunchMarksEc2ResourceCreateFailed() throws Exception {
+ Instance instance = new Instance();
+ instance.setInstanceId("i-launch-failed");
+ instance.setState(InstanceState.terminated());
+ Reservation reservation = new Reservation();
+ reservation.getInstances().add(instance);
+ when(ec2Service.runInstances(anyString(), anyString(), anyString(), anyInt(), anyInt(), any(), anyList(),
+ any(), any(), anyList(), any(), any(), any())).thenReturn(reservation);
+ doThrow(new AwsException("InternalError", "EC2 instance i-launch-failed failed to launch because its container terminated during launch", 500))
+ .when(ec2Service).awaitContainerLaunch(instance);
+
+ StackResource resource = provisioner.provision("Server", "AWS::EC2::Instance", properties(), engine(),
+ "us-east-1", "000000000000", "test-stack");
+
+ assertEquals("CREATE_FAILED", resource.getStatus());
+ assertEquals("i-launch-failed", resource.getPhysicalId());
+ assertEquals("true", resource.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));
+ assertTrue(resource.getStatusReason().contains("container terminated during launch"));
+ verify(ec2Service).awaitContainerLaunch(instance);
+ }
+
+ @Test
+ void timedOutContainerLaunchMarksEc2ResourceCreateFailed() throws Exception {
+ Instance instance = new Instance();
+ instance.setInstanceId("i-launch-timed-out");
+ instance.setState(InstanceState.pending());
+ Reservation reservation = new Reservation();
+ reservation.getInstances().add(instance);
+ when(ec2Service.runInstances(anyString(), anyString(), anyString(), anyInt(), anyInt(), any(), anyList(),
+ any(), any(), anyList(), any(), any(), any())).thenReturn(reservation);
+ doThrow(new AwsException("InternalError",
+ "EC2 instance i-launch-timed-out failed to launch because it did not reach running state before the launch timeout",
+ 500)).when(ec2Service).awaitContainerLaunch(instance);
+
+ StackResource resource = provisioner.provision("Server", "AWS::EC2::Instance", properties(), engine(),
+ "us-east-1", "000000000000", "test-stack");
+
+ assertEquals("CREATE_FAILED", resource.getStatus());
+ assertEquals("i-launch-timed-out", resource.getPhysicalId());
+ assertEquals("true", resource.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));
+ assertTrue(resource.getStatusReason().contains("launch timeout"));
+ verify(ec2Service).awaitContainerLaunch(instance);
+ }
+
+ @Test
+ void successfulContainerLaunchDoesNotKeepRollbackOwnershipMarker() throws Exception {
+ Instance instance = new Instance();
+ instance.setInstanceId("i-launch-succeeded");
+ instance.setState(InstanceState.running());
+ Reservation reservation = new Reservation();
+ reservation.getInstances().add(instance);
+ when(ec2Service.runInstances(anyString(), anyString(), anyString(), anyInt(), anyInt(), any(), anyList(),
+ any(), any(), anyList(), any(), any(), any())).thenReturn(reservation);
+
+ StackResource resource = provisioner.provision("Server", "AWS::EC2::Instance", properties(), engine(),
+ "us-east-1", "000000000000", "test-stack");
+
+ assertEquals("CREATE_COMPLETE", resource.getStatus());
+ assertNull(resource.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));
+ verify(ec2Service).awaitContainerLaunch(instance);
+ }
+
+ private JsonNode properties() throws Exception {
+ return mapper.readTree("""
+ {"ImageId":"ami-12345678","InstanceType":"t3.micro"}
+ """);
+ }
+
+ private CloudFormationTemplateEngine engine() {
+ return new CloudFormationTemplateEngine("000000000000", "us-east-1", "test-stack", "stack/id",
+ Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), mapper, (Function<String, String>) name -> null);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java
index 179e568e20..b42cdd352b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java
@@ -42,6 +42,7 @@ import java.util.function.BooleanSupplier;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
@@ -49,11 +50,12 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Answers.RETURNS_SELF;
import static org.mockito.Answers.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.after;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.withSettings;
@@ -389,10 +391,118 @@ class Ec2ContainerManagerTest {
harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");
awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));
- assertEquals(TEST_CONTAINER_ID, instance.getDockerContainerId());
+ verify(harness.lifecycleManager, timeout(2_000)).removeIfExists(TEST_CONTAINER_ID);
+ verify(harness.portAllocator, timeout(2_000)).release(2201);
+ assertNull(instance.getDockerContainerId());
verify(harness.metadataServer, never()).registerContainer(anyString(), anyString(), any());
}
+ @Test
+ void launchRetriesWithNextPortWhenDockerReportsHostPortCollision() throws Exception {
+ Ec2ContainerManager.containerBridgeIpAttempts = 1;
+ Ec2ContainerManager.containerBridgeIpPollMillis = 1;
+ LaunchHarness harness = launchHarness();
+ when(harness.portAllocator.allocate(anyInt(), anyInt())).thenReturn(2201, 2202);
+ when(harness.lifecycleManager.create(any(ContainerSpec.class)))
+ .thenReturn("container-conflict", TEST_CONTAINER_ID);
+ when(harness.lifecycleManager.startCreated(anyString(), any(ContainerSpec.class)))
+ .thenThrow(new RuntimeException("Bind for 0.0.0.0:2201 failed: port is already allocated"))
+ .thenReturn(null);
+
+ InspectContainerCmd inspect = mock(InspectContainerCmd.class);
+ InspectContainerResponse withIp = inspectResponse("172.18.0.12");
+ when(harness.dockerClient.inspectContainerCmd(TEST_CONTAINER_ID)).thenReturn(inspect);
+ when(inspect.exec()).thenReturn(withIp);
+ harness.stubSuccessfulExecs(new CountDownLatch(0), new CountDownLatch(0));
+
+ Instance instance = instance("i-port-collision");
+ harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");
+
+ awaitUntil(() -> "running".equals(instance.getState().getName()), Duration.ofSeconds(2));
+ assertEquals(2202, instance.getSshHostPort());
+ verify(harness.lifecycleManager).removeIfExists("container-conflict");
+ verify(harness.portAllocator).markReserved(2201);
+ verify(harness.builder).withPortBinding(22, 2201);
+ verify(harness.builder).withPortBinding(22, 2202);
+ }
+
+ @Test
+ void launchReleasesPortAndCleansUpContainerAfterNonRetryableStartFailure() throws Exception {
+ LaunchHarness harness = launchHarness();
+ when(harness.lifecycleManager.startCreated(eq(TEST_CONTAINER_ID), any(ContainerSpec.class)))
+ .thenThrow(new RuntimeException("Docker daemon unavailable"));
+
+ Instance instance = instance("i-start-failure");
+ harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");
+
+ awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));
+ verify(harness.lifecycleManager).removeIfExists(TEST_CONTAINER_ID);
+ verify(harness.portAllocator).release(2201);
+ }
+
+ @Test
+ void launchTerminatesBeforeCleanupFailure() throws Exception {
+ LaunchHarness harness = launchHarness();
+ when(harness.lifecycleManager.startCreated(eq(TEST_CONTAINER_ID), any(ContainerSpec.class)))
+ .thenThrow(new RuntimeException("Docker daemon unavailable"));
+ doThrow(new RuntimeException("port-forward cleanup failed"))
+ .when(harness.portForwardManager).unpublishAll(any(Instance.class));
+
+ Instance instance = instance("i-cleanup-failure");
+ harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");
+
+ awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));
+ verify(harness.portForwardManager).unpublishAll(instance);
+ }
+
+ @Test
+ void cancelledLaunchRemovesContainerThatStartsAfterCancellation() throws Exception {
+ LaunchHarness harness = launchHarness();
+ CountDownLatch startEntered = new CountDownLatch(1);
+ CountDownLatch allowStart = new CountDownLatch(1);
+ when(harness.lifecycleManager.startCreated(eq(TEST_CONTAINER_ID), any(ContainerSpec.class))).thenAnswer(invocation -> {
+ startEntered.countDown();
+ assertTrue(allowStart.await(2, TimeUnit.SECONDS));
+ return null;
+ });
+
+ Instance instance = instance("i-cancelled-launch");
+ harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");
+
+ assertTrue(startEntered.await(2, TimeUnit.SECONDS), "container startup should begin");
+ assertTrue(harness.manager.cancelLaunch(instance));
+ verify(harness.lifecycleManager).removeIfExists(TEST_CONTAINER_ID);
+ verify(harness.portAllocator).release(2201);
+
+ allowStart.countDown();
+ verify(harness.portAllocator, after(500).times(1)).release(2201);
+ }
+
+ @Test
+ void cancelledLaunchRemovesContainerCreatedAfterCancellation() throws Exception {
+ LaunchHarness harness = launchHarness();
+ CountDownLatch createEntered = new CountDownLatch(1);
+ CountDownLatch allowCreate = new CountDownLatch(1);
+ when(harness.lifecycleManager.create(any(ContainerSpec.class))).thenAnswer(invocation -> {
+ createEntered.countDown();
+ assertTrue(allowCreate.await(2, TimeUnit.SECONDS));
+ return TEST_CONTAINER_ID;
+ });
+
+ Instance instance = instance("i-cancelled-during-create");
+ harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");
+
+ assertTrue(createEntered.await(2, TimeUnit.SECONDS), "container creation should begin");
+ assertTrue(harness.manager.cancelLaunch(instance));
+ assertEquals("terminated", instance.getState().getName());
+
+ allowCreate.countDown();
+ verify(harness.lifecycleManager, timeout(2_000)).removeIfExists(TEST_CONTAINER_ID);
+ verify(harness.portAllocator, timeout(2_000)).release(2201);
+ verify(harness.lifecycleManager, never()).startCreated(eq(TEST_CONTAINER_ID), any(ContainerSpec.class));
+ assertEquals("terminated", instance.getState().getName());
+ }
+
@Test
void launchMarksInstanceRunningBeforeUserDataCompletes() throws Exception {
Ec2ContainerManager.containerBridgeIpAttempts = 2;
@@ -593,6 +703,7 @@ class Ec2ContainerManagerTest {
DockerClient dockerClient = mock(DockerClient.class);
Ec2MetadataServer metadataServer = mock(Ec2MetadataServer.class);
ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);
+ Ec2PortForwardManager portForwardManager = mock(Ec2PortForwardManager.class);
Ec2ContainerManager manager = new Ec2ContainerManager(
containerBuilder,
lifecycleManager,
@@ -603,9 +714,9 @@ class Ec2ContainerManagerTest {
portAllocator,
config,
metadataServer,
- mock(Ec2PortForwardManager.class));
- return new LaunchHarness(manager, dockerClient, metadataServer, logStreamer, builder,
- new CopyOnWriteArrayList<>());
+ portForwardManager);
+ return new LaunchHarness(manager, lifecycleManager, dockerClient, metadataServer, logStreamer, builder,
+ portAllocator, portForwardManager, new CopyOnWriteArrayList<>());
}
/**
@@ -649,10 +760,13 @@ class Ec2ContainerManagerTest {
}
private record LaunchHarness(Ec2ContainerManager manager,
+ ContainerLifecycleManager lifecycleManager,
DockerClient dockerClient,
Ec2MetadataServer metadataServer,
ContainerLogStreamer logStreamer,
ContainerBuilder.Builder builder,
+ PortAllocator portAllocator,
+ Ec2PortForwardManager portForwardManager,
List<String[]> executedCommands) {
void stubSuccessfulExecs(CountDownLatch userDataStarted, CountDownLatch finishUserData) throws Exception {
AtomicReference<String[]> currentCommand = new AtomicReference<>();
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
index abfcec7ee9..ff612c3ea0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.services.ec2.model.IpPermission;
import io.github.hectorvent.floci.services.ec2.model.Ipv6Range;
import io.github.hectorvent.floci.services.ec2.model.SecurityGroupRule;
import io.github.hectorvent.floci.services.ec2.model.UserIdGroupPair;
+import io.github.hectorvent.floci.services.ec2.model.InstanceState;
import io.github.hectorvent.floci.services.ec2.model.LaunchTemplate;
import io.github.hectorvent.floci.services.ec2.model.ManagedPrefixList;
import io.github.hectorvent.floci.services.ec2.model.SecurityGroupRule;
@@ -39,9 +40,9 @@ import io.github.hectorvent.floci.services.ec2.model.Vpc;
import io.github.hectorvent.floci.services.ec2.model.VpcEndpoint;
import io.github.hectorvent.floci.services.ec2.model.Volume;
import io.github.hectorvent.floci.services.ec2.model.VolumeAttachment;
-import io.github.hectorvent.floci.services.ec2.model.InstanceState;
import org.junit.jupiter.api.Test;
+import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -80,6 +81,40 @@ class Ec2ServiceTest {
verifyNoInteractions(containerManager);
}
+ @Test
+ void awaitContainerLaunchReportsTerminatedContainer() {
+ Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());
+ Instance instance = new Instance();
+ instance.setInstanceId("i-terminated");
+ instance.setState(InstanceState.terminated());
+
+ AwsException error = assertThrows(AwsException.class, () -> service.awaitContainerLaunch(instance));
+
+ assertEquals("InternalError", error.getErrorCode());
+ assertTrue(error.getMessage().contains("container terminated during launch"));
+ }
+
+ @Test
+ void awaitContainerLaunchTimesOutWhileInstanceIsPending() {
+ Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);
+ Ec2Service service = new Ec2Service(mockConfig(false), containerManager,
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());
+ Instance instance = new Instance();
+ instance.setInstanceId("i-pending");
+ instance.setState(InstanceState.pending());
+ when(containerManager.cancelLaunch(instance)).thenReturn(true);
+
+ AwsException error = assertThrows(AwsException.class,
+ () -> service.awaitContainerLaunch(instance, Duration.ZERO));
+
+ assertEquals("InternalError", error.getErrorCode());
+ assertTrue(error.getMessage().contains("did not reach running state before the launch timeout"));
+ verify(containerManager).cancelLaunch(instance);
+ }
+
@Test
void runInstancesRequiresImageIdInsteadOfDefaulting() {
Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),