Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2): register all guest network addresses with IMDS (#3546)
luiseterc3c6568bac548
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.javaindex 8f1b49747d..72ce554579 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.services.ec2.model.InstanceState;import io.github.hectorvent.floci.services.ec2.net.VpcNetworkManager;import io.github.hectorvent.floci.services.ec2.portforward.Ec2PortForwardManager;import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.InspectContainerResponse;import com.github.dockerjava.api.exception.NotFoundException;import com.github.dockerjava.api.async.ResultCallback;import com.github.dockerjava.api.model.Container;@@ -38,6 +39,7 @@ import java.nio.charset.StandardCharsets;import java.time.Duration;import java.util.ArrayList;import java.util.Base64;+import java.util.HashSet;import java.util.List;import java.util.Locale;import java.util.Map;@@ -367,6 +369,8 @@ public class Ec2ContainerManager {return;}+ refreshMetadataAddresses(instance, containerId);+if (!markRunning(instance)) {failLaunch(instance, leasedPrivateIp);return;@@ -579,6 +583,7 @@ public class Ec2ContainerManager {if (sshHostPort > 0) {portAllocator.release(sshHostPort);}+ metadataServer.unregisterInstance(instance);if (containerIp != null && !containerIp.isBlank()) {try {metadataServer.unregisterContainer(containerIp, instance);@@ -782,6 +787,7 @@ public class Ec2ContainerManager {}metadataServer.registerContainer(containerIp, instanceId, instance);refreshImdsSourceRegistration(instance, containerId, containerIp);+ refreshMetadataAddresses(instance, containerId);}} catch (InterruptedException e) {Thread.currentThread().interrupt();@@ -852,6 +858,28 @@ public class Ec2ContainerManager {return removed;}+ private void refreshMetadataAddresses(Instance instance, String containerId) {+ try {+ InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();+ if (inspect.getNetworkSettings() == null || inspect.getNetworkSettings().getNetworks() == null) {+ return;+ }+ Set<String> addresses = new HashSet<>();+ for (ContainerNetwork network : inspect.getNetworkSettings().getNetworks().values()) {+ if (network != null && network.getIpAddress() != null && !network.getIpAddress().isBlank()) {+ addresses.add(network.getIpAddress());+ }+ }+ // An incomplete Docker response must not discard the last known registrations.+ if (!addresses.isEmpty()) {+ metadataServer.reconcileContainerAddresses(addresses, instance);+ }+ } catch (RuntimeException e) {+ LOG.warnv("Could not refresh IMDS addresses for EC2 instance {0}, keeping existing registrations: {1}",+ instance.getInstanceId(), e.getMessage());+ }+ }+boolean restoreMetadataRegistration(Instance instance) {if (instance == null || instance.getDockerContainerId() == null) {return false;@@ -882,6 +910,7 @@ public class Ec2ContainerManager {}metadataServer.registerContainer(containerIp, instance.getInstanceId(), instance);refreshImdsSourceRegistration(instance, containerId, containerIp);+ refreshMetadataAddresses(instance, containerId);return true;}@@ -1066,6 +1095,7 @@ public class Ec2ContainerManager {}metadataServer.unregisterContainer(containerIp, instance);metadataServer.unregisterContainer(imdsSourceIp, instance);+ metadataServer.unregisterInstance(instance);// Give the address back only now that the container is gone: releasing it while// Docker still holds the endpoint would hand the same IP to the next launch and// have Docker refuse it.diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServer.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServer.javaindex 450ffcd78c..ac65a6fb11 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServer.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServer.java@@ -20,6 +20,7 @@ import java.time.ZoneOffset;import java.time.format.DateTimeFormatter;import java.util.Map;import java.util.Optional;+import java.util.Set;import java.util.UUID;import java.util.concurrent.CompletableFuture;import java.util.concurrent.ConcurrentHashMap;@@ -73,6 +74,19 @@ public class Ec2MetadataServer {}}+ /** Reconcile every Docker attachment without retaining stale addresses after restart. */+ void reconcileContainerAddresses(Set<String> addresses, Instance instance) {+ for (String address : addresses) {+ registerContainer(address, instance.getInstanceId(), instance);+ }+ containerIpToInstance.entrySet().removeIf(entry ->+ entry.getValue() == instance && !addresses.contains(entry.getKey()));+ }++ void unregisterInstance(Instance instance) {+ containerIpToInstance.entrySet().removeIf(entry -> entry.getValue() == instance);+ }+Optional<Instance> registeredContainer(String containerIp) {return Optional.ofNullable(containerIpToInstance.get(containerIp));}diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.javaindex 1f977bdc1a..4bff4728bf 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java@@ -39,8 +39,10 @@ import java.io.Closeable;import java.time.Duration;import java.util.ArrayList;import java.util.Arrays;+import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import java.util.Set;import java.util.Optional;import java.util.concurrent.ArrayBlockingQueue;import java.util.concurrent.CopyOnWriteArrayList;@@ -62,6 +64,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyInt;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.anySet;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Answers.RETURNS_SELF;import static org.mockito.Answers.RETURNS_DEEP_STUBS;@@ -203,6 +206,40 @@ class Ec2ContainerManagerTest {assertEquals("172.17.0.4", instance.getImdsSourceIp());}+ @Test+ void restoreRegistersAllNetworksAndReplacesStaleSharedAddress() {+ for (boolean sharedFirst : List.of(true, false)) {+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);+ when(lifecycle.isContainerRunning(TEST_CONTAINER_ID)).thenReturn(true);+ DockerClient docker = mock(DockerClient.class);+ InspectContainerCmd command = mock(InspectContainerCmd.class);+ when(docker.inspectContainerCmd(TEST_CONTAINER_ID)).thenReturn(command);+ InspectContainerResponse response = mock(InspectContainerResponse.class);+ NetworkSettings settings = mock(NetworkSettings.class);+ when(response.getNetworkSettings()).thenReturn(settings);+ Map<String, ContainerNetwork> networks = new LinkedHashMap<>();+ networks.put("bridge", new ContainerNetwork().withIpv4Address("172.17.0.4"));+ String first = sharedFirst ? "shared" : "vpc";+ String second = sharedFirst ? "vpc" : "shared";+ networks.put(first, new ContainerNetwork().withIpv4Address(sharedFirst ? "192.0.2.10" : "10.0.1.10"));+ networks.put(second, new ContainerNetwork().withIpv4Address(sharedFirst ? "10.0.1.10" : "192.0.2.10"));+ when(settings.getNetworks()).thenReturn(networks);+ when(command.exec()).thenReturn(response);+ Ec2MetadataServer server = new Ec2MetadataServer(null, null, null);+ Ec2ContainerManager manager = managerWith(lifecycle, docker, server);+ Instance guest = instance("i-multinetwork");+ guest.setDockerContainerId(TEST_CONTAINER_ID);+ server.registerContainer("192.0.2.9", guest.getInstanceId(), guest);++ assertTrue(manager.restoreMetadataRegistration(guest));++ for (String address : List.of("172.17.0.4", "10.0.1.10", "192.0.2.10")) {+ assertEquals(guest, server.registeredContainer(address).orElseThrow());+ }+ assertTrue(server.registeredContainer("192.0.2.9").isEmpty());+ }+ }+@Testvoid restoreMetadataRegistrationLeavesABridgeOnlyInstanceWithNoSeparateImdsSource() {ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);@@ -254,6 +291,7 @@ class Ec2ContainerManagerTest {// the registration down here would stop this healthy instance's metadata requests// resolving, with nothing registered in place of what was removed.verify(metadataServer, never()).unregisterContainer("172.17.0.4", instance);+ verify(metadataServer, never()).reconcileContainerAddresses(anySet(), any());assertEquals("172.17.0.4", instance.getImdsSourceIp(),"a transient Docker failure must not drop an existing IMDS source registration");}@@ -286,6 +324,8 @@ class Ec2ContainerManagerTest {verify(metadataServer, timeout(2000)).registerContainer("172.17.0.9", "i-vpc-started", instance);verify(metadataServer, timeout(2000)).unregisterContainer("172.17.0.4", instance);+ verify(metadataServer, timeout(2000)).reconcileContainerAddresses(+ Set.of("10.0.1.10", "172.17.0.9"), instance);assertEquals("172.17.0.9", instance.getImdsSourceIp(),"the stale address would otherwise stay registered while IMDS requests arrive "+ "from an address nothing knows about");@@ -838,6 +878,26 @@ class Ec2ContainerManagerTest {assertTrue(Ec2ContainerManager.preferredMetadataSourceIp(Map.of("bridge", bridge)).isEmpty());}+ @Test+ void launchRegistersAllGuestAddressesBeforeUserData() throws Exception {+ LaunchHarness harness = launchHarness();+ InspectContainerCmd command = mock(InspectContainerCmd.class);+ InspectContainerResponse response = vpcAttachedInspectResponse("10.0.1.10", "172.17.0.4");+ Map<String, ContainerNetwork> networks = new LinkedHashMap<>(response.getNetworkSettings().getNetworks());+ networks.put("shared", new ContainerNetwork().withIpv4Address("192.0.2.10"));+ when(response.getNetworkSettings().getNetworks()).thenReturn(networks);+ when(harness.dockerClient.inspectContainerCmd(TEST_CONTAINER_ID)).thenReturn(command);+ when(command.exec()).thenReturn(response);+ harness.stubSuccessfulExecs(new CountDownLatch(0), new CountDownLatch(0));+ Instance guest = instance("i-three-networks");++ harness.manager.launch(guest, "ubuntu:24.04", null, "us-west-2");++ awaitUntil(() -> "running".equals(guest.getState().getName()), Duration.ofSeconds(2));+ verify(harness.metadataServer).reconcileContainerAddresses(+ Set.of("10.0.1.10", "172.17.0.4", "192.0.2.10"), guest);+ }+@Testvoid launchWaitsForContainerBridgeIpBeforeRegisteringImds() throws Exception {Ec2ContainerManager.containerBridgeIpAttempts = 3;@@ -858,7 +918,7 @@ class Ec2ContainerManagerTest {assertEquals(TEST_CONTAINER_ID, instance.getDockerContainerId());assertEquals("172.18.0.9", instance.getContainerBridgeIp());assertEquals("172.18.0.9", instance.getPrivateIpAddress());- verify(inspect, times(2)).exec();+ verify(inspect, times(3)).exec();verify(harness.metadataServer).registerContainer("172.18.0.9", "i-waitip", instance);}diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServerTest.javaindex 5c1bdb8672..f61f3abe99 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServerTest.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.services.iam.model.InstanceProfile;import org.junit.jupiter.api.Test;import java.util.List;+import java.util.Set;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -16,6 +17,23 @@ import static org.mockito.Mockito.when;class Ec2MetadataServerTest {+ @Test+ void reconciliationAndTerminationPreserveOtherInstances() {+ Ec2MetadataServer server = new Ec2MetadataServer(null, null, null);+ Instance first = new Instance();+ first.setInstanceId("i-first");+ Instance second = new Instance();+ second.setInstanceId("i-second");+ server.reconcileContainerAddresses(Set.of("192.0.2.1", "192.0.2.2"), first);+ server.reconcileContainerAddresses(Set.of("192.0.2.2", "192.0.2.3"), second);+ server.reconcileContainerAddresses(Set.of("192.0.2.4"), first);+ assertTrue(server.registeredContainer("192.0.2.1").isEmpty());+ assertEquals(second, server.registeredContainer("192.0.2.2").orElseThrow());+ server.unregisterInstance(first);+ assertTrue(server.registeredContainer("192.0.2.4").isEmpty());+ assertEquals(second, server.registeredContainer("192.0.2.3").orElseThrow());+ }+@Testvoid instanceMetadataListsTagKeys() {Instance instance = new Instance();diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MultiNetworkMetadataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MultiNetworkMetadataIntegrationTest.javanew file mode 100644index 0000000000..006c4285b7--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MultiNetworkMetadataIntegrationTest.java@@ -0,0 +1,92 @@+package io.github.hectorvent.floci.services.ec2;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.InspectContainerCmd;+import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.model.ContainerNetwork;+import com.github.dockerjava.api.model.NetworkSettings;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import io.github.hectorvent.floci.core.common.docker.PortAllocator;+import io.github.hectorvent.floci.services.ec2.model.Instance;+import io.github.hectorvent.floci.services.ec2.net.VpcNetworkManager;+import io.github.hectorvent.floci.services.ec2.portforward.Ec2PortForwardManager;+import io.vertx.core.Vertx;+import org.junit.jupiter.api.Test;++import java.net.ServerSocket;+import java.net.URI;+import java.net.http.HttpClient;+import java.net.http.HttpRequest;+import java.net.http.HttpResponse;+import java.time.Duration;+import java.util.LinkedHashMap;+import java.util.Map;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.*;++class Ec2MultiNetworkMetadataIntegrationTest {+ @Test+ void restoredGuestCanReadMetadataThroughItsSharedNetworkAddress() throws Exception {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ int port;+ try (ServerSocket socket = new ServerSocket(0)) {+ port = socket.getLocalPort();+ }+ when(config.services().ec2().imdsPort()).thenReturn(port);+ Vertx vertx = Vertx.vertx();+ Ec2MetadataServer server = new Ec2MetadataServer(vertx, config, null);+ DockerClient docker = mock(DockerClient.class);+ InspectContainerCmd command = mock(InspectContainerCmd.class);+ InspectContainerResponse response = mock(InspectContainerResponse.class);+ NetworkSettings settings = mock(NetworkSettings.class);+ when(docker.inspectContainerCmd("guest")).thenReturn(command);+ when(command.exec()).thenReturn(response);+ when(response.getNetworkSettings()).thenReturn(settings);+ Map<String, ContainerNetwork> networks = new LinkedHashMap<>();+ networks.put("vpc", new ContainerNetwork().withIpv4Address("10.0.1.10"));+ networks.put("bridge", new ContainerNetwork().withIpv4Address("172.17.0.4"));+ // Use loopback for the actual HTTP client's source, standing in for the shared network.+ networks.put("shared", new ContainerNetwork().withIpv4Address("127.0.0.1"));+ when(settings.getNetworks()).thenReturn(networks);+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);+ when(lifecycle.isContainerRunning("guest")).thenReturn(true);+ Ec2ContainerManager manager = new Ec2ContainerManager(mock(ContainerBuilder.class), lifecycle,+ mock(ContainerLogStreamer.class), mock(ContainerDetector.class), mock(DockerHostResolver.class),+ docker, mock(PortAllocator.class), config, server, mock(Ec2PortForwardManager.class),+ mock(RegionResolver.class), mock(ContainerNetworkReachability.class), mock(VpcNetworkManager.class));+ Instance instance = new Instance();+ instance.setInstanceId("i-multinetwork");+ instance.setDockerContainerId("guest");+ instance.setUserData("#!/bin/sh\necho metadata\n");+ String endpoint = "http://127.0.0.1:" + port;+ try (HttpClient client = HttpClient.newHttpClient()) {+ server.start().get(10, TimeUnit.SECONDS);+ assertTrue(manager.restoreMetadataRegistration(instance));+ HttpResponse<String> token = client.send(HttpRequest.newBuilder(URI.create(endpoint + "/latest/api/token"))+ .timeout(Duration.ofSeconds(5)).header("X-aws-ec2-metadata-token-ttl-seconds", "60")+ .PUT(HttpRequest.BodyPublishers.noBody()).build(), HttpResponse.BodyHandlers.ofString());+ assertEquals(200, token.statusCode());+ for (Map.Entry<String, String> entry : Map.of("meta-data/instance-id", "i-multinetwork",+ "user-data", instance.getUserData()).entrySet()) {+ HttpResponse<String> metadata = client.send(HttpRequest.newBuilder(+ URI.create(endpoint + "/latest/" + entry.getKey()))+ .timeout(Duration.ofSeconds(5)).header("X-aws-ec2-metadata-token", token.body())+ .GET().build(), HttpResponse.BodyHandlers.ofString());+ assertEquals(200, metadata.statusCode());+ assertEquals(entry.getValue(), metadata.body());+ }+ } finally {+ manager.stop();+ server.stop();+ vertx.close().toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);+ }+ }+}