Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): register reachable task addresses in Cloud Map and bound synthesised ENI addresses (#4509)
Abhigya014156a546da
diff --git a/docs/services/ec2.md b/docs/services/ec2.mdindex 222e4e6399..b8c75ee84c 100644--- a/docs/services/ec2.md+++ b/docs/services/ec2.md@@ -862,6 +862,11 @@ Limits worth knowing before reading a passing test as evidence:Set `FLOCI_SERVICES_EC2_VPC_NETWORKS_ENABLED=false` to go back to synthesised private addressesand the shared default bridge. It is also off whenever `mock` is on.+Synthesised addresses come from the subnet's own CIDR block, skipping the first four addresses+and the last one as AWS reserves them, and skipping every address a persisted instance, network+interface or NAT gateway in the subnet still holds, so a restart does not hand one out twice. A+subnet with no free address left fails the request with `InsufficientFreeAddressesInSubnet`.+## RequirementsEC2 requires the Docker socket to be accessible (same as Lambda, ECS, and other container services):diff --git a/docs/services/ecs.md b/docs/services/ecs.mdindex 92fa31d403..c1c290f5c7 100644--- a/docs/services/ecs.md+++ b/docs/services/ecs.md@@ -469,10 +469,15 @@ Known differences from AWS:A service that declares `serviceRegistries` has each of its running tasks registered as aCloud Map instance of the named service, and deregistered when the task stops. The instance-carries `AWS_INSTANCE_IPV4` (the task's ENI address on an awsvpc task, otherwise the-container's address on the Docker network) and `AWS_INSTANCE_PORT` (the entry's `port`, or-the host port its `containerPort` was published on), alongside the metadata attributes AWS-records: `AVAILABILITY_ZONE`, `REGION`, `ECS_SERVICE_NAME`, `ECS_CLUSTER_NAME` and+carries `AWS_INSTANCE_IPV4` and `AWS_INSTANCE_PORT`, chosen by network mode:++| Network mode | `AWS_INSTANCE_IPV4` | `AWS_INSTANCE_PORT` |+|---|---|---|+| `awsvpc` | the address the task's container holds on its Docker network | the entry's `containerPort` |+| `bridge` or `host` | the address Floci reaches the container at: `127.0.0.1` when Floci runs natively, the container's Docker network address when Floci runs in Docker | the host port the entry's `containerPort` was published on, or the `containerPort` when it has no binding |++An explicit `port` on the registry entry is registered as-is in every mode, as on AWS. The+instance also carries the metadata attributes AWS records: `AVAILABILITY_ZONE`, `REGION`, `ECS_SERVICE_NAME`, `ECS_CLUSTER_NAME` and`ECS_TASK_DEFINITION_FAMILY`. The task id is the instance id, as on AWS, so a replacementtask supersedes its predecessor.@@ -488,6 +493,12 @@ Known differences from AWS:reachable port is the published host port rather than the container port. Floci servesonly A records, so such a task registers its address and its published host port andresolves by address; the port is readable through `DiscoverInstances` rather than DNS.+- An awsvpc task registers its container's Docker network address, not its ENI address. On+ AWS the two are the same interface. In Floci a task's containers never join the subnet's+ network, so the ENI address `DescribeTasks` reports is not one a peer can connect to. The+ registrar falls back to the ENI address only when it cannot read an address from a running+ container. Because the address is the container's own, the port is its `containerPort`, not+ any host port Floci also published it on.- `EC2_INSTANCE_ID` is not recorded. Every Floci task runs as a container rather than on aregistered EC2 host, which is the Fargate case on AWS, where the attribute is also absent.- An instance's health status never moves. AWS registers a task `UNHEALTHY` and promotes it todiff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.javaindex 17c84cdc03..8b4dedd4b9 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java@@ -84,6 +84,7 @@ import io.github.hectorvent.floci.services.ec2.model.VpcIpv6CidrBlockAssociationimport io.github.hectorvent.floci.services.ec2.model.VpcPeeringConnection;import io.github.hectorvent.floci.services.ec2.model.VpcPeeringConnectionStateReason;import io.github.hectorvent.floci.services.ec2.model.VpcPeeringConnectionVpcInfo;+import io.github.hectorvent.floci.services.ec2.net.Cidr4;import io.github.hectorvent.floci.services.ec2.net.VpcNetworkManager;import io.github.hectorvent.floci.services.ec2.portforward.Ec2PortForwardManager;import io.github.hectorvent.floci.services.iam.IamService;@@ -101,8 +102,10 @@ import java.time.ZoneOffset;import java.time.format.DateTimeFormatter;import java.util.ArrayList;import java.util.Base64;+import java.util.Collection;import java.util.Collections;import java.util.Comparator;+import java.util.HashMap;import java.util.HashSet;import java.util.LinkedHashMap;import java.util.LinkedHashSet;@@ -116,7 +119,6 @@ import java.util.Set;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.CopyOnWriteArrayList;-import java.util.concurrent.atomic.AtomicInteger;import java.util.function.BiPredicate;import java.util.function.Function;import java.util.regex.Pattern;@@ -137,6 +139,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {private static final DateTimeFormatter ISO_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC);private static final int DEFAULT_ROOT_VOLUME_SIZE_GIB = 8;+ private static final long SYNTHETIC_FIRST_OFFSET = 10;+ private static final String SYNTHETIC_FALLBACK_CIDR = "172.31.0.0/24";private static final String DEFAULT_ROOT_VOLUME_TYPE = "gp3";private static final Set<String> VALID_VOLUME_TYPES =Set.of("standard", "io1", "io2", "gp2", "sc1", "st1", "gp3");@@ -239,8 +243,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {private final StorageBackend<String, List<Tag>> tags;private final StorageBackend<String, CapacityReservation> capacityReservations;private final Set<String> seededAccountRegions = ConcurrentHashMap.newKeySet();- // subnetId → counter for IP assignment (runtime-only, not persisted)- private final Map<String, AtomicInteger> subnetIpCounters = new ConcurrentHashMap<>();+ // region::subnetId → offset the next synthesised address is tried at. Only an ordering hint:+ // restart forgets it, and privateIpsInUse is what keeps addresses from being handed out twice.+ private final Map<String, Long> subnetIpCursors = new HashMap<>();+ // Held from a synthesised address being chosen until the resource holding it is persisted, so+ // a concurrent allocation, which reads persisted resources to see what is taken, cannot pick+ // the same address in between. Taken after imageRegistryLock where a caller holds both.+ private final Object privateIpAllocationLock = new Object();/*** Null in the hermetic unit tests, which reach the constructors that do not take it; CDI always@@ -2968,111 +2977,114 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {if (dryRun) {throw new AwsException("DryRunOperation", "Request would have succeeded, but DryRun flag is set.", 412);}- for (int i = 0; i < count; i++) {- String instanceId = "i-" + randomHex(17);- String privateIp = suppliedEni != null- ? suppliedEni.getPrivateIpAddress()- : assignPrivateIp(region, finalSubnetId);-- Instance inst = new Instance();- inst.setInstanceId(instanceId);- inst.setImageId(imageId);- inst.setState(InstanceState.pending());- inst.setInstanceType(effectiveInstanceType);- inst.setPlacement(new Placement(az));- inst.setSubnetId(finalSubnetId);- inst.setVpcId(vpcId);- // AWS precedence (#1984): the launch-time AssociatePublicIpAddress- // override wins in both directions; the subnet's MapPublicIpOnLaunch- // attribute is only the default when the launch does not specify it.- inst.setAssociatePublicIp(associatePublicIp != null- ? associatePublicIp- : subnet != null && subnet.isMapPublicIpOnLaunch());- inst.setPrivateIpAddress(privateIp);- inst.setPrivateDnsName("ip-" + privateIp.replace('.', '-') + ".ec2.internal");- inst.setKeyName(keyName);- inst.setSecurityGroups(new ArrayList<>(sgIdentifiers));- inst.setArchitecture(architecture);- inst.setLaunchTime(Instant.now());- inst.setAmiLaunchIndex(i);- inst.setClientToken(clientToken);- inst.setRegion(region);- inst.setUserData(userData);- inst.setEncodedUserData(encodedUserData);- inst.setIamInstanceProfileArn(iamInstanceProfileArn);- if (iamInstanceProfileArn != null) {- inst.setIamInstanceProfileAssociationTime(inst.getLaunchTime());- }- inst.setMetadataOptions(LaunchTemplateData.MetadataOptions.merge(launchMetadataOptions, null));- inst.setCreditSpecificationCpuCredits(- acquiredCpuCredits(effectiveInstanceType, creditSpecificationCpuCredits));- if (instanceTags != null && !instanceTags.isEmpty()) {- inst.setTags(new ArrayList<>(instanceTags));- tags.put(instanceId, new ArrayList<>(instanceTags));- }-- // Network interface, either the caller-supplied standalone ENI (override-default-eni,- // floci-kt9) or a freshly-minted implicit primary interface.- InstanceNetworkInterface eni = new InstanceNetworkInterface();- eni.setNetworkInterfaceId(suppliedEni != null ? suppliedEni.getNetworkInterfaceId() : "eni-" + randomHex(17));- eni.setSubnetId(finalSubnetId);- eni.setVpcId(vpcId);- eni.setOwnerId(callerAccountId());- eni.setDescription(suppliedEni != null ? suppliedEni.getDescription() : null);- eni.setMacAddress(suppliedEni != null ? suppliedEni.getMacAddress() : null);- eni.setPrivateIpAddress(privateIp);- eni.setPrivateDnsName(inst.getPrivateDnsName());- eni.setGroups(new ArrayList<>(sgIdentifiers));- eni.setAttachmentId("eni-attach-" + randomHex(17));- eni.setDeviceIndex(suppliedEni != null ? networkInterfaceDeviceIndex : 0);- if (inst.getLaunchTime() != null) {- eni.setAttachTime(ISO_FMT.format(inst.getLaunchTime()));- }- inst.getNetworkInterfaces().add(eni);- if (suppliedEni != null) {- // The standalone record stays authoritative rather than being folded into the- // instance: AWS defaults deleteOnTermination to false for an interface the caller- // created and handed to a launch, so it outlives the instance and returns to- // "available" on termination instead of vanishing with it. Double-counting is- // avoided in describeNetworkInterfaces, which skips the instance-side copy of any- // id the standalone store owns.- NetworkInterfaceAttachment launchAttachment = new NetworkInterfaceAttachment();- launchAttachment.setAttachmentId(eni.getAttachmentId());- launchAttachment.setDeviceIndex(eni.getDeviceIndex());- launchAttachment.setStatus("attached");- launchAttachment.setInstanceId(instanceId);- launchAttachment.setInstanceOwnerId(callerAccountId());- launchAttachment.setAttachTime(eni.getAttachTime());- launchAttachment.setDeleteOnTermination(false);- suppliedEni.setAttachment(launchAttachment);- suppliedEni.setStatus("in-use");- networkInterfaces.put(key(region, suppliedEni.getNetworkInterfaceId()), suppliedEni);- }+ synchronized (privateIpAllocationLock) {+ List<String> privateIps = allocateLaunchAddresses(region, finalSubnetId, suppliedEni, count);+ for (int i = 0; i < count; i++) {+ String instanceId = "i-" + randomHex(17);+ String privateIp = privateIps.get(i);++ Instance inst = new Instance();+ inst.setInstanceId(instanceId);+ inst.setImageId(imageId);+ inst.setState(InstanceState.pending());+ inst.setInstanceType(effectiveInstanceType);+ inst.setPlacement(new Placement(az));+ inst.setSubnetId(finalSubnetId);+ inst.setVpcId(vpcId);+ // AWS precedence (#1984): the launch-time AssociatePublicIpAddress+ // override wins in both directions; the subnet's MapPublicIpOnLaunch+ // attribute is only the default when the launch does not specify it.+ inst.setAssociatePublicIp(associatePublicIp != null+ ? associatePublicIp+ : subnet != null && subnet.isMapPublicIpOnLaunch());+ inst.setPrivateIpAddress(privateIp);+ inst.setPrivateDnsName("ip-" + privateIp.replace('.', '-') + ".ec2.internal");+ inst.setKeyName(keyName);+ inst.setSecurityGroups(new ArrayList<>(sgIdentifiers));+ inst.setArchitecture(architecture);+ inst.setLaunchTime(Instant.now());+ inst.setAmiLaunchIndex(i);+ inst.setClientToken(clientToken);+ inst.setRegion(region);+ inst.setUserData(userData);+ inst.setEncodedUserData(encodedUserData);+ inst.setIamInstanceProfileArn(iamInstanceProfileArn);+ if (iamInstanceProfileArn != null) {+ inst.setIamInstanceProfileAssociationTime(inst.getLaunchTime());+ }+ inst.setMetadataOptions(LaunchTemplateData.MetadataOptions.merge(launchMetadataOptions, null));+ inst.setCreditSpecificationCpuCredits(+ acquiredCpuCredits(effectiveInstanceType, creditSpecificationCpuCredits));+ if (instanceTags != null && !instanceTags.isEmpty()) {+ inst.setTags(new ArrayList<>(instanceTags));+ tags.put(instanceId, new ArrayList<>(instanceTags));+ }- // Root EBS volume- String rootVolId = "vol-" + randomHex(17);- inst.setRootVolumeId(rootVolId);- Volume rootVol = new Volume();- rootVol.setVolumeId(rootVolId);- rootVol.setAvailabilityZone(az);- rootVol.setVolumeType(DEFAULT_ROOT_VOLUME_TYPE);- rootVol.setSize(DEFAULT_ROOT_VOLUME_SIZE_GIB);- rootVol.setState("in-use");- rootVol.setRegion(region);- rootVol.setCreateTime(Instant.now());- VolumeAttachment att = new VolumeAttachment();- att.setVolumeId(rootVolId);- att.setInstanceId(instanceId);- att.setDevice(inst.getRootDeviceName());- att.setState("attached");- att.setDeleteOnTermination(true);- att.setAttachTime(Instant.now());- rootVol.getAttachments().add(att);- volumes.put(key(region, rootVolId), rootVol);+ // Network interface, either the caller-supplied standalone ENI (override-default-eni,+ // floci-kt9) or a freshly-minted implicit primary interface.+ InstanceNetworkInterface eni = new InstanceNetworkInterface();+ eni.setNetworkInterfaceId(suppliedEni != null+ ? suppliedEni.getNetworkInterfaceId()+ : "eni-" + randomHex(17));+ eni.setSubnetId(finalSubnetId);+ eni.setVpcId(vpcId);+ eni.setOwnerId(callerAccountId());+ eni.setDescription(suppliedEni != null ? suppliedEni.getDescription() : null);+ eni.setMacAddress(suppliedEni != null ? suppliedEni.getMacAddress() : null);+ eni.setPrivateIpAddress(privateIp);+ eni.setPrivateDnsName(inst.getPrivateDnsName());+ eni.setGroups(new ArrayList<>(sgIdentifiers));+ eni.setAttachmentId("eni-attach-" + randomHex(17));+ eni.setDeviceIndex(suppliedEni != null ? networkInterfaceDeviceIndex : 0);+ if (inst.getLaunchTime() != null) {+ eni.setAttachTime(ISO_FMT.format(inst.getLaunchTime()));+ }+ inst.getNetworkInterfaces().add(eni);+ if (suppliedEni != null) {+ // The standalone record stays authoritative rather than being folded into the+ // instance: AWS defaults deleteOnTermination to false for an interface the caller+ // created and handed to a launch, so it outlives the instance and returns to+ // "available" on termination instead of vanishing with it. Double-counting is+ // avoided in describeNetworkInterfaces, which skips the instance-side copy of any+ // id the standalone store owns.+ NetworkInterfaceAttachment launchAttachment = new NetworkInterfaceAttachment();+ launchAttachment.setAttachmentId(eni.getAttachmentId());+ launchAttachment.setDeviceIndex(eni.getDeviceIndex());+ launchAttachment.setStatus("attached");+ launchAttachment.setInstanceId(instanceId);+ launchAttachment.setInstanceOwnerId(callerAccountId());+ launchAttachment.setAttachTime(eni.getAttachTime());+ launchAttachment.setDeleteOnTermination(false);+ suppliedEni.setAttachment(launchAttachment);+ suppliedEni.setStatus("in-use");+ networkInterfaces.put(key(region, suppliedEni.getNetworkInterfaceId()), suppliedEni);+ }- instances.put(key(region, instanceId), inst);- launched.add(inst);- reservation.getInstances().add(inst);+ // Root EBS volume+ String rootVolId = "vol-" + randomHex(17);+ inst.setRootVolumeId(rootVolId);+ Volume rootVol = new Volume();+ rootVol.setVolumeId(rootVolId);+ rootVol.setAvailabilityZone(az);+ rootVol.setVolumeType(DEFAULT_ROOT_VOLUME_TYPE);+ rootVol.setSize(DEFAULT_ROOT_VOLUME_SIZE_GIB);+ rootVol.setState("in-use");+ rootVol.setRegion(region);+ rootVol.setCreateTime(Instant.now());+ VolumeAttachment att = new VolumeAttachment();+ att.setVolumeId(rootVolId);+ att.setInstanceId(instanceId);+ att.setDevice(inst.getRootDeviceName());+ att.setState("attached");+ att.setDeleteOnTermination(true);+ att.setAttachTime(Instant.now());+ rootVol.getAttachments().add(att);+ volumes.put(key(region, rootVolId), rootVol);++ instances.put(key(region, instanceId), inst);+ launched.add(inst);+ reservation.getInstances().add(inst);+ }}}@@ -3302,7 +3314,44 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {"The availability zone '" + availabilityZone + "' has no subnet in this region.", 400));}+ /**+ * Every private address a launch needs, allocated before any of its instances is stored, so+ * a subnet without room for the whole launch fails it without leaving the instances that did+ * fit behind. Callers must hold {@link #privateIpAllocationLock}.+ */+ private List<String> allocateLaunchAddresses(String region, String subnetId, NetworkInterface suppliedEni,+ int count) {+ List<String> addresses = new ArrayList<>(count);+ if (suppliedEni != null) {+ for (int i = 0; i < count; i++) {+ addresses.add(suppliedEni.getPrivateIpAddress());+ }+ return addresses;+ }+ try {+ for (int i = 0; i < count; i++) {+ addresses.add(assignPrivateIp(region, subnetId, addresses));+ }+ } catch (AwsException e) {+ if (vpcNetworkManager != null) {+ for (String address : addresses) {+ vpcNetworkManager.releasePrivateIp(region, subnetId, address);+ }+ }+ throw e;+ }+ return addresses;+ }+private String assignPrivateIp(String region, String subnetId) {+ return assignPrivateIp(region, subnetId, List.of());+ }++ /**+ * @param pending addresses already handed out to resources that are not stored yet, which the+ * synthesised fallback must not hand out again+ */+ private String assignPrivateIp(String region, String subnetId, Collection<String> pending) {// A Docker-backed subnet allocates the real thing: an address on the network the// instance's container will actually hold. Only when there is no such network does// this fall back to the synthesised address below, which nothing can connect to.@@ -3315,17 +3364,88 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {if (subnetId == null) {return "172.31.0." + (10 + new Random().nextInt(200));}- AtomicInteger counter = subnetIpCounters.computeIfAbsent(region + "::" + subnetId, k -> new AtomicInteger(10));- int offset = counter.getAndIncrement();- Subnet subnet = subnets.get(key(region, subnetId)).orElse(null);- if (subnet == null) {- return "172.31.0." + offset;+ Cidr4 cidr = subnets.get(key(region, subnetId))+ .flatMap(subnet -> Cidr4.parse(subnet.getCidrBlock()))+ .or(() -> Cidr4.parse(SYNTHETIC_FALLBACK_CIDR))+ .orElseThrow();+ // AWS reserves the first four addresses of a subnet and its last one.+ long first = 4;+ long last = cidr.size() - 2;+ if (last < first) {+ throw insufficientFreeAddresses(subnetId);+ }+ String cursorKey = key(region, subnetId);+ synchronized (privateIpAllocationLock) {+ // The addresses persisted resources already hold, so a restart, which forgets the+ // cursor, never hands out an address that is still in use.+ Set<String> inUse = privateIpsInUse(region, subnetId);+ inUse.addAll(pending);+ long start = Math.clamp(subnetIpCursors.getOrDefault(cursorKey, SYNTHETIC_FIRST_OFFSET), first, last);+ long span = last - first + 1;+ for (long step = 0; step < span; step++) {+ long offset = first + (start - first + step) % span;+ String address = cidr.addressAt(offset).orElseThrow();+ if (!inUse.contains(address)) {+ subnetIpCursors.put(cursorKey, offset + 1);+ return address;+ }+ }+ }+ throw insufficientFreeAddresses(subnetId);+ }++ private static AwsException insufficientFreeAddresses(String subnetId) {+ return new AwsException("InsufficientFreeAddressesInSubnet",+ "There are not enough free addresses in subnet '" + subnetId + "' to satisfy the requested number of instances.",+ 400);+ }++ /** Every private address a live instance, network interface or NAT gateway holds in the subnet. */+ private Set<String> privateIpsInUse(String region, String subnetId) {+ String regionPrefix = region + "::";+ Set<String> inUse = new HashSet<>();+ for (NetworkInterface ni : networkInterfaces.scan(k -> k.startsWith(regionPrefix))) {+ if (!subnetId.equals(ni.getSubnetId())) {+ continue;+ }+ addIfSet(inUse, ni.getPrivateIpAddress());+ for (NetworkInterfacePrivateIpAddress ip : nullToEmpty(ni.getPrivateIpAddresses())) {+ addIfSet(inUse, ip.getPrivateIpAddress());+ }+ }+ for (Instance instance : instances.scan(k -> k.startsWith(regionPrefix))) {+ String state = instance.getState() == null ? null : instance.getState().getName();+ if ("terminated".equals(state)) {+ continue;+ }+ if (subnetId.equals(instance.getSubnetId())) {+ addIfSet(inUse, instance.getPrivateIpAddress());+ }+ for (InstanceNetworkInterface ni : nullToEmpty(instance.getNetworkInterfaces())) {+ if (subnetId.equals(ni.getSubnetId())) {+ addIfSet(inUse, ni.getPrivateIpAddress());+ }+ }+ }+ for (NatGateway natGateway : natGateways.scan(k -> k.startsWith(regionPrefix))) {+ if (!subnetId.equals(natGateway.getSubnetId()) || "deleted".equals(natGateway.getState())) {+ continue;+ }+ for (NatGatewayAddress address : nullToEmpty(natGateway.getNatGatewayAddresses())) {+ addIfSet(inUse, address.getPrivateIp());+ }+ }+ return inUse;+ }++ private static <T> List<T> nullToEmpty(List<T> list) {+ return list == null ? List.of() : list;+ }++ private static void addIfSet(Set<String> addresses, String address) {+ if (address != null && !address.isBlank()) {+ addresses.add(address);}- // Parse base IP from CIDR- String cidr = subnet.getCidrBlock();- String baseIp = cidr.split("/")[0];- String[] parts = baseIp.split("\\.");- return parts[0] + "." + parts[1] + "." + parts[2] + "." + offset;}public List<Reservation> describeInstances(String region, List<String> instanceIds, Map<String, List<String>> filters) {@@ -7973,12 +8093,14 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {natGateway.setConnectivityType(connectivityType != null && !connectivityType.isBlank() ? connectivityType : "public");natGateway.setCreateTime(Instant.now());natGateway.setRegion(region);- natGateway.getNatGatewayAddresses().add(natGatewayAddress(region, subnetId, allocationId));- if (natGatewayTags != null && !natGatewayTags.isEmpty()) {- natGateway.setTags(new ArrayList<>(natGatewayTags));- tags.put(natGateway.getNatGatewayId(), new ArrayList<>(natGatewayTags));+ synchronized (privateIpAllocationLock) {+ natGateway.getNatGatewayAddresses().add(natGatewayAddress(region, subnetId, allocationId));+ if (natGatewayTags != null && !natGatewayTags.isEmpty()) {+ natGateway.setTags(new ArrayList<>(natGatewayTags));+ tags.put(natGateway.getNatGatewayId(), new ArrayList<>(natGatewayTags));+ }+ natGateways.put(key(region, natGateway.getNatGatewayId()), natGateway);}- natGateways.put(key(region, natGateway.getNatGatewayId()), natGateway);return natGateway;}@@ -9199,49 +9321,51 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {}}- String eniId = "eni-" + randomHex(17);- String primaryIp = (privateIpAddress != null && !privateIpAddress.isBlank())- ? privateIpAddress : assignPrivateIp(region, subnetId);- String primaryDns = "ip-" + primaryIp.replace('.', '-') + ".ec2.internal";+ synchronized (privateIpAllocationLock) {+ String eniId = "eni-" + randomHex(17);+ String primaryIp = (privateIpAddress != null && !privateIpAddress.isBlank())+ ? privateIpAddress : assignPrivateIp(region, subnetId);+ String primaryDns = "ip-" + primaryIp.replace('.', '-') + ".ec2.internal";- NetworkInterface ni = new NetworkInterface();- ni.setNetworkInterfaceId(eniId);- ni.setSubnetId(subnetId);- ni.setVpcId(subnet.getVpcId());- ni.setAvailabilityZone(subnet.getAvailabilityZone());- ni.setDescription(description);- ni.setOwnerId(callerAccountId());- ni.setStatus("available");- ni.setMacAddress(randomMac());- ni.setPrivateIpAddress(primaryIp);- ni.setPrivateDnsName(primaryDns);- ni.setGroups(sgIdentifiers);- if (tagList != null) {- ni.getTagSet().addAll(tagList);- }-- List<NetworkInterfacePrivateIpAddress> ipList = new ArrayList<>();- NetworkInterfacePrivateIpAddress primary = new NetworkInterfacePrivateIpAddress();- primary.setPrivateIpAddress(primaryIp);- primary.setPrivateDnsName(primaryDns);- primary.setPrimary(true);- ipList.add(primary);- if (privateIpAddresses != null) {- for (String extra : privateIpAddresses) {- if (extra == null || extra.isBlank() || extra.equals(primaryIp)) {- continue;+ NetworkInterface ni = new NetworkInterface();+ ni.setNetworkInterfaceId(eniId);+ ni.setSubnetId(subnetId);+ ni.setVpcId(subnet.getVpcId());+ ni.setAvailabilityZone(subnet.getAvailabilityZone());+ ni.setDescription(description);+ ni.setOwnerId(callerAccountId());+ ni.setStatus("available");+ ni.setMacAddress(randomMac());+ ni.setPrivateIpAddress(primaryIp);+ ni.setPrivateDnsName(primaryDns);+ ni.setGroups(sgIdentifiers);+ if (tagList != null) {+ ni.getTagSet().addAll(tagList);+ }++ List<NetworkInterfacePrivateIpAddress> ipList = new ArrayList<>();+ NetworkInterfacePrivateIpAddress primary = new NetworkInterfacePrivateIpAddress();+ primary.setPrivateIpAddress(primaryIp);+ primary.setPrivateDnsName(primaryDns);+ primary.setPrimary(true);+ ipList.add(primary);+ if (privateIpAddresses != null) {+ for (String extra : privateIpAddresses) {+ if (extra == null || extra.isBlank() || extra.equals(primaryIp)) {+ continue;+ }+ NetworkInterfacePrivateIpAddress secondary = new NetworkInterfacePrivateIpAddress();+ secondary.setPrivateIpAddress(extra);+ secondary.setPrivateDnsName("ip-" + extra.replace('.', '-') + ".ec2.internal");+ secondary.setPrimary(false);+ ipList.add(secondary);}- NetworkInterfacePrivateIpAddress secondary = new NetworkInterfacePrivateIpAddress();- secondary.setPrivateIpAddress(extra);- secondary.setPrivateDnsName("ip-" + extra.replace('.', '-') + ".ec2.internal");- secondary.setPrimary(false);- ipList.add(secondary);}- }- ni.setPrivateIpAddresses(ipList);+ ni.setPrivateIpAddresses(ipList);- networkInterfaces.put(key(region, eniId), ni);- return ni;+ networkInterfaces.put(key(region, eniId), ni);+ return ni;+ }}/**diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex 3610a2ac41..d35205d55b 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -1892,12 +1892,15 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab/*** The addresses at which a Cloud Map instance resolves to a running task: its ENI address,* within the task's VPC only, and each container's address on Docker, on any port since a- * DNS answer carries none. A loopback address counts only on the task's host ports.+ * DNS answer carries none. A loopback address counts only on the task's host ports. An+ * awsvpc task's containers count at the address each holds on its Docker network, which is+ * where {@link EcsServiceDiscoveryRegistrar} registers them, even when Floci runs natively.*/private Set<EcsTaskAddress> instanceAddresses(EcsTask task, String region) {Set<EcsTaskAddress> addresses = new LinkedHashSet<>();String eniAddress = task.getPrivateIpAddress();- if (eniAddress != null && !eniAddress.isBlank() && !EcsTaskAddress.isLoopback(eniAddress)) {+ boolean awsvpc = eniAddress != null && !eniAddress.isBlank();+ if (awsvpc && !EcsTaskAddress.isLoopback(eniAddress)) {String vpcId = containerManager.taskVpcId(task, region);if (vpcId != null) {addresses.add(new EcsTaskAddress(eniAddress, null, vpcId));@@ -1907,7 +1910,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabreturn addresses;}for (Container container : task.getContainers()) {- String host = containerManager.resolveContainerHost(container);+ String host = awsvpc+ ? containerManager.resolvePeerAddress(container).orElse(null)+ : containerManager.resolveContainerHost(container);if (host == null || host.isBlank()) {continue;}diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrar.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrar.javaindex 1c82e9b572..9a3bc72ece 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrar.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrar.java@@ -246,9 +246,12 @@ public class EcsServiceDiscoveryRegistrar {}/**- * Builds the instance attributes AWS records for an ECS-registered instance. The address is- * the task's own ENI address in awsvpc mode, and otherwise the container's address on the- * Docker network, which is the one another container on that network can reach.+ * Builds the instance attributes AWS records for an ECS-registered instance. On AWS the+ * address of an awsvpc task is its ENI address, because that is where the task answers. In+ * Floci the ENI address belongs to no network the task's containers joined, so an awsvpc task+ * registers the address its container actually holds, and falls back to the ENI address only+ * when no running container can say what it holds. A bridge-mode task has no ENI and is+ * registered at the container's host address, as before.** <p>The metadata attributes alongside it are the ones the ECS service discovery* documentation lists, so a caller can filter a {@code DiscoverInstances} response by them@@ -259,7 +262,10 @@ public class EcsServiceDiscoveryRegistrar {Map<String, Object> registry, String region) {Container container = containerFor(task, string(registry, "containerName"));String address = task.getPrivateIpAddress();- if ((address == null || address.isBlank()) && container != null) {+ boolean awsvpc = address != null && !address.isBlank();+ if (awsvpc) {+ address = containerManager.resolvePeerAddress(container).orElse(address);+ } else if (container != null) {address = containerManager.resolveContainerHost(container);}if (address == null || address.isBlank()) {@@ -268,7 +274,7 @@ public class EcsServiceDiscoveryRegistrar {Map<String, String> attributes = new LinkedHashMap<>();attributes.put("AWS_INSTANCE_IPV4", address);- Integer port = instancePort(container, registry);+ Integer port = instancePort(container, registry, awsvpc);if (port != null) {attributes.put("AWS_INSTANCE_PORT", String.valueOf(port));}@@ -316,19 +322,20 @@ public class EcsServiceDiscoveryRegistrar {}/**- * The port an SRV record would carry. An explicit {@code port} wins, as it does on AWS;- * otherwise the host port the selected container's {@code containerPort} was published on,- * so a bridge-mode task advertises the port that is reachable rather than the one inside the- * container. A {@code containerPort} with no matching binding falls back to itself, which is- * the awsvpc case where the two are always equal.+ * The port an SRV record would carry. An explicit {@code port} wins, as it does on AWS.+ * An awsvpc task is registered at its container's own address, where it listens on the+ * {@code containerPort} itself, even when Floci also published that port on a dynamic host+ * port. A bridge-mode task is registered at the host, so it advertises the host port its+ * {@code containerPort} was published on, falling back to the {@code containerPort} when+ * there is no matching binding.*/- private Integer instancePort(Container container, Map<String, Object> registry) {+ private Integer instancePort(Container container, Map<String, Object> registry, boolean awsvpc) {Integer port = integer(registry, "port");if (port != null) {return port;}Integer containerPort = integer(registry, "containerPort");- if (container == null || containerPort == null || container.getNetworkBindings() == null) {+ if (awsvpc || container == null || containerPort == null || container.getNetworkBindings() == null) {return containerPort;}return container.getNetworkBindings().stream()diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.javaindex 0f99f1213f..31fc92a742 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java@@ -1916,13 +1916,40 @@ public class EcsContainerManager {if (!containerDetector.isRunningInContainer()) {return "127.0.0.1";}+ return dockerNetworkAddress(container).orElse("127.0.0.1");+ }++ /**+ * The address another task container reaches this one at: its IP on the Docker network it+ * joined, whether Floci itself runs natively or in Docker. A task in a security-group+ * protected namespace has no network of its own, so the namespace holder's address is the one+ * it answers on.+ *+ * <p>This is deliberately not the task ENI address. ECS containers never join the subnet's+ * VPC network, so the ENI address is control-plane state that no peer can connect to.+ *+ * @return the address, or empty when the container has not started or Docker cannot say+ */+ public Optional<String> resolvePeerAddress(Container container) {+ if (container == null || container.getDockerId() == null || container.getDockerId().isBlank()) {+ return Optional.empty();+ }+ return dockerNetworkAddress(container);+ }++ private Optional<String> dockerNetworkAddress(Container container) {String dockerId = container.getDockerId();if (dockerId == null || dockerId.isBlank()) {- return "127.0.0.1";+ return Optional.empty();}try {- var inspect = lifecycleManager.getDockerClient().inspectContainerCmd(dockerId).exec();- var networks = inspect.getNetworkSettings().getNetworks();+ InspectContainerResponse inspect = lifecycleManager.getDockerClient().inspectContainerCmd(dockerId).exec();+ String networkMode = inspect.getHostConfig() == null ? null : inspect.getHostConfig().getNetworkMode();+ if (networkMode != null && networkMode.startsWith("container:")) {+ inspect = lifecycleManager.getDockerClient()+ .inspectContainerCmd(networkMode.substring("container:".length())).exec();+ }+ Map<String, ContainerNetwork> networks = inspect.getNetworkSettings().getNetworks();// A container can be on multiple networks; getNetworks() is unordered.// Pick an IP that the Floci/ELBv2 process can actually route to:@@ -1932,26 +1959,26 @@ public class EcsContainerManager {// 3. otherwise the first non-blank IP.String configured = config.services().ecs().dockerNetwork().orElse(null);if (configured != null && !configured.isBlank()) {- var net = networks.get(configured);+ ContainerNetwork net = networks.get(configured);if (net != null && isUsableIp(net.getIpAddress())) {- return net.getIpAddress();+ return Optional.of(net.getIpAddress());}}- for (var entry : networks.entrySet()) {+ for (Map.Entry<String, ContainerNetwork> entry : networks.entrySet()) {if (!isDefaultDockerNetwork(entry.getKey())&& isUsableIp(entry.getValue().getIpAddress())) {- return entry.getValue().getIpAddress();+ return Optional.of(entry.getValue().getIpAddress());}}- for (var net : networks.values()) {+ for (ContainerNetwork net : networks.values()) {if (isUsableIp(net.getIpAddress())) {- return net.getIpAddress();+ return Optional.of(net.getIpAddress());}}} catch (Exception e) {LOG.warnv("Could not resolve container IP for {0}: {1}", dockerId, e.getMessage());}- return "127.0.0.1";+ return Optional.empty();}private static boolean isUsableIp(String ip) {diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.javaindex 5121a67631..16633f4aa6 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java@@ -58,12 +58,15 @@ import java.io.StringReader;import java.security.interfaces.RSAPrivateCrtKey;import java.time.Duration;import java.util.HashMap;+import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.Set;import java.util.concurrent.CountDownLatch;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;+import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;import java.util.function.BiConsumer;import java.util.function.Predicate;@@ -2080,6 +2083,130 @@ class Ec2ServiceTest {verify(vpcNetworks, times(2)).reservePrivateIp("us-east-1", "subnet-a", "10.0.1.10");}+ @Test+ void synthesisedAddressesStayInsideTheSubnetCidrPastTheFirstOctetBoundary() {+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());+ String vpcId = service.createVpc("us-east-1", "10.70.0.0/16", false).getVpcId();+ String subnetId = service.createSubnet("us-east-1", vpcId, "10.70.0.0/23", "us-east-1a").getSubnetId();++ Set<String> addresses = new HashSet<>();+ for (int i = 0; i < 300; i++) {+ String address = service.createNetworkInterface("us-east-1", subnetId, null,+ null, List.of(), List.of(), List.of()).getPrivateIpAddress();+ assertTrue(address.matches("10\\.70\\.[01]\\.(\\d{1,3})"), address);+ assertTrue(Integer.parseInt(address.substring(address.lastIndexOf('.') + 1)) <= 255, address);+ assertTrue(addresses.add(address), "handed out twice: " + address);+ }+ assertTrue(addresses.contains("10.70.1.10"), "the counter must carry into the next /24 octet");+ }++ @Test+ void synthesisedAddressesSkipReservedOnesAndReportExhaustion() {+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());+ String vpcId = service.createVpc("us-east-1", "10.71.0.0/16", false).getVpcId();+ String subnetId = service.createSubnet("us-east-1", vpcId, "10.71.0.0/28", "us-east-1a").getSubnetId();++ Set<String> addresses = new HashSet<>();+ for (int i = 0; i < 11; i++) {+ addresses.add(service.createNetworkInterface("us-east-1", subnetId, null,+ null, List.of(), List.of(), List.of()).getPrivateIpAddress());+ }++ Set<String> usable = new HashSet<>();+ for (int host = 4; host <= 14; host++) {+ usable.add("10.71.0." + host);+ }+ assertEquals(usable, addresses);+ assertEquals("InsufficientFreeAddressesInSubnet", assertThrows(AwsException.class,+ () -> service.createNetworkInterface("us-east-1", subnetId, null,+ null, List.of(), List.of(), List.of())).getErrorCode());+ }++ @Test+ void aLaunchTooLargeForTheSubnetStoresNoneOfItsInstances() {+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());+ String vpcId = service.createVpc("us-east-1", "10.74.0.0/16", false).getVpcId();+ String subnetId = service.createSubnet("us-east-1", vpcId, "10.74.0.0/28", "us-east-1a").getSubnetId();+ int volumesBefore = service.describeVolumes("us-east-1", List.of(), Map.of()).size();++ AwsException error = assertThrows(AwsException.class,+ () -> service.runInstances("us-east-1", "ami-1234567890abcdef0", "t3.micro",+ 12, 12, null, List.of(), subnetId, null, List.of(), null, null));++ assertEquals("InsufficientFreeAddressesInSubnet", error.getErrorCode());+ assertTrue(service.describeInstances("us-east-1", List.of(),+ Map.of("subnet-id", List.of(subnetId))).isEmpty());+ assertEquals(volumesBefore, service.describeVolumes("us-east-1", List.of(), Map.of()).size());+ Reservation reservation = service.runInstances("us-east-1", "ami-1234567890abcdef0", "t3.micro",+ 11, 11, null, List.of(), subnetId, null, List.of(), null, null);+ assertEquals(11, reservation.getInstances().stream().map(Instance::getPrivateIpAddress).distinct().count());+ }++ @Test+ void concurrentCreatesInASmallSubnetNeverShareASynthesisedAddress() throws Exception {+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());+ String vpcId = service.createVpc("us-east-1", "10.73.0.0/16", false).getVpcId();+ int rounds = 20;+ int usable = 11;+ ExecutorService executor = Executors.newFixedThreadPool(usable);+ try {+ for (int round = 0; round < rounds; round++) {+ String subnetId = service.createSubnet("us-east-1", vpcId, "10.73." + round + ".0/28",+ "us-east-1a").getSubnetId();+ CountDownLatch start = new CountDownLatch(1);+ List<Future<String>> creates = new ArrayList<>();+ for (int i = 0; i < usable; i++) {+ creates.add(executor.submit(() -> {+ start.await();+ return service.createNetworkInterface("us-east-1", subnetId, null,+ null, List.of(), List.of(), List.of()).getPrivateIpAddress();+ }));+ }+ start.countDown();+ Set<String> addresses = new HashSet<>();+ for (Future<String> create : creates) {+ String address = create.get(10, TimeUnit.SECONDS);+ assertTrue(addresses.add(address), "handed out twice: " + address);+ }+ assertEquals(usable, addresses.size());+ }+ } finally {+ executor.shutdownNow();+ }+ }++ @Test+ void synthesisedAddressesAreNotReusedAfterARestart() {+ Map<String, AccountAwareStorageBackend<?>> stores = Map.of(+ "ec2-vpcs.json", AccountAwareStorageBackend.<Vpc>inMemory("000000000000"),+ "ec2-subnets.json", AccountAwareStorageBackend.<Subnet>inMemory("000000000000"),+ "ec2-network-interfaces.json", AccountAwareStorageBackend.<NetworkInterface>inMemory("000000000000"));+ Ec2Service before = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory(stores));+ String vpcId = before.createVpc("us-east-1", "10.72.0.0/16", false).getVpcId();+ String subnetId = before.createSubnet("us-east-1", vpcId, "10.72.1.0/24", "us-east-1a").getSubnetId();+ String held = before.createNetworkInterface("us-east-1", subnetId, null,+ null, List.of(), List.of(), List.of()).getPrivateIpAddress();++ Ec2Service after = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory(stores));+ String next = after.createNetworkInterface("us-east-1", subnetId, null,+ null, List.of(), List.of(), List.of()).getPrivateIpAddress();++ assertEquals("10.72.1.10", held);+ assertNotEquals(held, next);+ }+private static Instance persistedInstance(String instanceId, String privateIp, InstanceState state) {Instance instance = new Instance();instance.setInstanceId(instanceId);diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.javaindex a7a156678d..53f1aa86c2 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.ecs;import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.model.ContainerNetwork;import io.github.hectorvent.floci.services.cloudmap.CloudMapService;import io.github.hectorvent.floci.services.cloudmap.model.Operation;import io.github.hectorvent.floci.services.cloudmap.model.Service;@@ -103,9 +104,15 @@ class EcsServiceDiscoveryDockerIntegrationTest {assertEquals("RUNNING", task.getLastStatus(), "the container must be up: "+ task.getStoppedReason());assertNotNull(task.getPrivateIpAddress(), "an awsvpc task must carry its ENI address");+ List<String> containerAddresses = containerAddresses(task);- assertEquals(List.of(task.getPrivateIpAddress()), cloudMapService.resolveDnsName(dnsName),+ List<String> resolved = cloudMapService.resolveDnsName(dnsName);+ assertEquals(1, resolved.size(),"starting the task must register it, so the Cloud Map name resolves to it");+ assertTrue(containerAddresses.contains(resolved.getFirst()),+ "the name must resolve to an address the container holds, " + containerAddresses+ + ", not to the ENI address " + task.getPrivateIpAddress()+ + " that no peer can connect to");// Updating the registry moves the running task to the replacement. Scaling to zero// keeps the reconciler from replacing it between the stop and the final assertions.@@ -120,7 +127,7 @@ class EcsServiceDiscoveryDockerIntegrationTest {ecsService.updateService(update, REGION);assertTrue(cloudMapService.listInstances(cloudMapSvc.getId()).isEmpty(),"updating the registry must remove the old Cloud Map instance");- assertEquals(List.of(task.getPrivateIpAddress()),+ assertEquals(resolved,cloudMapService.resolveDnsName("replacement." + namespaceName),"updating the registry must register the running task in the new service");ecsService.stopTask(clusterName, taskArn, "service discovery test", REGION);@@ -140,6 +147,16 @@ class EcsServiceDiscoveryDockerIntegrationTest {}}+ /** Every address the task container holds on a Docker network, i.e. where a peer reaches it. */+ private List<String> containerAddresses(EcsTask task) {+ String dockerId = task.getContainers().getFirst().getDockerId();+ return dockerClient.inspectContainerCmd(dockerId).exec()+ .getNetworkSettings().getNetworks().values().stream()+ .map(ContainerNetwork::getIpAddress)+ .filter(address -> address != null && !address.isBlank())+ .toList();+ }+private CreateServiceRequest createServiceRequest(String clusterName, TaskDefinition taskDef,Service cloudMapSvc) {AwsVpcConfiguration awsvpc = new AwsVpcConfiguration();diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrarAddressTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrarAddressTest.javanew file mode 100644index 0000000000..28a07b1b23--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrarAddressTest.java@@ -0,0 +1,144 @@+package io.github.hectorvent.floci.services.ecs;++import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.cloudmap.CloudMapService;+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;+import io.github.hectorvent.floci.services.ecs.model.Container;+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;+import io.github.hectorvent.floci.services.ecs.model.EcsTask;+import io.github.hectorvent.floci.services.ecs.model.NetworkBinding;+import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;++import java.util.List;+import java.util.Map;+import java.util.Optional;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.ArgumentMatchers.isNull;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * Which address the registrar puts into Cloud Map. A task ENI address is not on any network the+ * task's containers joined, so registering it gives a peer a name that resolves and a connection+ * that times out.+ */+class EcsServiceDiscoveryRegistrarAddressTest {++ private static final String REGION = "us-east-1";+ private static final String REGISTRY_ARN =+ "arn:aws:servicediscovery:us-east-1:000000000000:service/srv-abc123";++ private final CloudMapService cloudMapService = mock(CloudMapService.class);+ private final EcsContainerManager containerManager = mock(EcsContainerManager.class);+ private final EcsServiceDiscoveryRegistrar registrar =+ new EcsServiceDiscoveryRegistrar(cloudMapService, containerManager, storageFactory());++ @Test+ void awsvpcTaskRegistersTheAddressItsContainerHolds() {+ EcsTask task = task("10.70.1.10");+ when(containerManager.resolvePeerAddress(any(Container.class))).thenReturn(Optional.of("172.31.0.19"));++ registrar.registerTask(task, service(), REGION);++ assertEquals("172.31.0.19", registeredAddress());+ verify(containerManager, never()).resolveContainerHost(any(Container.class));+ }++ @Test+ void awsvpcTaskKeepsItsEniAddressWhenNoContainerAnswers() {+ EcsTask task = task("10.70.1.10");+ when(containerManager.resolvePeerAddress(any(Container.class))).thenReturn(Optional.empty());++ registrar.registerTask(task, service(), REGION);++ assertEquals("10.70.1.10", registeredAddress());+ }++ @Test+ void bridgeTaskStillRegistersTheContainerHost() {+ EcsTask task = task(null);+ when(containerManager.resolveContainerHost(any(Container.class))).thenReturn("127.0.0.1");++ registrar.registerTask(task, service(), REGION);++ assertEquals("127.0.0.1", registeredAddress());+ verify(containerManager, never()).resolvePeerAddress(any(Container.class));+ }++ @Test+ void awsvpcTaskAdvertisesTheContainerPortItsAddressAnswersOn() {+ EcsTask task = task("10.70.1.10");+ task.getContainers().getFirst().setNetworkBindings(+ List.of(new NetworkBinding("0.0.0.0", 8080, 49153, "tcp")));+ when(containerManager.resolvePeerAddress(any(Container.class))).thenReturn(Optional.of("172.31.0.19"));++ registrar.registerTask(task, serviceWithContainerPort(8080), REGION);++ Map<String, String> attributes = registeredAttributes();+ assertEquals("172.31.0.19", attributes.get("AWS_INSTANCE_IPV4"));+ assertEquals("8080", attributes.get("AWS_INSTANCE_PORT"));+ }++ @Test+ void bridgeTaskAdvertisesThePublishedHostPort() {+ EcsTask task = task(null);+ task.getContainers().getFirst().setNetworkBindings(+ List.of(new NetworkBinding("0.0.0.0", 8080, 49153, "tcp")));+ when(containerManager.resolveContainerHost(any(Container.class))).thenReturn("127.0.0.1");++ registrar.registerTask(task, serviceWithContainerPort(8080), REGION);++ assertEquals("49153", registeredAttributes().get("AWS_INSTANCE_PORT"));+ }++ private String registeredAddress() {+ return registeredAttributes().get("AWS_INSTANCE_IPV4");+ }++ @SuppressWarnings("unchecked")+ private Map<String, String> registeredAttributes() {+ ArgumentCaptor<Map<String, String>> attributes = ArgumentCaptor.forClass(Map.class);+ verify(cloudMapService).registerInstance(eq("srv-abc123"), anyString(), isNull(),+ attributes.capture(), eq(REGION));+ return attributes.getValue();+ }++ private static EcsServiceModel service() {+ EcsServiceModel svc = new EcsServiceModel();+ svc.setServiceRegistries(List.of(Map.of("registryArn", REGISTRY_ARN, "containerName", "app")));+ return svc;+ }++ private static EcsServiceModel serviceWithContainerPort(int containerPort) {+ EcsServiceModel svc = new EcsServiceModel();+ svc.setServiceRegistries(List.of(Map.of("registryArn", REGISTRY_ARN, "containerName", "app",+ "containerPort", containerPort)));+ return svc;+ }++ private static EcsTask task(String eniAddress) {+ Container container = new Container();+ container.setName("app");+ container.setDockerId("docker-app");+ EcsTask task = new EcsTask();+ task.setTaskArn("arn:aws:ecs:" + REGION + ":000000000000:task/c/0123456789abcdef");+ task.setPrivateIpAddress(eniAddress);+ task.setContainers(List.of(container));+ return task;+ }++ private static StorageFactory storageFactory() {+ StorageFactory storageFactory = mock(StorageFactory.class);+ when(storageFactory.create(anyString(), anyString(), any()))+ .thenAnswer(invocation -> AccountAwareStorageBackend.inMemory("000000000000"));+ return storageFactory;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.javaindex cde3674653..7de7cadfe3 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java@@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.Optional;import java.util.Set;import static org.hamcrest.MatcherAssert.assertThat;@@ -167,7 +168,9 @@ class EcsServiceRestartLeftoversTest {task.setPrivateIpAddress("10.0.1.5");return new EcsTaskHandle(task.getTaskArn(), Map.of("web", "docker-id"), Map.of());});- when(containerManager.resolveContainerHost(any())).thenReturn("172.19.0.4");+ // Floci running natively: targets reach the task on loopback, Cloud Map registers its Docker address.+ when(containerManager.resolveContainerHost(any())).thenReturn("127.0.0.1");+ when(containerManager.resolvePeerAddress(any())).thenReturn(Optional.of("172.19.0.4"));when(containerManager.taskVpcId(any(), anyString())).thenReturn("vpc-app");EcsLoadBalancerRegistrar lbRegistrar = mock(EcsLoadBalancerRegistrar.class);EcsServiceDiscoveryRegistrar discoveryRegistrar = mock(EcsServiceDiscoveryRegistrar.class);@@ -194,7 +197,7 @@ class EcsServiceRestartLeftoversTest {assertEquals("RUNNING", task.getLastStatus());// A target only on the ports the container serves, and never at the VPC-scoped ENI address.verify(lbRegistrar).evictUnrecordedTargets(task, Set.of(TARGET_GROUP_ARN),- Set.of(new EcsTaskAddress("172.19.0.4", 80)), REGION);+ Set.of(new EcsTaskAddress("127.0.0.1", 80)), REGION);verify(discoveryRegistrar).evictUnrecordedInstances(task, Set.of("srv-alpha"),Set.of(new EcsTaskAddress("10.0.1.5", null, "vpc-app"), new EcsTaskAddress("172.19.0.4", null)),REGION);