Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): register reachable task addresses in Cloud Map and bound synthesised ENI addresses (#4509)

Abhigya014156a546da
diff --git a/docs/services/ec2.md b/docs/services/ec2.md
index 222e4e6399..b8c75ee84c 100644
--- a/docs/services/ec2.md
+++ b/docs/services/ec2.md
@@ -862,6 +862,11 @@ Limits worth knowing before reading a passing test as evidence:
Set `FLOCI_SERVICES_EC2_VPC_NETWORKS_ENABLED=false` to go back to synthesised private addresses
and the shared default bridge. It is also off whenever `mock` is on.
+Synthesised addresses come from the subnet's own CIDR block, skipping the first four addresses
+and the last one as AWS reserves them, and skipping every address a persisted instance, network
+interface or NAT gateway in the subnet still holds, so a restart does not hand one out twice. A
+subnet with no free address left fails the request with `InsufficientFreeAddressesInSubnet`.
+
## Requirements
EC2 requires the Docker socket to be accessible (same as Lambda, ECS, and other container services):
diff --git a/docs/services/ecs.md b/docs/services/ecs.md
index 92fa31d403..c1c290f5c7 100644
--- a/docs/services/ecs.md
+++ b/docs/services/ecs.md
@@ -469,10 +469,15 @@ Known differences from AWS:
A service that declares `serviceRegistries` has each of its running tasks registered as a
Cloud Map instance of the named service, and deregistered when the task stops. The instance
-carries `AWS_INSTANCE_IPV4` (the task's ENI address on an awsvpc task, otherwise the
-container's address on the Docker network) and `AWS_INSTANCE_PORT` (the entry's `port`, or
-the host port its `containerPort` was published on), alongside the metadata attributes AWS
-records: `AVAILABILITY_ZONE`, `REGION`, `ECS_SERVICE_NAME`, `ECS_CLUSTER_NAME` and
+carries `AWS_INSTANCE_IPV4` and `AWS_INSTANCE_PORT`, chosen by network mode:
+
+| Network mode | `AWS_INSTANCE_IPV4` | `AWS_INSTANCE_PORT` |
+|---|---|---|
+| `awsvpc` | the address the task's container holds on its Docker network | the entry's `containerPort` |
+| `bridge` or `host` | the address Floci reaches the container at: `127.0.0.1` when Floci runs natively, the container's Docker network address when Floci runs in Docker | the host port the entry's `containerPort` was published on, or the `containerPort` when it has no binding |
+
+An explicit `port` on the registry entry is registered as-is in every mode, as on AWS. The
+instance also carries the metadata attributes AWS records: `AVAILABILITY_ZONE`, `REGION`, `ECS_SERVICE_NAME`, `ECS_CLUSTER_NAME` and
`ECS_TASK_DEFINITION_FAMILY`. The task id is the instance id, as on AWS, so a replacement
task supersedes its predecessor.
@@ -488,6 +493,12 @@ Known differences from AWS:
reachable port is the published host port rather than the container port. Floci serves
only A records, so such a task registers its address and its published host port and
resolves by address; the port is readable through `DiscoverInstances` rather than DNS.
+- An awsvpc task registers its container's Docker network address, not its ENI address. On
+ AWS the two are the same interface. In Floci a task's containers never join the subnet's
+ network, so the ENI address `DescribeTasks` reports is not one a peer can connect to. The
+ registrar falls back to the ENI address only when it cannot read an address from a running
+ container. Because the address is the container's own, the port is its `containerPort`, not
+ any host port Floci also published it on.
- `EC2_INSTANCE_ID` is not recorded. Every Floci task runs as a container rather than on a
registered EC2 host, which is the Fargate case on AWS, where the attribute is also absent.
- An instance's health status never moves. AWS registers a task `UNHEALTHY` and promotes it to
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
index 17c84cdc03..8b4dedd4b9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
@@ -84,6 +84,7 @@ import io.github.hectorvent.floci.services.ec2.model.VpcIpv6CidrBlockAssociation
import io.github.hectorvent.floci.services.ec2.model.VpcPeeringConnection;
import io.github.hectorvent.floci.services.ec2.model.VpcPeeringConnectionStateReason;
import io.github.hectorvent.floci.services.ec2.model.VpcPeeringConnectionVpcInfo;
+import io.github.hectorvent.floci.services.ec2.net.Cidr4;
import io.github.hectorvent.floci.services.ec2.net.VpcNetworkManager;
import io.github.hectorvent.floci.services.ec2.portforward.Ec2PortForwardManager;
import io.github.hectorvent.floci.services.iam.IamService;
@@ -101,8 +102,10 @@ import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Base64;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@@ -116,7 +119,6 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiPredicate;
import java.util.function.Function;
import java.util.regex.Pattern;
@@ -137,6 +139,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
private static final DateTimeFormatter ISO_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
.withZone(ZoneOffset.UTC);
private static final int DEFAULT_ROOT_VOLUME_SIZE_GIB = 8;
+ private static final long SYNTHETIC_FIRST_OFFSET = 10;
+ private static final String SYNTHETIC_FALLBACK_CIDR = "172.31.0.0/24";
private static final String DEFAULT_ROOT_VOLUME_TYPE = "gp3";
private static final Set<String> VALID_VOLUME_TYPES =
Set.of("standard", "io1", "io2", "gp2", "sc1", "st1", "gp3");
@@ -239,8 +243,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
private final StorageBackend<String, List<Tag>> tags;
private final StorageBackend<String, CapacityReservation> capacityReservations;
private final Set<String> seededAccountRegions = ConcurrentHashMap.newKeySet();
- // subnetId → counter for IP assignment (runtime-only, not persisted)
- private final Map<String, AtomicInteger> subnetIpCounters = new ConcurrentHashMap<>();
+ // region::subnetId → offset the next synthesised address is tried at. Only an ordering hint:
+ // restart forgets it, and privateIpsInUse is what keeps addresses from being handed out twice.
+ private final Map<String, Long> subnetIpCursors = new HashMap<>();
+ // Held from a synthesised address being chosen until the resource holding it is persisted, so
+ // a concurrent allocation, which reads persisted resources to see what is taken, cannot pick
+ // the same address in between. Taken after imageRegistryLock where a caller holds both.
+ private final Object privateIpAllocationLock = new Object();
/**
* Null in the hermetic unit tests, which reach the constructors that do not take it; CDI always
@@ -2968,111 +2977,114 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
if (dryRun) {
throw new AwsException("DryRunOperation", "Request would have succeeded, but DryRun flag is set.", 412);
}
- for (int i = 0; i < count; i++) {
- String instanceId = "i-" + randomHex(17);
- String privateIp = suppliedEni != null
- ? suppliedEni.getPrivateIpAddress()
- : assignPrivateIp(region, finalSubnetId);
-
- Instance inst = new Instance();
- inst.setInstanceId(instanceId);
- inst.setImageId(imageId);
- inst.setState(InstanceState.pending());
- inst.setInstanceType(effectiveInstanceType);
- inst.setPlacement(new Placement(az));
- inst.setSubnetId(finalSubnetId);
- inst.setVpcId(vpcId);
- // AWS precedence (#1984): the launch-time AssociatePublicIpAddress
- // override wins in both directions; the subnet's MapPublicIpOnLaunch
- // attribute is only the default when the launch does not specify it.
- inst.setAssociatePublicIp(associatePublicIp != null
- ? associatePublicIp
- : subnet != null && subnet.isMapPublicIpOnLaunch());
- inst.setPrivateIpAddress(privateIp);
- inst.setPrivateDnsName("ip-" + privateIp.replace('.', '-') + ".ec2.internal");
- inst.setKeyName(keyName);
- inst.setSecurityGroups(new ArrayList<>(sgIdentifiers));
- inst.setArchitecture(architecture);
- inst.setLaunchTime(Instant.now());
- inst.setAmiLaunchIndex(i);
- inst.setClientToken(clientToken);
- inst.setRegion(region);
- inst.setUserData(userData);
- inst.setEncodedUserData(encodedUserData);
- inst.setIamInstanceProfileArn(iamInstanceProfileArn);
- if (iamInstanceProfileArn != null) {
- inst.setIamInstanceProfileAssociationTime(inst.getLaunchTime());
- }
- inst.setMetadataOptions(LaunchTemplateData.MetadataOptions.merge(launchMetadataOptions, null));
- inst.setCreditSpecificationCpuCredits(
- acquiredCpuCredits(effectiveInstanceType, creditSpecificationCpuCredits));
- if (instanceTags != null && !instanceTags.isEmpty()) {
- inst.setTags(new ArrayList<>(instanceTags));
- tags.put(instanceId, new ArrayList<>(instanceTags));
- }
-
- // Network interface, either the caller-supplied standalone ENI (override-default-eni,
- // floci-kt9) or a freshly-minted implicit primary interface.
- InstanceNetworkInterface eni = new InstanceNetworkInterface();
- eni.setNetworkInterfaceId(suppliedEni != null ? suppliedEni.getNetworkInterfaceId() : "eni-" + randomHex(17));
- eni.setSubnetId(finalSubnetId);
- eni.setVpcId(vpcId);
- eni.setOwnerId(callerAccountId());
- eni.setDescription(suppliedEni != null ? suppliedEni.getDescription() : null);
- eni.setMacAddress(suppliedEni != null ? suppliedEni.getMacAddress() : null);
- eni.setPrivateIpAddress(privateIp);
- eni.setPrivateDnsName(inst.getPrivateDnsName());
- eni.setGroups(new ArrayList<>(sgIdentifiers));
- eni.setAttachmentId("eni-attach-" + randomHex(17));
- eni.setDeviceIndex(suppliedEni != null ? networkInterfaceDeviceIndex : 0);
- if (inst.getLaunchTime() != null) {
- eni.setAttachTime(ISO_FMT.format(inst.getLaunchTime()));
- }
- inst.getNetworkInterfaces().add(eni);
- if (suppliedEni != null) {
- // The standalone record stays authoritative rather than being folded into the
- // instance: AWS defaults deleteOnTermination to false for an interface the caller
- // created and handed to a launch, so it outlives the instance and returns to
- // "available" on termination instead of vanishing with it. Double-counting is
- // avoided in describeNetworkInterfaces, which skips the instance-side copy of any
- // id the standalone store owns.
- NetworkInterfaceAttachment launchAttachment = new NetworkInterfaceAttachment();
- launchAttachment.setAttachmentId(eni.getAttachmentId());
- launchAttachment.setDeviceIndex(eni.getDeviceIndex());
- launchAttachment.setStatus("attached");
- launchAttachment.setInstanceId(instanceId);
- launchAttachment.setInstanceOwnerId(callerAccountId());
- launchAttachment.setAttachTime(eni.getAttachTime());
- launchAttachment.setDeleteOnTermination(false);
- suppliedEni.setAttachment(launchAttachment);
- suppliedEni.setStatus("in-use");
- networkInterfaces.put(key(region, suppliedEni.getNetworkInterfaceId()), suppliedEni);
- }
+ synchronized (privateIpAllocationLock) {
+ List<String> privateIps = allocateLaunchAddresses(region, finalSubnetId, suppliedEni, count);
+ for (int i = 0; i < count; i++) {
+ String instanceId = "i-" + randomHex(17);
+ String privateIp = privateIps.get(i);
+
+ Instance inst = new Instance();
+ inst.setInstanceId(instanceId);
+ inst.setImageId(imageId);
+ inst.setState(InstanceState.pending());
+ inst.setInstanceType(effectiveInstanceType);
+ inst.setPlacement(new Placement(az));
+ inst.setSubnetId(finalSubnetId);
+ inst.setVpcId(vpcId);
+ // AWS precedence (#1984): the launch-time AssociatePublicIpAddress
+ // override wins in both directions; the subnet's MapPublicIpOnLaunch
+ // attribute is only the default when the launch does not specify it.
+ inst.setAssociatePublicIp(associatePublicIp != null
+ ? associatePublicIp
+ : subnet != null && subnet.isMapPublicIpOnLaunch());
+ inst.setPrivateIpAddress(privateIp);
+ inst.setPrivateDnsName("ip-" + privateIp.replace('.', '-') + ".ec2.internal");
+ inst.setKeyName(keyName);
+ inst.setSecurityGroups(new ArrayList<>(sgIdentifiers));
+ inst.setArchitecture(architecture);
+ inst.setLaunchTime(Instant.now());
+ inst.setAmiLaunchIndex(i);
+ inst.setClientToken(clientToken);
+ inst.setRegion(region);
+ inst.setUserData(userData);
+ inst.setEncodedUserData(encodedUserData);
+ inst.setIamInstanceProfileArn(iamInstanceProfileArn);
+ if (iamInstanceProfileArn != null) {
+ inst.setIamInstanceProfileAssociationTime(inst.getLaunchTime());
+ }
+ inst.setMetadataOptions(LaunchTemplateData.MetadataOptions.merge(launchMetadataOptions, null));
+ inst.setCreditSpecificationCpuCredits(
+ acquiredCpuCredits(effectiveInstanceType, creditSpecificationCpuCredits));
+ if (instanceTags != null && !instanceTags.isEmpty()) {
+ inst.setTags(new ArrayList<>(instanceTags));
+ tags.put(instanceId, new ArrayList<>(instanceTags));
+ }
- // Root EBS volume
- String rootVolId = "vol-" + randomHex(17);
- inst.setRootVolumeId(rootVolId);
- Volume rootVol = new Volume();
- rootVol.setVolumeId(rootVolId);
- rootVol.setAvailabilityZone(az);
- rootVol.setVolumeType(DEFAULT_ROOT_VOLUME_TYPE);
- rootVol.setSize(DEFAULT_ROOT_VOLUME_SIZE_GIB);
- rootVol.setState("in-use");
- rootVol.setRegion(region);
- rootVol.setCreateTime(Instant.now());
- VolumeAttachment att = new VolumeAttachment();
- att.setVolumeId(rootVolId);
- att.setInstanceId(instanceId);
- att.setDevice(inst.getRootDeviceName());
- att.setState("attached");
- att.setDeleteOnTermination(true);
- att.setAttachTime(Instant.now());
- rootVol.getAttachments().add(att);
- volumes.put(key(region, rootVolId), rootVol);
+ // Network interface, either the caller-supplied standalone ENI (override-default-eni,
+ // floci-kt9) or a freshly-minted implicit primary interface.
+ InstanceNetworkInterface eni = new InstanceNetworkInterface();
+ eni.setNetworkInterfaceId(suppliedEni != null
+ ? suppliedEni.getNetworkInterfaceId()
+ : "eni-" + randomHex(17));
+ eni.setSubnetId(finalSubnetId);
+ eni.setVpcId(vpcId);
+ eni.setOwnerId(callerAccountId());
+ eni.setDescription(suppliedEni != null ? suppliedEni.getDescription() : null);
+ eni.setMacAddress(suppliedEni != null ? suppliedEni.getMacAddress() : null);
+ eni.setPrivateIpAddress(privateIp);
+ eni.setPrivateDnsName(inst.getPrivateDnsName());
+ eni.setGroups(new ArrayList<>(sgIdentifiers));
+ eni.setAttachmentId("eni-attach-" + randomHex(17));
+ eni.setDeviceIndex(suppliedEni != null ? networkInterfaceDeviceIndex : 0);
+ if (inst.getLaunchTime() != null) {
+ eni.setAttachTime(ISO_FMT.format(inst.getLaunchTime()));
+ }
+ inst.getNetworkInterfaces().add(eni);
+ if (suppliedEni != null) {
+ // The standalone record stays authoritative rather than being folded into the
+ // instance: AWS defaults deleteOnTermination to false for an interface the caller
+ // created and handed to a launch, so it outlives the instance and returns to
+ // "available" on termination instead of vanishing with it. Double-counting is
+ // avoided in describeNetworkInterfaces, which skips the instance-side copy of any
+ // id the standalone store owns.
+ NetworkInterfaceAttachment launchAttachment = new NetworkInterfaceAttachment();
+ launchAttachment.setAttachmentId(eni.getAttachmentId());
+ launchAttachment.setDeviceIndex(eni.getDeviceIndex());
+ launchAttachment.setStatus("attached");
+ launchAttachment.setInstanceId(instanceId);
+ launchAttachment.setInstanceOwnerId(callerAccountId());
+ launchAttachment.setAttachTime(eni.getAttachTime());
+ launchAttachment.setDeleteOnTermination(false);
+ suppliedEni.setAttachment(launchAttachment);
+ suppliedEni.setStatus("in-use");
+ networkInterfaces.put(key(region, suppliedEni.getNetworkInterfaceId()), suppliedEni);
+ }
- instances.put(key(region, instanceId), inst);
- launched.add(inst);
- reservation.getInstances().add(inst);
+ // Root EBS volume
+ String rootVolId = "vol-" + randomHex(17);
+ inst.setRootVolumeId(rootVolId);
+ Volume rootVol = new Volume();
+ rootVol.setVolumeId(rootVolId);
+ rootVol.setAvailabilityZone(az);
+ rootVol.setVolumeType(DEFAULT_ROOT_VOLUME_TYPE);
+ rootVol.setSize(DEFAULT_ROOT_VOLUME_SIZE_GIB);
+ rootVol.setState("in-use");
+ rootVol.setRegion(region);
+ rootVol.setCreateTime(Instant.now());
+ VolumeAttachment att = new VolumeAttachment();
+ att.setVolumeId(rootVolId);
+ att.setInstanceId(instanceId);
+ att.setDevice(inst.getRootDeviceName());
+ att.setState("attached");
+ att.setDeleteOnTermination(true);
+ att.setAttachTime(Instant.now());
+ rootVol.getAttachments().add(att);
+ volumes.put(key(region, rootVolId), rootVol);
+
+ instances.put(key(region, instanceId), inst);
+ launched.add(inst);
+ reservation.getInstances().add(inst);
+ }
}
}
@@ -3302,7 +3314,44 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
"The availability zone '" + availabilityZone + "' has no subnet in this region.", 400));
}
+ /**
+ * Every private address a launch needs, allocated before any of its instances is stored, so
+ * a subnet without room for the whole launch fails it without leaving the instances that did
+ * fit behind. Callers must hold {@link #privateIpAllocationLock}.
+ */
+ private List<String> allocateLaunchAddresses(String region, String subnetId, NetworkInterface suppliedEni,
+ int count) {
+ List<String> addresses = new ArrayList<>(count);
+ if (suppliedEni != null) {
+ for (int i = 0; i < count; i++) {
+ addresses.add(suppliedEni.getPrivateIpAddress());
+ }
+ return addresses;
+ }
+ try {
+ for (int i = 0; i < count; i++) {
+ addresses.add(assignPrivateIp(region, subnetId, addresses));
+ }
+ } catch (AwsException e) {
+ if (vpcNetworkManager != null) {
+ for (String address : addresses) {
+ vpcNetworkManager.releasePrivateIp(region, subnetId, address);
+ }
+ }
+ throw e;
+ }
+ return addresses;
+ }
+
private String assignPrivateIp(String region, String subnetId) {
+ return assignPrivateIp(region, subnetId, List.of());
+ }
+
+ /**
+ * @param pending addresses already handed out to resources that are not stored yet, which the
+ * synthesised fallback must not hand out again
+ */
+ private String assignPrivateIp(String region, String subnetId, Collection<String> pending) {
// A Docker-backed subnet allocates the real thing: an address on the network the
// instance's container will actually hold. Only when there is no such network does
// this fall back to the synthesised address below, which nothing can connect to.
@@ -3315,17 +3364,88 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
if (subnetId == null) {
return "172.31.0." + (10 + new Random().nextInt(200));
}
- AtomicInteger counter = subnetIpCounters.computeIfAbsent(region + "::" + subnetId, k -> new AtomicInteger(10));
- int offset = counter.getAndIncrement();
- Subnet subnet = subnets.get(key(region, subnetId)).orElse(null);
- if (subnet == null) {
- return "172.31.0." + offset;
+ Cidr4 cidr = subnets.get(key(region, subnetId))
+ .flatMap(subnet -> Cidr4.parse(subnet.getCidrBlock()))
+ .or(() -> Cidr4.parse(SYNTHETIC_FALLBACK_CIDR))
+ .orElseThrow();
+ // AWS reserves the first four addresses of a subnet and its last one.
+ long first = 4;
+ long last = cidr.size() - 2;
+ if (last < first) {
+ throw insufficientFreeAddresses(subnetId);
+ }
+ String cursorKey = key(region, subnetId);
+ synchronized (privateIpAllocationLock) {
+ // The addresses persisted resources already hold, so a restart, which forgets the
+ // cursor, never hands out an address that is still in use.
+ Set<String> inUse = privateIpsInUse(region, subnetId);
+ inUse.addAll(pending);
+ long start = Math.clamp(subnetIpCursors.getOrDefault(cursorKey, SYNTHETIC_FIRST_OFFSET), first, last);
+ long span = last - first + 1;
+ for (long step = 0; step < span; step++) {
+ long offset = first + (start - first + step) % span;
+ String address = cidr.addressAt(offset).orElseThrow();
+ if (!inUse.contains(address)) {
+ subnetIpCursors.put(cursorKey, offset + 1);
+ return address;
+ }
+ }
+ }
+ throw insufficientFreeAddresses(subnetId);
+ }
+
+ private static AwsException insufficientFreeAddresses(String subnetId) {
+ return new AwsException("InsufficientFreeAddressesInSubnet",
+ "There are not enough free addresses in subnet '" + subnetId + "' to satisfy the requested number of instances.",
+ 400);
+ }
+
+ /** Every private address a live instance, network interface or NAT gateway holds in the subnet. */
+ private Set<String> privateIpsInUse(String region, String subnetId) {
+ String regionPrefix = region + "::";
+ Set<String> inUse = new HashSet<>();
+ for (NetworkInterface ni : networkInterfaces.scan(k -> k.startsWith(regionPrefix))) {
+ if (!subnetId.equals(ni.getSubnetId())) {
+ continue;
+ }
+ addIfSet(inUse, ni.getPrivateIpAddress());
+ for (NetworkInterfacePrivateIpAddress ip : nullToEmpty(ni.getPrivateIpAddresses())) {
+ addIfSet(inUse, ip.getPrivateIpAddress());
+ }
+ }
+ for (Instance instance : instances.scan(k -> k.startsWith(regionPrefix))) {
+ String state = instance.getState() == null ? null : instance.getState().getName();
+ if ("terminated".equals(state)) {
+ continue;
+ }
+ if (subnetId.equals(instance.getSubnetId())) {
+ addIfSet(inUse, instance.getPrivateIpAddress());
+ }
+ for (InstanceNetworkInterface ni : nullToEmpty(instance.getNetworkInterfaces())) {
+ if (subnetId.equals(ni.getSubnetId())) {
+ addIfSet(inUse, ni.getPrivateIpAddress());
+ }
+ }
+ }
+ for (NatGateway natGateway : natGateways.scan(k -> k.startsWith(regionPrefix))) {
+ if (!subnetId.equals(natGateway.getSubnetId()) || "deleted".equals(natGateway.getState())) {
+ continue;
+ }
+ for (NatGatewayAddress address : nullToEmpty(natGateway.getNatGatewayAddresses())) {
+ addIfSet(inUse, address.getPrivateIp());
+ }
+ }
+ return inUse;
+ }
+
+ private static <T> List<T> nullToEmpty(List<T> list) {
+ return list == null ? List.of() : list;
+ }
+
+ private static void addIfSet(Set<String> addresses, String address) {
+ if (address != null && !address.isBlank()) {
+ addresses.add(address);
}
- // Parse base IP from CIDR
- String cidr = subnet.getCidrBlock();
- String baseIp = cidr.split("/")[0];
- String[] parts = baseIp.split("\\.");
- return parts[0] + "." + parts[1] + "." + parts[2] + "." + offset;
}
public List<Reservation> describeInstances(String region, List<String> instanceIds, Map<String, List<String>> filters) {
@@ -7973,12 +8093,14 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
natGateway.setConnectivityType(connectivityType != null && !connectivityType.isBlank() ? connectivityType : "public");
natGateway.setCreateTime(Instant.now());
natGateway.setRegion(region);
- natGateway.getNatGatewayAddresses().add(natGatewayAddress(region, subnetId, allocationId));
- if (natGatewayTags != null && !natGatewayTags.isEmpty()) {
- natGateway.setTags(new ArrayList<>(natGatewayTags));
- tags.put(natGateway.getNatGatewayId(), new ArrayList<>(natGatewayTags));
+ synchronized (privateIpAllocationLock) {
+ natGateway.getNatGatewayAddresses().add(natGatewayAddress(region, subnetId, allocationId));
+ if (natGatewayTags != null && !natGatewayTags.isEmpty()) {
+ natGateway.setTags(new ArrayList<>(natGatewayTags));
+ tags.put(natGateway.getNatGatewayId(), new ArrayList<>(natGatewayTags));
+ }
+ natGateways.put(key(region, natGateway.getNatGatewayId()), natGateway);
}
- natGateways.put(key(region, natGateway.getNatGatewayId()), natGateway);
return natGateway;
}
@@ -9199,49 +9321,51 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
}
}
- String eniId = "eni-" + randomHex(17);
- String primaryIp = (privateIpAddress != null && !privateIpAddress.isBlank())
- ? privateIpAddress : assignPrivateIp(region, subnetId);
- String primaryDns = "ip-" + primaryIp.replace('.', '-') + ".ec2.internal";
+ synchronized (privateIpAllocationLock) {
+ String eniId = "eni-" + randomHex(17);
+ String primaryIp = (privateIpAddress != null && !privateIpAddress.isBlank())
+ ? privateIpAddress : assignPrivateIp(region, subnetId);
+ String primaryDns = "ip-" + primaryIp.replace('.', '-') + ".ec2.internal";
- NetworkInterface ni = new NetworkInterface();
- ni.setNetworkInterfaceId(eniId);
- ni.setSubnetId(subnetId);
- ni.setVpcId(subnet.getVpcId());
- ni.setAvailabilityZone(subnet.getAvailabilityZone());
- ni.setDescription(description);
- ni.setOwnerId(callerAccountId());
- ni.setStatus("available");
- ni.setMacAddress(randomMac());
- ni.setPrivateIpAddress(primaryIp);
- ni.setPrivateDnsName(primaryDns);
- ni.setGroups(sgIdentifiers);
- if (tagList != null) {
- ni.getTagSet().addAll(tagList);
- }
-
- List<NetworkInterfacePrivateIpAddress> ipList = new ArrayList<>();
- NetworkInterfacePrivateIpAddress primary = new NetworkInterfacePrivateIpAddress();
- primary.setPrivateIpAddress(primaryIp);
- primary.setPrivateDnsName(primaryDns);
- primary.setPrimary(true);
- ipList.add(primary);
- if (privateIpAddresses != null) {
- for (String extra : privateIpAddresses) {
- if (extra == null || extra.isBlank() || extra.equals(primaryIp)) {
- continue;
+ NetworkInterface ni = new NetworkInterface();
+ ni.setNetworkInterfaceId(eniId);
+ ni.setSubnetId(subnetId);
+ ni.setVpcId(subnet.getVpcId());
+ ni.setAvailabilityZone(subnet.getAvailabilityZone());
+ ni.setDescription(description);
+ ni.setOwnerId(callerAccountId());
+ ni.setStatus("available");
+ ni.setMacAddress(randomMac());
+ ni.setPrivateIpAddress(primaryIp);
+ ni.setPrivateDnsName(primaryDns);
+ ni.setGroups(sgIdentifiers);
+ if (tagList != null) {
+ ni.getTagSet().addAll(tagList);
+ }
+
+ List<NetworkInterfacePrivateIpAddress> ipList = new ArrayList<>();
+ NetworkInterfacePrivateIpAddress primary = new NetworkInterfacePrivateIpAddress();
+ primary.setPrivateIpAddress(primaryIp);
+ primary.setPrivateDnsName(primaryDns);
+ primary.setPrimary(true);
+ ipList.add(primary);
+ if (privateIpAddresses != null) {
+ for (String extra : privateIpAddresses) {
+ if (extra == null || extra.isBlank() || extra.equals(primaryIp)) {
+ continue;
+ }
+ NetworkInterfacePrivateIpAddress secondary = new NetworkInterfacePrivateIpAddress();
+ secondary.setPrivateIpAddress(extra);
+ secondary.setPrivateDnsName("ip-" + extra.replace('.', '-') + ".ec2.internal");
+ secondary.setPrimary(false);
+ ipList.add(secondary);
}
- NetworkInterfacePrivateIpAddress secondary = new NetworkInterfacePrivateIpAddress();
- secondary.setPrivateIpAddress(extra);
- secondary.setPrivateDnsName("ip-" + extra.replace('.', '-') + ".ec2.internal");
- secondary.setPrimary(false);
- ipList.add(secondary);
}
- }
- ni.setPrivateIpAddresses(ipList);
+ ni.setPrivateIpAddresses(ipList);
- networkInterfaces.put(key(region, eniId), ni);
- return ni;
+ networkInterfaces.put(key(region, eniId), ni);
+ return ni;
+ }
}
/**
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index 3610a2ac41..d35205d55b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -1892,12 +1892,15 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
/**
* The addresses at which a Cloud Map instance resolves to a running task: its ENI address,
* within the task's VPC only, and each container's address on Docker, on any port since a
- * DNS answer carries none. A loopback address counts only on the task's host ports.
+ * DNS answer carries none. A loopback address counts only on the task's host ports. An
+ * awsvpc task's containers count at the address each holds on its Docker network, which is
+ * where {@link EcsServiceDiscoveryRegistrar} registers them, even when Floci runs natively.
*/
private Set<EcsTaskAddress> instanceAddresses(EcsTask task, String region) {
Set<EcsTaskAddress> addresses = new LinkedHashSet<>();
String eniAddress = task.getPrivateIpAddress();
- if (eniAddress != null && !eniAddress.isBlank() && !EcsTaskAddress.isLoopback(eniAddress)) {
+ boolean awsvpc = eniAddress != null && !eniAddress.isBlank();
+ if (awsvpc && !EcsTaskAddress.isLoopback(eniAddress)) {
String vpcId = containerManager.taskVpcId(task, region);
if (vpcId != null) {
addresses.add(new EcsTaskAddress(eniAddress, null, vpcId));
@@ -1907,7 +1910,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
return addresses;
}
for (Container container : task.getContainers()) {
- String host = containerManager.resolveContainerHost(container);
+ String host = awsvpc
+ ? containerManager.resolvePeerAddress(container).orElse(null)
+ : containerManager.resolveContainerHost(container);
if (host == null || host.isBlank()) {
continue;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrar.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrar.java
index 1c82e9b572..9a3bc72ece 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrar.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrar.java
@@ -246,9 +246,12 @@ public class EcsServiceDiscoveryRegistrar {
}
/**
- * Builds the instance attributes AWS records for an ECS-registered instance. The address is
- * the task's own ENI address in awsvpc mode, and otherwise the container's address on the
- * Docker network, which is the one another container on that network can reach.
+ * Builds the instance attributes AWS records for an ECS-registered instance. On AWS the
+ * address of an awsvpc task is its ENI address, because that is where the task answers. In
+ * Floci the ENI address belongs to no network the task's containers joined, so an awsvpc task
+ * registers the address its container actually holds, and falls back to the ENI address only
+ * when no running container can say what it holds. A bridge-mode task has no ENI and is
+ * registered at the container's host address, as before.
*
* <p>The metadata attributes alongside it are the ones the ECS service discovery
* documentation lists, so a caller can filter a {@code DiscoverInstances} response by them
@@ -259,7 +262,10 @@ public class EcsServiceDiscoveryRegistrar {
Map<String, Object> registry, String region) {
Container container = containerFor(task, string(registry, "containerName"));
String address = task.getPrivateIpAddress();
- if ((address == null || address.isBlank()) && container != null) {
+ boolean awsvpc = address != null && !address.isBlank();
+ if (awsvpc) {
+ address = containerManager.resolvePeerAddress(container).orElse(address);
+ } else if (container != null) {
address = containerManager.resolveContainerHost(container);
}
if (address == null || address.isBlank()) {
@@ -268,7 +274,7 @@ public class EcsServiceDiscoveryRegistrar {
Map<String, String> attributes = new LinkedHashMap<>();
attributes.put("AWS_INSTANCE_IPV4", address);
- Integer port = instancePort(container, registry);
+ Integer port = instancePort(container, registry, awsvpc);
if (port != null) {
attributes.put("AWS_INSTANCE_PORT", String.valueOf(port));
}
@@ -316,19 +322,20 @@ public class EcsServiceDiscoveryRegistrar {
}
/**
- * The port an SRV record would carry. An explicit {@code port} wins, as it does on AWS;
- * otherwise the host port the selected container's {@code containerPort} was published on,
- * so a bridge-mode task advertises the port that is reachable rather than the one inside the
- * container. A {@code containerPort} with no matching binding falls back to itself, which is
- * the awsvpc case where the two are always equal.
+ * The port an SRV record would carry. An explicit {@code port} wins, as it does on AWS.
+ * An awsvpc task is registered at its container's own address, where it listens on the
+ * {@code containerPort} itself, even when Floci also published that port on a dynamic host
+ * port. A bridge-mode task is registered at the host, so it advertises the host port its
+ * {@code containerPort} was published on, falling back to the {@code containerPort} when
+ * there is no matching binding.
*/
- private Integer instancePort(Container container, Map<String, Object> registry) {
+ private Integer instancePort(Container container, Map<String, Object> registry, boolean awsvpc) {
Integer port = integer(registry, "port");
if (port != null) {
return port;
}
Integer containerPort = integer(registry, "containerPort");
- if (container == null || containerPort == null || container.getNetworkBindings() == null) {
+ if (awsvpc || container == null || containerPort == null || container.getNetworkBindings() == null) {
return containerPort;
}
return container.getNetworkBindings().stream()
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
index 0f99f1213f..31fc92a742 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
@@ -1916,13 +1916,40 @@ public class EcsContainerManager {
if (!containerDetector.isRunningInContainer()) {
return "127.0.0.1";
}
+ return dockerNetworkAddress(container).orElse("127.0.0.1");
+ }
+
+ /**
+ * The address another task container reaches this one at: its IP on the Docker network it
+ * joined, whether Floci itself runs natively or in Docker. A task in a security-group
+ * protected namespace has no network of its own, so the namespace holder's address is the one
+ * it answers on.
+ *
+ * <p>This is deliberately not the task ENI address. ECS containers never join the subnet's
+ * VPC network, so the ENI address is control-plane state that no peer can connect to.
+ *
+ * @return the address, or empty when the container has not started or Docker cannot say
+ */
+ public Optional<String> resolvePeerAddress(Container container) {
+ if (container == null || container.getDockerId() == null || container.getDockerId().isBlank()) {
+ return Optional.empty();
+ }
+ return dockerNetworkAddress(container);
+ }
+
+ private Optional<String> dockerNetworkAddress(Container container) {
String dockerId = container.getDockerId();
if (dockerId == null || dockerId.isBlank()) {
- return "127.0.0.1";
+ return Optional.empty();
}
try {
- var inspect = lifecycleManager.getDockerClient().inspectContainerCmd(dockerId).exec();
- var networks = inspect.getNetworkSettings().getNetworks();
+ InspectContainerResponse inspect = lifecycleManager.getDockerClient().inspectContainerCmd(dockerId).exec();
+ String networkMode = inspect.getHostConfig() == null ? null : inspect.getHostConfig().getNetworkMode();
+ if (networkMode != null && networkMode.startsWith("container:")) {
+ inspect = lifecycleManager.getDockerClient()
+ .inspectContainerCmd(networkMode.substring("container:".length())).exec();
+ }
+ Map<String, ContainerNetwork> networks = inspect.getNetworkSettings().getNetworks();
// A container can be on multiple networks; getNetworks() is unordered.
// Pick an IP that the Floci/ELBv2 process can actually route to:
@@ -1932,26 +1959,26 @@ public class EcsContainerManager {
// 3. otherwise the first non-blank IP.
String configured = config.services().ecs().dockerNetwork().orElse(null);
if (configured != null && !configured.isBlank()) {
- var net = networks.get(configured);
+ ContainerNetwork net = networks.get(configured);
if (net != null && isUsableIp(net.getIpAddress())) {
- return net.getIpAddress();
+ return Optional.of(net.getIpAddress());
}
}
- for (var entry : networks.entrySet()) {
+ for (Map.Entry<String, ContainerNetwork> entry : networks.entrySet()) {
if (!isDefaultDockerNetwork(entry.getKey())
&& isUsableIp(entry.getValue().getIpAddress())) {
- return entry.getValue().getIpAddress();
+ return Optional.of(entry.getValue().getIpAddress());
}
}
- for (var net : networks.values()) {
+ for (ContainerNetwork net : networks.values()) {
if (isUsableIp(net.getIpAddress())) {
- return net.getIpAddress();
+ return Optional.of(net.getIpAddress());
}
}
} catch (Exception e) {
LOG.warnv("Could not resolve container IP for {0}: {1}", dockerId, e.getMessage());
}
- return "127.0.0.1";
+ return Optional.empty();
}
private static boolean isUsableIp(String ip) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
index 5121a67631..16633f4aa6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
@@ -58,12 +58,15 @@ import java.io.StringReader;
import java.security.interfaces.RSAPrivateCrtKey;
import java.time.Duration;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.Predicate;
@@ -2080,6 +2083,130 @@ class Ec2ServiceTest {
verify(vpcNetworks, times(2)).reservePrivateIp("us-east-1", "subnet-a", "10.0.1.10");
}
+ @Test
+ void synthesisedAddressesStayInsideTheSubnetCidrPastTheFirstOctetBoundary() {
+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());
+ String vpcId = service.createVpc("us-east-1", "10.70.0.0/16", false).getVpcId();
+ String subnetId = service.createSubnet("us-east-1", vpcId, "10.70.0.0/23", "us-east-1a").getSubnetId();
+
+ Set<String> addresses = new HashSet<>();
+ for (int i = 0; i < 300; i++) {
+ String address = service.createNetworkInterface("us-east-1", subnetId, null,
+ null, List.of(), List.of(), List.of()).getPrivateIpAddress();
+ assertTrue(address.matches("10\\.70\\.[01]\\.(\\d{1,3})"), address);
+ assertTrue(Integer.parseInt(address.substring(address.lastIndexOf('.') + 1)) <= 255, address);
+ assertTrue(addresses.add(address), "handed out twice: " + address);
+ }
+ assertTrue(addresses.contains("10.70.1.10"), "the counter must carry into the next /24 octet");
+ }
+
+ @Test
+ void synthesisedAddressesSkipReservedOnesAndReportExhaustion() {
+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());
+ String vpcId = service.createVpc("us-east-1", "10.71.0.0/16", false).getVpcId();
+ String subnetId = service.createSubnet("us-east-1", vpcId, "10.71.0.0/28", "us-east-1a").getSubnetId();
+
+ Set<String> addresses = new HashSet<>();
+ for (int i = 0; i < 11; i++) {
+ addresses.add(service.createNetworkInterface("us-east-1", subnetId, null,
+ null, List.of(), List.of(), List.of()).getPrivateIpAddress());
+ }
+
+ Set<String> usable = new HashSet<>();
+ for (int host = 4; host <= 14; host++) {
+ usable.add("10.71.0." + host);
+ }
+ assertEquals(usable, addresses);
+ assertEquals("InsufficientFreeAddressesInSubnet", assertThrows(AwsException.class,
+ () -> service.createNetworkInterface("us-east-1", subnetId, null,
+ null, List.of(), List.of(), List.of())).getErrorCode());
+ }
+
+ @Test
+ void aLaunchTooLargeForTheSubnetStoresNoneOfItsInstances() {
+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());
+ String vpcId = service.createVpc("us-east-1", "10.74.0.0/16", false).getVpcId();
+ String subnetId = service.createSubnet("us-east-1", vpcId, "10.74.0.0/28", "us-east-1a").getSubnetId();
+ int volumesBefore = service.describeVolumes("us-east-1", List.of(), Map.of()).size();
+
+ AwsException error = assertThrows(AwsException.class,
+ () -> service.runInstances("us-east-1", "ami-1234567890abcdef0", "t3.micro",
+ 12, 12, null, List.of(), subnetId, null, List.of(), null, null));
+
+ assertEquals("InsufficientFreeAddressesInSubnet", error.getErrorCode());
+ assertTrue(service.describeInstances("us-east-1", List.of(),
+ Map.of("subnet-id", List.of(subnetId))).isEmpty());
+ assertEquals(volumesBefore, service.describeVolumes("us-east-1", List.of(), Map.of()).size());
+ Reservation reservation = service.runInstances("us-east-1", "ami-1234567890abcdef0", "t3.micro",
+ 11, 11, null, List.of(), subnetId, null, List.of(), null, null);
+ assertEquals(11, reservation.getInstances().stream().map(Instance::getPrivateIpAddress).distinct().count());
+ }
+
+ @Test
+ void concurrentCreatesInASmallSubnetNeverShareASynthesisedAddress() throws Exception {
+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory());
+ String vpcId = service.createVpc("us-east-1", "10.73.0.0/16", false).getVpcId();
+ int rounds = 20;
+ int usable = 11;
+ ExecutorService executor = Executors.newFixedThreadPool(usable);
+ try {
+ for (int round = 0; round < rounds; round++) {
+ String subnetId = service.createSubnet("us-east-1", vpcId, "10.73." + round + ".0/28",
+ "us-east-1a").getSubnetId();
+ CountDownLatch start = new CountDownLatch(1);
+ List<Future<String>> creates = new ArrayList<>();
+ for (int i = 0; i < usable; i++) {
+ creates.add(executor.submit(() -> {
+ start.await();
+ return service.createNetworkInterface("us-east-1", subnetId, null,
+ null, List.of(), List.of(), List.of()).getPrivateIpAddress();
+ }));
+ }
+ start.countDown();
+ Set<String> addresses = new HashSet<>();
+ for (Future<String> create : creates) {
+ String address = create.get(10, TimeUnit.SECONDS);
+ assertTrue(addresses.add(address), "handed out twice: " + address);
+ }
+ assertEquals(usable, addresses.size());
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void synthesisedAddressesAreNotReusedAfterARestart() {
+ Map<String, AccountAwareStorageBackend<?>> stores = Map.of(
+ "ec2-vpcs.json", AccountAwareStorageBackend.<Vpc>inMemory("000000000000"),
+ "ec2-subnets.json", AccountAwareStorageBackend.<Subnet>inMemory("000000000000"),
+ "ec2-network-interfaces.json", AccountAwareStorageBackend.<NetworkInterface>inMemory("000000000000"));
+ Ec2Service before = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory(stores));
+ String vpcId = before.createVpc("us-east-1", "10.72.0.0/16", false).getVpcId();
+ String subnetId = before.createSubnet("us-east-1", vpcId, "10.72.1.0/24", "us-east-1a").getSubnetId();
+ String held = before.createNetworkInterface("us-east-1", subnetId, null,
+ null, List.of(), List.of(), List.of()).getPrivateIpAddress();
+
+ Ec2Service after = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(), new InMemoryStorageFactory(stores));
+ String next = after.createNetworkInterface("us-east-1", subnetId, null,
+ null, List.of(), List.of(), List.of()).getPrivateIpAddress();
+
+ assertEquals("10.72.1.10", held);
+ assertNotEquals(held, next);
+ }
+
private static Instance persistedInstance(String instanceId, String privateIp, InstanceState state) {
Instance instance = new Instance();
instance.setInstanceId(instanceId);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.java
index a7a156678d..53f1aa86c2 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.ecs;
import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.model.ContainerNetwork;
import io.github.hectorvent.floci.services.cloudmap.CloudMapService;
import io.github.hectorvent.floci.services.cloudmap.model.Operation;
import io.github.hectorvent.floci.services.cloudmap.model.Service;
@@ -103,9 +104,15 @@ class EcsServiceDiscoveryDockerIntegrationTest {
assertEquals("RUNNING", task.getLastStatus(), "the container must be up: "
+ task.getStoppedReason());
assertNotNull(task.getPrivateIpAddress(), "an awsvpc task must carry its ENI address");
+ List<String> containerAddresses = containerAddresses(task);
- assertEquals(List.of(task.getPrivateIpAddress()), cloudMapService.resolveDnsName(dnsName),
+ List<String> resolved = cloudMapService.resolveDnsName(dnsName);
+ assertEquals(1, resolved.size(),
"starting the task must register it, so the Cloud Map name resolves to it");
+ assertTrue(containerAddresses.contains(resolved.getFirst()),
+ "the name must resolve to an address the container holds, " + containerAddresses
+ + ", not to the ENI address " + task.getPrivateIpAddress()
+ + " that no peer can connect to");
// Updating the registry moves the running task to the replacement. Scaling to zero
// keeps the reconciler from replacing it between the stop and the final assertions.
@@ -120,7 +127,7 @@ class EcsServiceDiscoveryDockerIntegrationTest {
ecsService.updateService(update, REGION);
assertTrue(cloudMapService.listInstances(cloudMapSvc.getId()).isEmpty(),
"updating the registry must remove the old Cloud Map instance");
- assertEquals(List.of(task.getPrivateIpAddress()),
+ assertEquals(resolved,
cloudMapService.resolveDnsName("replacement." + namespaceName),
"updating the registry must register the running task in the new service");
ecsService.stopTask(clusterName, taskArn, "service discovery test", REGION);
@@ -140,6 +147,16 @@ class EcsServiceDiscoveryDockerIntegrationTest {
}
}
+ /** Every address the task container holds on a Docker network, i.e. where a peer reaches it. */
+ private List<String> containerAddresses(EcsTask task) {
+ String dockerId = task.getContainers().getFirst().getDockerId();
+ return dockerClient.inspectContainerCmd(dockerId).exec()
+ .getNetworkSettings().getNetworks().values().stream()
+ .map(ContainerNetwork::getIpAddress)
+ .filter(address -> address != null && !address.isBlank())
+ .toList();
+ }
+
private CreateServiceRequest createServiceRequest(String clusterName, TaskDefinition taskDef,
Service cloudMapSvc) {
AwsVpcConfiguration awsvpc = new AwsVpcConfiguration();
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrarAddressTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrarAddressTest.java
new file mode 100644
index 0000000000..28a07b1b23
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryRegistrarAddressTest.java
@@ -0,0 +1,144 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.cloudmap.CloudMapService;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.model.Container;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
+import io.github.hectorvent.floci.services.ecs.model.EcsTask;
+import io.github.hectorvent.floci.services.ecs.model.NetworkBinding;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Which address the registrar puts into Cloud Map. A task ENI address is not on any network the
+ * task's containers joined, so registering it gives a peer a name that resolves and a connection
+ * that times out.
+ */
+class EcsServiceDiscoveryRegistrarAddressTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String REGISTRY_ARN =
+ "arn:aws:servicediscovery:us-east-1:000000000000:service/srv-abc123";
+
+ private final CloudMapService cloudMapService = mock(CloudMapService.class);
+ private final EcsContainerManager containerManager = mock(EcsContainerManager.class);
+ private final EcsServiceDiscoveryRegistrar registrar =
+ new EcsServiceDiscoveryRegistrar(cloudMapService, containerManager, storageFactory());
+
+ @Test
+ void awsvpcTaskRegistersTheAddressItsContainerHolds() {
+ EcsTask task = task("10.70.1.10");
+ when(containerManager.resolvePeerAddress(any(Container.class))).thenReturn(Optional.of("172.31.0.19"));
+
+ registrar.registerTask(task, service(), REGION);
+
+ assertEquals("172.31.0.19", registeredAddress());
+ verify(containerManager, never()).resolveContainerHost(any(Container.class));
+ }
+
+ @Test
+ void awsvpcTaskKeepsItsEniAddressWhenNoContainerAnswers() {
+ EcsTask task = task("10.70.1.10");
+ when(containerManager.resolvePeerAddress(any(Container.class))).thenReturn(Optional.empty());
+
+ registrar.registerTask(task, service(), REGION);
+
+ assertEquals("10.70.1.10", registeredAddress());
+ }
+
+ @Test
+ void bridgeTaskStillRegistersTheContainerHost() {
+ EcsTask task = task(null);
+ when(containerManager.resolveContainerHost(any(Container.class))).thenReturn("127.0.0.1");
+
+ registrar.registerTask(task, service(), REGION);
+
+ assertEquals("127.0.0.1", registeredAddress());
+ verify(containerManager, never()).resolvePeerAddress(any(Container.class));
+ }
+
+ @Test
+ void awsvpcTaskAdvertisesTheContainerPortItsAddressAnswersOn() {
+ EcsTask task = task("10.70.1.10");
+ task.getContainers().getFirst().setNetworkBindings(
+ List.of(new NetworkBinding("0.0.0.0", 8080, 49153, "tcp")));
+ when(containerManager.resolvePeerAddress(any(Container.class))).thenReturn(Optional.of("172.31.0.19"));
+
+ registrar.registerTask(task, serviceWithContainerPort(8080), REGION);
+
+ Map<String, String> attributes = registeredAttributes();
+ assertEquals("172.31.0.19", attributes.get("AWS_INSTANCE_IPV4"));
+ assertEquals("8080", attributes.get("AWS_INSTANCE_PORT"));
+ }
+
+ @Test
+ void bridgeTaskAdvertisesThePublishedHostPort() {
+ EcsTask task = task(null);
+ task.getContainers().getFirst().setNetworkBindings(
+ List.of(new NetworkBinding("0.0.0.0", 8080, 49153, "tcp")));
+ when(containerManager.resolveContainerHost(any(Container.class))).thenReturn("127.0.0.1");
+
+ registrar.registerTask(task, serviceWithContainerPort(8080), REGION);
+
+ assertEquals("49153", registeredAttributes().get("AWS_INSTANCE_PORT"));
+ }
+
+ private String registeredAddress() {
+ return registeredAttributes().get("AWS_INSTANCE_IPV4");
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, String> registeredAttributes() {
+ ArgumentCaptor<Map<String, String>> attributes = ArgumentCaptor.forClass(Map.class);
+ verify(cloudMapService).registerInstance(eq("srv-abc123"), anyString(), isNull(),
+ attributes.capture(), eq(REGION));
+ return attributes.getValue();
+ }
+
+ private static EcsServiceModel service() {
+ EcsServiceModel svc = new EcsServiceModel();
+ svc.setServiceRegistries(List.of(Map.of("registryArn", REGISTRY_ARN, "containerName", "app")));
+ return svc;
+ }
+
+ private static EcsServiceModel serviceWithContainerPort(int containerPort) {
+ EcsServiceModel svc = new EcsServiceModel();
+ svc.setServiceRegistries(List.of(Map.of("registryArn", REGISTRY_ARN, "containerName", "app",
+ "containerPort", containerPort)));
+ return svc;
+ }
+
+ private static EcsTask task(String eniAddress) {
+ Container container = new Container();
+ container.setName("app");
+ container.setDockerId("docker-app");
+ EcsTask task = new EcsTask();
+ task.setTaskArn("arn:aws:ecs:" + REGION + ":000000000000:task/c/0123456789abcdef");
+ task.setPrivateIpAddress(eniAddress);
+ task.setContainers(List.of(container));
+ return task;
+ }
+
+ private static StorageFactory storageFactory() {
+ StorageFactory storageFactory = mock(StorageFactory.class);
+ when(storageFactory.create(anyString(), anyString(), any()))
+ .thenAnswer(invocation -> AccountAwareStorageBackend.inMemory("000000000000"));
+ return storageFactory;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java
index cde3674653..7de7cadfe3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRestartLeftoversTest.java
@@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import static org.hamcrest.MatcherAssert.assertThat;
@@ -167,7 +168,9 @@ class EcsServiceRestartLeftoversTest {
task.setPrivateIpAddress("10.0.1.5");
return new EcsTaskHandle(task.getTaskArn(), Map.of("web", "docker-id"), Map.of());
});
- when(containerManager.resolveContainerHost(any())).thenReturn("172.19.0.4");
+ // Floci running natively: targets reach the task on loopback, Cloud Map registers its Docker address.
+ when(containerManager.resolveContainerHost(any())).thenReturn("127.0.0.1");
+ when(containerManager.resolvePeerAddress(any())).thenReturn(Optional.of("172.19.0.4"));
when(containerManager.taskVpcId(any(), anyString())).thenReturn("vpc-app");
EcsLoadBalancerRegistrar lbRegistrar = mock(EcsLoadBalancerRegistrar.class);
EcsServiceDiscoveryRegistrar discoveryRegistrar = mock(EcsServiceDiscoveryRegistrar.class);
@@ -194,7 +197,7 @@ class EcsServiceRestartLeftoversTest {
assertEquals("RUNNING", task.getLastStatus());
// A target only on the ports the container serves, and never at the VPC-scoped ENI address.
verify(lbRegistrar).evictUnrecordedTargets(task, Set.of(TARGET_GROUP_ARN),
- Set.of(new EcsTaskAddress("172.19.0.4", 80)), REGION);
+ Set.of(new EcsTaskAddress("127.0.0.1", 80)), REGION);
verify(discoveryRegistrar).evictUnrecordedInstances(task, Set.of("srv-alpha"),
Set.of(new EcsTaskAddress("10.0.1.5", null, "vpc-app"), new EcsTaskAddress("172.19.0.4", null)),
REGION);