Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2): launch instances into capacity reservation groups (#6404)
Andre Kurait60ef203fa880
diff --git a/compatibility-tests/sdk-test-python/tests/test_ec2_capacity_reservation_group_target.py b/compatibility-tests/sdk-test-python/tests/test_ec2_capacity_reservation_group_target.pynew file mode 100644index 0000000000..f6d1a82fad--- /dev/null+++ b/compatibility-tests/sdk-test-python/tests/test_ec2_capacity_reservation_group_target.py@@ -0,0 +1,127 @@+"""Exercise capacity group targets through EC2 and Resource Groups SDKs."""+import uuid++import boto3+import pytest+from botocore.exceptions import ClientError++ACCOUNT = "111111111111"+REGION = "us-east-1"+++def sdk_client(service, aws_config, client_config, account, region):+ config = dict(aws_config, aws_access_key_id=account, aws_secret_access_key="test", region_name=region)+ return boto3.client(service, config=client_config, **config)+++def pool(aws_config, client_config):+ clients = {name: sdk_client(name, aws_config, client_config, ACCOUNT, REGION)+ for name in ("ec2", "resource-groups")}+ ec2, groups = clients["ec2"], clients["resource-groups"]+ created = {"reservations": [], "instances": [], "templates": []}+ group = groups.create_group(Name="pool-" + uuid.uuid4().hex, Configuration=[+ {"Type": "AWS::EC2::CapacityReservationPool"},+ {"Type": "AWS::ResourceGroups::Generic", "Parameters": [+ {"Name": "allowed-resource-types", "Values": ["AWS::EC2::CapacityReservation"]},+ ]},+ ])["Group"]["GroupArn"]++ def reserve(count=1, zone=REGION + "a", member=True):+ reservation = ec2.create_capacity_reservation(+ InstanceType="t3.micro", InstancePlatform="Linux/UNIX", AvailabilityZone=zone,+ InstanceCount=count, InstanceMatchCriteria="targeted" if member else "open",+ )["CapacityReservation"]+ created["reservations"].append(reservation["CapacityReservationId"])+ if member:+ reply = groups.group_resources(Group=group, ResourceArns=[reservation["CapacityReservationArn"]])+ assert reply["Succeeded"] == [reservation["CapacityReservationArn"]]+ assert not reply["Failed"] and not reply["Pending"]+ return reservation["CapacityReservationId"]++ try:+ yield clients, group, reserve, created+ finally:+ if created["instances"]:+ ec2.terminate_instances(InstanceIds=created["instances"])+ for template in created["templates"]:+ ec2.delete_launch_template(LaunchTemplateId=template)+ groups.delete_group(Group=group)+ for reservation in created["reservations"]:+ ec2.cancel_capacity_reservation(CapacityReservationId=reservation)+ for sdk in clients.values():+ sdk.close()++[email protected]("template", [False, True])+def test_group_round_trip_matching_and_strict_fallback(pool, template):+ clients, group, reserve, created = pool+ ec2 = clients["ec2"]+ members = {reserve(), reserve()}+ outside = reserve(4, member=False)+ specification = {+ "CapacityReservationPreference": "capacity-reservations-only",+ "CapacityReservationTarget": {"CapacityReservationResourceGroupArn": group},+ }+ request = {"MinCount": 2, "MaxCount": 2}+ data = {"ImageId": "ami-test", "InstanceType": "t3.micro",+ "CapacityReservationSpecification": specification}+ if template:+ selected = ec2.create_launch_template(+ LaunchTemplateName="pool-template-" + uuid.uuid4().hex,+ LaunchTemplateData=data)["LaunchTemplate"]["LaunchTemplateId"]+ created["templates"].append(selected)+ request["LaunchTemplate"] = {"LaunchTemplateId": selected}+ else:+ request.update(data)+ launched = ec2.run_instances(**request)["Instances"]+ created["instances"].extend(row["InstanceId"] for row in launched)+ assert {row["CapacityReservationId"] for row in launched} == members+ described = ec2.describe_instances(InstanceIds=created["instances"])["Reservations"]+ response_specification = {"CapacityReservationTarget": specification["CapacityReservationTarget"]}+ assert all(instance["CapacityReservationSpecification"] == response_specification+ for reservation in described for instance in reservation["Instances"])+ with pytest.raises(ClientError) as error:+ ec2.run_instances(**request)+ assert error.value.response["Error"]["Code"] == "InsufficientInstanceCapacity"+ assert ec2.describe_capacity_reservations(CapacityReservationIds=[outside])[+ "CapacityReservations"][0]["AvailableInstanceCount"] == 4+++def test_group_default_uses_on_demand_without_outside_reservations(pool):+ clients, group, reserve, created = pool+ ec2 = clients["ec2"]+ outside = reserve(2, member=False)+ instance = ec2.run_instances(+ ImageId="ami-test", InstanceType="t3.micro", MinCount=1, MaxCount=1,+ CapacityReservationSpecification={+ "CapacityReservationTarget": {"CapacityReservationResourceGroupArn": group}},+ )["Instances"][0]+ created["instances"].append(instance["InstanceId"])+ assert "CapacityReservationId" not in instance+ assert "CapacityReservationPreference" not in instance["CapacityReservationSpecification"]+ assert instance["CapacityReservationSpecification"]["CapacityReservationTarget"] == {+ "CapacityReservationResourceGroupArn": group}+ assert ec2.describe_capacity_reservations(CapacityReservationIds=[outside])[+ "CapacityReservations"][0]["AvailableInstanceCount"] == 2++[email protected]("account,region", [("222222222222", REGION), (ACCOUNT, "us-west-2")])+def test_group_target_cannot_cross_account_or_region(aws_config, client_config, pool, account, region):+ _, group, _, _ = pool+ ec2 = sdk_client("ec2", aws_config, client_config, account, region)+ try:+ before = {instance["InstanceId"] for reservation in ec2.describe_instances()["Reservations"]+ for instance in reservation["Instances"]}+ with pytest.raises(ClientError) as error:+ ec2.run_instances(+ ImageId="ami-test", InstanceType="t3.micro", MinCount=1, MaxCount=1,+ CapacityReservationSpecification={+ "CapacityReservationTarget": {"CapacityReservationResourceGroupArn": group}},+ )+ assert error.value.response["Error"]["Code"] == "InvalidParameterValue"+ after = {instance["InstanceId"] for reservation in ec2.describe_instances()["Reservations"]+ for instance in reservation["Instances"]}+ assert after == before+ finally:+ ec2.close()diff --git a/docs/services/ec2.md b/docs/services/ec2.mdindex 77311f64cd..4c34896101 100644--- a/docs/services/ec2.md+++ b/docs/services/ec2.md@@ -782,7 +782,19 @@ instance's retained launch preference and target. A failed membership write reposubsequent access reconciles released membership from persisted instance state.Membership and available counts publish in one atomic storage batch. Older snapshots without-membership remain readable. Capacity reservation resource-group targeting is unsupported.+membership remain readable.++`CapacityReservationResourceGroupArn` targets the members of an existing capacity pool+in the caller's account and region. Only members with matching instance type, platform,+Availability Zone and tenancy are eligible. Both open and targeted member reservations+can supply capacity. The default permits On-Demand fallback when the group has no+matching available capacity; `capacity-reservations-only` forbids that fallback at launch+and restart. A group target cannot accompany a reservation ID, `none`, or a Spot request.+DescribeInstances retains the group target separately from the acquired reservation ID.+Running group-targeted instances can acquire newly added matching members or move to+another matching member after cancellation or expiration. Existing membership survives+group deletion until the instance releases its reservation.+Capacity reservation zone IDs must be among those returned by `DescribeAvailabilityZones`.diff --git a/docs/services/resource-groups.md b/docs/services/resource-groups.mdindex 4cdf48d550..c2ff757367 100644--- a/docs/services/resource-groups.md+++ b/docs/services/resource-groups.md@@ -48,7 +48,7 @@ Deleting a group removes its membership and tokens. It does not cancel or delete## Scope-Only the five operations and capacity-pool configuration above are implemented. Dynamic tag/CloudFormation queries, other service configurations, independent tag-update operations and `UngroupResources` are not supported. Unsupported configurations are rejected. Membership does not allocate physical capacity or make instance scheduling consume a reservation. No asynchronous membership transition is simulated: accepted additions complete synchronously and `Pending` is empty.+Only the five operations and capacity-pool configuration above are implemented. Dynamic tag/CloudFormation queries, other service configurations, independent tag-update operations and `UngroupResources` are not supported. Unsupported configurations are rejected. Membership does not allocate physical capacity. EC2 launches that target the group consume matching available member reservations; the launch preference controls On-Demand fallback. No asynchronous membership transition is simulated: accepted additions complete synchronously and `Pending` is empty.Group state follows the configured Floci storage mode. Pagination tokens are in memory and are intentionally invalidated by restart/reset.diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.javaindex 4ff5558bc0..3cfcfac317 100644--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java@@ -591,13 +591,15 @@ public class AutoScalingReconciler {private Reservation launchWithCapacity(AutoScalingGroup asg, LaunchSource launchSource,LaunchPlacement placement, String clientToken,LaunchTemplateData.CapacityReservationSpecification capacitySpecification) {+ String launchZone = Ec2Service.isModelledAvailabilityZone(asg.getRegion(), placement.availabilityZone())+ ? placement.availabilityZone() : null;return ec2Service.runInstances(asg.getRegion(), launchSource.imageId(), launchSource.instanceType(), 1, 1,launchSource.keyName(), launchSource.securityGroupIds(), placement.subnetId(), clientToken,propagatedInstanceTags(asg, launchSource), launchSource.userData(),launchSource.launchConfigurationName() == null ? launchSource.iamInstanceProfile(): asgService.resolveLaunchConfigurationProfile(launchSource.iamInstanceProfile()),- launchSource.associatePublicIpAddress(), null, 0, null, launchSource.metadataOptions(), null,+ launchSource.associatePublicIpAddress(), null, 0, launchZone, launchSource.metadataOptions(), null,launchSource.encodedUserData() != null ? launchSource.encodedUserData(): encodeLegacyUserData(launchSource.userData()),false, null, null, launchSource.blockDeviceMappings(), false, null,diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2QueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2QueryHandler.javaindex 33e833e4cd..71f384c1d0 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2QueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2QueryHandler.java@@ -5188,12 +5188,14 @@ public class Ec2QueryHandler {.elem("enableResourceNameDnsARecord", "false").elem("enableResourceNameDnsAAAARecord", "false").end("privateDnsNameOptions")- .start("capacityReservationSpecification")- .elem("capacityReservationPreference", inst.getCapacityReservationPreference());- if (inst.getCapacityReservationTarget() != null) {+ .start("capacityReservationSpecification");+ if (inst.getCapacityReservationTarget() != null || inst.getCapacityReservationGroupArn() != null) {xml.start("capacityReservationTarget").elem("capacityReservationId", inst.getCapacityReservationTarget())+ .elem("capacityReservationResourceGroupArn", inst.getCapacityReservationGroupArn()).end("capacityReservationTarget");+ } else {+ xml.elem("capacityReservationPreference", inst.getCapacityReservationPreference());}xml.end("capacityReservationSpecification");String capacityReservationId = capacityAssociations.get(inst.getInstanceId());diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.javaindex 6d5e6a45b4..ba19309488 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java@@ -105,6 +105,7 @@ import io.github.hectorvent.floci.services.ec2.portforward.Ec2PortForwardManagerimport io.github.hectorvent.floci.services.iam.IamPolicyEvaluator;import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.iam.model.InstanceProfile;+import io.github.hectorvent.floci.services.resourcegroups.ResourceGroupsService;import jakarta.annotation.PostConstruct;import jakarta.enterprise.context.ApplicationScoped;import jakarta.enterprise.context.ContextNotActiveException;@@ -318,6 +319,10 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabprivate final StorageBackend<String, List<Tag>> tags;private final StorageBackend<String, CapacityReservation> capacityReservations;private final Object capacityReservationLock = new Object();+ // Lazily seeded per owner/region, guarded by capacityReservationLock; values remain in storage.+ private final Map<String, Set<String>> capacityGroupInstanceIds = new HashMap<>();+ // CDI Instance clashes with the EC2 Instance model imported above.+ jakarta.enterprise.inject.Instance<ResourceGroupsService> resourceGroupsServices;private final StorageBackend<String, Host> hosts;private final StorageBackend<String, EgressOnlyInternetGateway> egressOnlyInternetGateways;// Serialises a launch onto a host against ModifyHosts/ReleaseHosts, so a host cannot be@@ -424,6 +429,14 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabthis.testClusterNodeInstanceProvider = provider;}+ Map<String, Set<String>> capacityGroupInstanceIdsForTest() {+ return capacityGroupInstanceIds;+ }++ boolean holdsCapacityReservationLockForTest() {+ return Thread.holdsLock(capacityReservationLock);+ }+void putInstanceForTest(Instance instance) {instances.put(key(instance.getRegion(), instance.getInstanceId()), instance);}@@ -487,7 +500,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabjakarta.enterprise.inject.Instance<VpcRouteTableListener> routeTableListenersInstance,jakarta.enterprise.inject.Instance<Ec2InstanceLaunchListener> instanceLaunchListenersInstance,Ec2EbsEncryptionService ebsEncryptionService,- jakarta.enterprise.inject.Instance<VpcDependencyProvider> vpcDependencyProviders) {+ jakarta.enterprise.inject.Instance<VpcDependencyProvider> vpcDependencyProviders,+ jakarta.enterprise.inject.Instance<ResourceGroupsService> resourceGroupsServices) {this(config, containerManager, portForwardManager, amiImageResolver, imageCatalog,instanceTypeCatalog, storageFactory, requestContextInstance, iamService, volumeBlockDeviceManager);this.vpcNetworkManager = vpcNetworkManager;@@ -496,6 +510,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabthis.instanceLaunchListenersInstance = instanceLaunchListenersInstance;this.ebsEncryptionService = ebsEncryptionService;this.vpcDependencyProviders = vpcDependencyProviders;+ this.resourceGroupsServices = resourceGroupsServices;}public Ec2Service(EmulatorConfig config, Ec2ContainerManager containerManager,@@ -1381,6 +1396,9 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab// VPC and subnets would never come back.@Overridepublic void clear() {+ synchronized (capacityReservationLock) {+ capacityGroupInstanceIds.clear();+ }seededAccountRegions.clear();seedDefaultRegionIfEnabled();}@@ -2051,6 +2069,10 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabreturn "subnet-default-" + region + "-" + azSuffix;}+ public static boolean isModelledAvailabilityZone(String region, String zone) {+ return Arrays.stream(MODELLED_ZONE_SUFFIXES).anyMatch(suffix -> (region + suffix).equals(zone));+ }+public static String defaultSecurityGroupId(String region) {return "sg-default-" + region;}@@ -3847,16 +3869,19 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, ResettabLaunchTemplateData.CapacityReservationTarget capacityTarget = capacitySpecification == null ? null: capacitySpecification.getCapacityReservationTarget();String capacityTargetId = capacityTarget == null ? null : capacityTarget.getCapacityReservationId();+ String capacityGroupArn = capacityTarget == null ? null+ : capacityTarget.getCapacityReservationResourceGroupArn();if (!Set.of("open", "none", "capacity-reservations-only").contains(capacityPreference)) {throw new AwsException("InvalidParameterValue", "Invalid Capacity Reservation preference.", 400);}- if (capacityTarget != null && capacityTarget.getCapacityReservationResourceGroupArn() != null) {- throw new AwsException("Unsupported", "Capacity Reservation resource group targeting is not supported.", 400);- }- if ((capacityTargetId != null && "none".equals(capacityPreference))- || (spot && (capacityTargetId != null || "capacity-reservations-only".equals(capacityPreference)))) {+ if ((capacityTargetId != null && capacityGroupArn != null)+ || ((capacityTargetId != null || capacityGroupArn != null) && "none".equals(capacityPreference))+ || (spot && (capacityTargetId != null || capacityGroupArn != null+ || "capacity-reservations-only".equals(capacityPreference)))) {throw new AwsException("InvalidParameterCombination", "Invalid Capacity Reservation target combination.", 400);}+ Set<String> capacityGroupMembers = capacityGroupArn == null ? null+ : capacityGroupMembers(region, callerAccountId(), capacityGroupArn, true);if (privateIpAddress != null && privateIpAddress.isBlank()) {privateIpAddress = null;}@@ -3981,7 +4006,9 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab}long available = target == null? capacityReservations.scan(k -> k.startsWith(region + "::")).stream()- .filter(candidate -> "open".equals(candidate.getInstanceMatchCriteria()))+ .filter(candidate -> capacityGroupMembers == null+ ? "open".equals(candidate.getInstanceMatchCriteria())+ : capacityGroupMembers.contains(candidate.getCapacityReservationArn())).filter(candidate -> matchesCapacityReservation(requested, candidate)).mapToLong(CapacityReservation::getAvailableInstanceCount).sum(): target.getAvailableInstanceCount();@@ -4060,6 +4087,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabinst.setCapacityReservationPlatform(capacityReservationPlatform(region, imageId));inst.setCapacityReservationPreference(capacityTargetId == null ? capacityPreference : null);inst.setCapacityReservationTarget(capacityTargetId);+ inst.setCapacityReservationGroupArn(capacityGroupArn);Placement placement = new Placement(az);placement.setHostId(hostId);if (tenancy != null && !tenancy.isBlank()) {@@ -7266,9 +7294,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabprivate String resolveSubnetZoneName(String region, String availabilityZone, String availabilityZoneId) {if (!isSet(availabilityZoneId)) {String zoneName = isSet(availabilityZone) ? availabilityZone : region + "a";- boolean zoneExists = Arrays.stream(MODELLED_ZONE_SUFFIXES)- .anyMatch(suffix -> zoneName.equals(region + suffix));- if (!zoneExists) {+ if (!isModelledAvailabilityZone(region, zoneName)) {throw new AwsException("InvalidParameterValue","Value (" + zoneName + ") for parameter availabilityZone is invalid. Subnets can "+ "currently only be created in the following availability zones: "@@ -10377,9 +10403,11 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab}private void reconcileCapacityMembership(String region) {+ List<CapacityReservation> reconciled = new ArrayList<>();Map<String, CapacityReservation> changed = new LinkedHashMap<>();for (CapacityReservation stored : capacityReservations.scan(k -> k.startsWith(region + "::"))) {CapacityReservation updated = new CapacityReservation(stored);+ reconciled.add(updated);boolean expired = "active".equals(updated.getState()) && updated.getEndDate() != null&& !updated.getEndDate().isAfter(Instant.now());if (expired) {@@ -10394,11 +10422,68 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabchanged.put(key(region, updated.getCapacityReservationId()), updated);}}+ List<CapacityReservation> available = reconciled.stream()+ .filter(reservation -> "active".equals(reservation.getState())+ && reservation.getAvailableInstanceCount() > 0).toList();+ List<Instance> groupInstances = capacityGroupInstances(region);+ if (!available.isEmpty()) {+ Set<String> associated = new HashSet<>();+ for (CapacityReservation reservation : reconciled) {+ associated.addAll(reservation.getInstanceIds());+ }+ Map<String, Set<String>> membersByGroup = new HashMap<>();+ for (Instance instance : groupInstances) {+ if (instance.getState() == null || !"running".equals(instance.getState().getName())+ || instance.isSpot() || instance.getCapacityReservationGroupArn() == null+ || instance.getCapacityReservationTarget() != null+ || !Objects.equals(callerAccountId(), instance.getOwnerId())+ || associated.contains(instance.getInstanceId())) {+ continue;+ }+ Set<String> groupMembers = membersByGroup.computeIfAbsent(instance.getCapacityReservationGroupArn(),+ arn -> capacityGroupMembers(region, instance.getOwnerId(), arn, false));+ CapacityReservation selected = available.stream()+ .filter(reservation -> groupMembers.contains(reservation.getCapacityReservationArn()))+ .filter(reservation -> reservation.getAvailableInstanceCount() > 0)+ .filter(reservation -> matchesCapacityReservation(instance, reservation)).findFirst().orElse(null);+ if (selected != null) {+ selected.getInstanceIds().add(instance.getInstanceId());+ selected.setAvailableInstanceCount(selected.getAvailableInstanceCount() - 1);+ associated.add(instance.getInstanceId());+ changed.put(key(region, selected.getCapacityReservationId()), selected);+ }+ }+ }if (!changed.isEmpty()) {persistCapacityMembership(changed);}}+ private List<Instance> capacityGroupInstances(String region) {+ String account = callerAccountId();+ Set<String> ids = capacityGroupInstanceIds.computeIfAbsent(account + "::" + region, ignored -> {+ Set<String> seeded = new HashSet<>();+ for (Instance instance : instances.scan(k -> k.startsWith(region + "::"))) {+ if (instance.getCapacityReservationGroupArn() != null && account.equals(instance.getOwnerId())) {+ seeded.add(instance.getInstanceId());+ }+ }+ return seeded;+ });+ List<Instance> candidates = new ArrayList<>();+ ids.removeIf(id -> {+ Instance instance = instances.get(key(region, id)).orElse(null);+ if (instance == null || instance.getCapacityReservationGroupArn() == null+ || !account.equals(instance.getOwnerId())+ || (instance.getState() != null && "terminated".equals(instance.getState().getName()))) {+ return true;+ }+ candidates.add(instance);+ return false;+ });+ return candidates;+ }+private boolean holdsCapacity(Instance instance) {return instance.getState() != null && ("pending".equals(instance.getState().getName())|| "running".equals(instance.getState().getName()));@@ -10406,6 +10491,14 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabvoid acquireCapacity(String region, List<Instance> admitted) {synchronized (capacityReservationLock) {+ Set<String> groupInstances = capacityGroupInstanceIds.get(callerAccountId() + "::" + region);+ if (groupInstances != null) {+ for (Instance instance : admitted) {+ if (instance.getCapacityReservationGroupArn() != null) {+ groupInstances.add(instance.getInstanceId());+ }+ }+ }reconcileCapacityMembership(region);List<CapacityReservation> candidates = capacityReservations.scan(k -> k.startsWith(region + "::")).stream().map(CapacityReservation::new).toList();@@ -10419,9 +10512,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabcontinue;}String target = instance.getCapacityReservationTarget();+ Set<String> groupMembers = instance.getCapacityReservationGroupArn() == null ? null+ : capacityGroupMembers(region, instance.getOwnerId(),+ instance.getCapacityReservationGroupArn(), false);CapacityReservation selected = candidates.stream()- .filter(r -> target == null ? "open".equals(r.getInstanceMatchCriteria())- : target.equals(r.getCapacityReservationId()))+ .filter(r -> groupMembers != null ? groupMembers.contains(r.getCapacityReservationArn())+ : target == null ? "open".equals(r.getInstanceMatchCriteria())+ : target.equals(r.getCapacityReservationId())).filter(r -> matchesCapacityReservation(instance, r)).filter(r -> r.getAvailableInstanceCount() > 0).findFirst().orElse(null);if (selected == null) {@@ -10479,6 +10576,23 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab&& Objects.equals(instance.getCapacityReservationPlatform(), reservation.getInstancePlatform());}+ private Set<String> capacityGroupMembers(String region, String account, String arn, boolean launching) {+ if (resourceGroupsServices == null || !resourceGroupsServices.isResolvable()) {+ throw new AwsException("Unsupported", "Capacity Reservation resource groups are unavailable.", 400);+ }+ try {+ return Set.copyOf(resourceGroupsServices.get().capacityReservationMembers(region, account, arn));+ } catch (AwsException error) {+ if ("NotFoundException".equals(error.getErrorCode()) && !launching) {+ return Set.of();+ }+ if (Set.of("NotFoundException", "BadRequestException").contains(error.getErrorCode())) {+ throw new AwsException("InvalidParameterValue", "The Capacity Reservation group ARN is invalid.", 400);+ }+ throw error;+ }+ }+/*** Creates a Capacity Reservation, {@code active} on the create response and on the first* describe. Real AWS passes through {@code payment-pending}/{@code assessing} first;@@ -10547,6 +10661,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabif ("running".equals(instance.getState().getName()) && !instance.isSpot()&& "open".equals(instance.getCapacityReservationPreference())&& instance.getCapacityReservationTarget() == null+ && instance.getCapacityReservationGroupArn() == null&& !associated.contains(instance.getInstanceId())&& matchesCapacityReservation(instance, reservation)) {reservation.getInstanceIds().add(instance.getInstanceId());@@ -10688,9 +10803,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab"Value (" + count + ") for parameter Quantity is invalid.", 400);}validateHostSettings(autoPlacement, hostRecovery, hostMaintenance);- boolean zoneExists = Arrays.stream(MODELLED_ZONE_SUFFIXES)- .anyMatch(suffix -> availabilityZone.equals(region + suffix));- if (!zoneExists) {+ if (!isModelledAvailabilityZone(region, availabilityZone)) {throw new AwsException("InvalidParameterValue","Invalid availability zone: [" + availabilityZone + "]", 400);}diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java b/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.javaindex 6ae52211c5..632c2d446a 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java@@ -22,6 +22,7 @@ public class Instance {private String capacityReservationPlatform = "Linux/UNIX";private String capacityReservationPreference = "open";private String capacityReservationTarget;+ private String capacityReservationGroupArn;private Placement placement;private String subnetId;private String vpcId;@@ -128,6 +129,8 @@ public class Instance {public void setCapacityReservationPreference(String preference) { capacityReservationPreference = preference; }public String getCapacityReservationTarget() { return capacityReservationTarget; }public void setCapacityReservationTarget(String target) { capacityReservationTarget = target; }+ public String getCapacityReservationGroupArn() { return capacityReservationGroupArn; }+ public void setCapacityReservationGroupArn(String arn) { capacityReservationGroupArn = arn; }public boolean isSpot() { return spot; }public void setSpot(boolean spot) { this.spot = spot; }diff --git a/src/main/java/io/github/hectorvent/floci/services/resourcegroups/ResourceGroupsService.java b/src/main/java/io/github/hectorvent/floci/services/resourcegroups/ResourceGroupsService.javaindex f6e7dda192..49f8748d62 100644--- a/src/main/java/io/github/hectorvent/floci/services/resourcegroups/ResourceGroupsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/resourcegroups/ResourceGroupsService.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.resourcegroups;import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.node.JsonNodeFactory;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.AwsRegions;@@ -86,6 +87,15 @@ public class ResourceGroupsService implements Resettable {return selected(region, account, request);}+ public List<String> capacityReservationMembers(String region, String account, String groupArn) {+ // EC2 holds its capacity lock; GroupResources calls EC2 while holding this service's monitor.+ // Read the immutable storage snapshot without acquiring that monitor in the reverse order.+ if (groupArn == null || !groupArn.startsWith("arn:")) {+ throw bad("A capacity reservation group ARN is required.");+ }+ return selected(region, account, JsonNodeFactory.instance.objectNode().put("Group", groupArn)).members();+ }+public synchronized ResourceGroup delete(String region, String account, JsonNode request) {ResourceGroupsRequestShapes.check("DeleteGroup", request);ResourceGroup group = selected(region, account, request);diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityGroupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityGroupIntegrationTest.javanew file mode 100644index 0000000000..9112fae74d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityGroupIntegrationTest.java@@ -0,0 +1,137 @@+package io.github.hectorvent.floci.services.autoscaling;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.autoscaling.model.AsgOptionalFields;+import io.github.hectorvent.floci.services.autoscaling.model.AutoScalingGroup;+import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.services.ec2.model.CapacityReservation;+import io.github.hectorvent.floci.services.ec2.model.Instance;+import io.github.hectorvent.floci.services.ec2.model.LaunchTemplate;+import io.github.hectorvent.floci.services.ec2.model.LaunchTemplateData;+import io.github.hectorvent.floci.services.resourcegroups.ResourceGroupsService;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.RestAssured;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.ec2.Ec2Client;+import software.amazon.awssdk.services.ec2.model.DescribeInstancesResponse;++import java.net.URI;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.UUID;+import java.util.stream.Collectors;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;++@QuarkusTest+class AutoScalingCapacityGroupIntegrationTest {+ private static final String ACCOUNT = "753190420019";+ private static final String REGION = "us-east-1";++ @Inject+ Ec2Service ec2;++ @Inject+ ResourceGroupsService groups;++ @Inject+ AutoScalingService service;++ @Inject+ AutoScalingReconciler reconciler;++ @Inject+ ObjectMapper mapper;++ @Test+ void templateGroupMembersAreConsumedInTheirMatchingZones() {+ RequestScopes.runAs(ACCOUNT, REGION, () -> {+ String name = "capacity-group-" + UUID.randomUUID();+ List<CapacityReservation> capacities = new ArrayList<>();+ String groupArn = null;+ LaunchTemplate template = null;+ boolean asgCreated = false;+ try {+ for (String zone : List.of(REGION + "a", REGION + "b")) {+ capacities.add(ec2.createCapacityReservation(REGION, "t3.micro", "Linux/UNIX", zone,+ null, 1, null, null, null, null, null, "targeted", null, null));+ }+ groupArn = groups.create(REGION, ACCOUNT, mapper.valueToTree(Map.of(+ "Name", name, "Configuration", List.of(+ Map.of("Type", "AWS::EC2::CapacityReservationPool"),+ Map.of("Type", "AWS::ResourceGroups::Generic", "Parameters", List.of(+ Map.of("Name", "allowed-resource-types",+ "Values", List.of("AWS::EC2::CapacityReservation")))))))).arn();+ groups.add(REGION, ACCOUNT, mapper.valueToTree(Map.of("Group", groupArn, "ResourceArns",+ capacities.stream().map(CapacityReservation::getCapacityReservationArn).toList())));+ LaunchTemplateData data = new LaunchTemplateData();+ data.setImageId("ami-test");+ data.setInstanceType("t3.micro");+ LaunchTemplateData.CapacityReservationSpecification specification =+ new LaunchTemplateData.CapacityReservationSpecification();+ LaunchTemplateData.CapacityReservationTarget target = new LaunchTemplateData.CapacityReservationTarget();+ target.setCapacityReservationResourceGroupArn(groupArn);+ specification.setCapacityReservationTarget(target);+ data.setCapacityReservationSpecification(specification);+ template = ec2.createLaunchTemplate(REGION, name, data, List.of());+ AutoScalingGroup asg = service.createAutoScalingGroup(REGION, name, null,+ template.getLaunchTemplateId(), null, "$Latest", null, 0, 2, 2, 300,+ List.of(REGION + "a", REGION + "b"), List.of(), List.of(), List.of(), "EC2", 0,+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());+ asgCreated = true;+ reconciler.reconcile(asg);+ // The next pass promotes the newly launched mock instances to InService.+ reconciler.reconcile(asg);+ AutoScalingGroup current = service.describeAutoScalingGroups(REGION, List.of(name)).getFirst();+ assertEquals(2, current.getInstances().size());+ assertEquals(Set.of("InService"), current.getInstances().stream()+ .map(instance -> instance.getLifecycleState()).collect(Collectors.toSet()));+ List<Instance> instances = ec2.describeInstances(REGION, current.getInstances().stream()+ .map(instance -> instance.getInstanceId()).toList(), Map.of()).stream()+ .flatMap(reservation -> reservation.getInstances().stream()).toList();+ assertEquals(Set.of(REGION + "a", REGION + "b"), instances.stream()+ .map(instance -> instance.getPlacement().getAvailabilityZone()).collect(Collectors.toSet()));+ assertEquals(capacities.stream().map(CapacityReservation::getCapacityReservationId)+ .collect(Collectors.toSet()), instances.stream()+ .map(ec2::instanceCapacityReservationId).collect(Collectors.toSet()));+ try (Ec2Client client = Ec2Client.builder()+ .endpointOverride(URI.create("http://localhost:" + RestAssured.port))+ .httpClientBuilder(UrlConnectionHttpClient.builder()).region(Region.of(REGION))+ .credentialsProvider(StaticCredentialsProvider.create(+ AwsBasicCredentials.create(ACCOUNT, "test"))).build()) {+ DescribeInstancesResponse response = client.describeInstances(request -> request.instanceIds(+ instances.stream().map(Instance::getInstanceId).toList()));+ assertEquals(2, response.reservations().stream().mapToInt(item -> item.instances().size()).sum());+ String expectedGroup = groupArn;+ response.reservations().forEach(reservation -> reservation.instances().forEach(instance -> {+ assertNull(instance.capacityReservationSpecification().capacityReservationPreferenceAsString());+ assertEquals(expectedGroup, instance.capacityReservationSpecification()+ .capacityReservationTarget().capacityReservationResourceGroupArn());+ }));+ }+ } finally {+ if (asgCreated) {+ service.deleteAutoScalingGroup(REGION, name, true);+ }+ if (template != null) {+ ec2.deleteLaunchTemplate(REGION, template.getLaunchTemplateId(), null);+ }+ if (groupArn != null) {+ groups.delete(REGION, ACCOUNT, mapper.valueToTree(Map.of("Group", groupArn)));+ }+ for (CapacityReservation capacity : capacities) {+ ec2.cancelCapacityReservation(REGION, capacity.getCapacityReservationId());+ }+ }+ });+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionTest.javaindex 051b5e82d8..7bab574224 100644--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionTest.java@@ -67,7 +67,7 @@ class AutoScalingCapacitySuspensionTest {when(ec2.describeLaunchTemplateVersions(eq(REGION), eq("lt-original"), isNull(), anyList())).thenReturn(List.of(template));when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),- anyList(), any(), any(), any(), isNull(), eq(0), isNull(), isNull(), isNull(), isNull(),+ anyList(), any(), any(), any(), isNull(), eq(0), eq(REGION + "a"), isNull(), isNull(), isNull(),anyBoolean(), isNull(), isNull(), any(), eq(false), isNull(), isNull())).thenReturn(reservation("i-replacement"));}@@ -84,7 +84,7 @@ class AutoScalingCapacitySuspensionTest {if (paused) {verify(ec2, never()).terminateInstances(anyString(), anyList());verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),- any(), any(), anyList(), any(), any(), any(), isNull(), eq(0), isNull(), isNull(), isNull(), isNull(),+ any(), any(), anyList(), any(), any(), any(), isNull(), eq(0), eq(REGION + "a"), isNull(), isNull(), isNull(),anyBoolean(), isNull(), isNull(), any(), eq(false), isNull(), isNull());assertEquals(List.of("i-original"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());@@ -96,7 +96,7 @@ class AutoScalingCapacitySuspensionTest {verify(ec2).terminateInstances(REGION, List.of("i-original"));} else {verify(ec2).runInstances(eq(REGION), any(), any(), eq(1), eq(1), any(), anyList(),- any(), any(), anyList(), any(), any(), any(), isNull(), eq(0), isNull(), isNull(), isNull(), isNull(),+ any(), any(), anyList(), any(), any(), any(), isNull(), eq(0), eq(REGION + "a"), isNull(), isNull(), isNull(),anyBoolean(), isNull(), isNull(), any(), eq(false), isNull(), isNull());}}@@ -164,7 +164,7 @@ class AutoScalingCapacitySuspensionTest {clearInvocations(ec2);reconciler.reconcile(group);verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),- any(), any(), anyList(), any(), any(), any(), isNull(), eq(0), isNull(), isNull(), isNull(), isNull(),+ any(), any(), anyList(), any(), any(), any(), isNull(), eq(0), eq(REGION + "a"), isNull(), isNull(), isNull(),anyBoolean(), isNull(), isNull(), any(), eq(false), isNull(), isNull());verify(ec2, never()).terminateInstances(anyString(), anyList());} finally {@@ -187,7 +187,7 @@ class AutoScalingCapacitySuspensionTest {clearInvocations(ec2);reconciler.reconcile(group);verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),- any(), any(), anyList(), any(), any(), any(), isNull(), eq(0), isNull(), isNull(), isNull(), isNull(),+ any(), any(), anyList(), any(), any(), any(), isNull(), eq(0), eq(REGION + "a"), isNull(), isNull(), isNull(),anyBoolean(), isNull(), isNull(), any(), eq(false), isNull(), isNull());if (recreate) {assertTrue(service.describeAutoScalingGroups(REGION, List.of("suspended-group"))diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/LaunchConfigurationUserDataStorageTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/LaunchConfigurationUserDataStorageTest.javaindex 8ed585aa02..860a21299e 100644--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/LaunchConfigurationUserDataStorageTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/LaunchConfigurationUserDataStorageTest.java@@ -103,7 +103,7 @@ class LaunchConfigurationUserDataStorageTest {result.setInstances(List.of(launched));when(ec2.runInstances(eq(REGION), eq("ami-original"), eq("t3.micro"), eq(1), eq(1),isNull(), anyList(), isNull(), isNull(), anyList(), eq(text), isNull(), isNull(),- isNull(), eq(0), isNull(), isNull(), isNull(), eq(encoded),+ isNull(), eq(0), eq("us-east-1a"), isNull(), isNull(), eq(encoded),anyBoolean(), isNull(), isNull(), any(), eq(false), isNull(), isNull())).thenReturn(result);AutoScalingReconciler reconciler = new AutoScalingReconciler(service, ec2, mock(ElbV2Service.class));try {@@ -111,7 +111,7 @@ class LaunchConfigurationUserDataStorageTest {assertEquals(List.of("i-created"), group.getInstances().stream().map(i -> i.getInstanceId()).toList());verify(ec2).runInstances(eq(REGION), eq("ami-original"), eq("t3.micro"), eq(1), eq(1),isNull(), anyList(), isNull(), isNull(), anyList(), eq(text), isNull(), isNull(),- isNull(), eq(0), isNull(), isNull(), isNull(), eq(encoded),+ isNull(), eq(0), eq("us-east-1a"), isNull(), isNull(), eq(encoded),anyBoolean(), isNull(), isNull(), any(), eq(false), isNull(), isNull());assertEquals(origin.equals("legacy") ? null : Boolean.TRUE, restored.getUserDataBase64Encoded());assertEquals(origin.equals("legacy") ? input : encoded, restored.getUserData());diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapacityReservationAssociationServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapacityReservationAssociationServiceTest.javaindex 54a0201955..0353fae2e4 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapacityReservationAssociationServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapacityReservationAssociationServiceTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.ec2;import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.ServiceConfigAccess;@@ -9,6 +10,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.ec2.model.CapacityReservation;import io.github.hectorvent.floci.services.ec2.model.Instance;import io.github.hectorvent.floci.services.ec2.model.LaunchTemplateData;+import io.github.hectorvent.floci.services.resourcegroups.ResourceGroupsService;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.io.TempDir;import org.junit.jupiter.params.ParameterizedTest;@@ -16,8 +18,11 @@ import org.junit.jupiter.params.provider.ValueSource;import java.nio.file.Path;import java.time.Duration;+import java.util.Arrays;+import java.util.HashSet;import java.util.List;import java.util.Map;+import java.util.Set;import java.util.concurrent.CountDownLatch;import java.util.concurrent.ExecutionException;import java.util.concurrent.ExecutorService;@@ -28,23 +33,29 @@ import java.util.concurrent.atomic.AtomicReference;import static org.awaitility.Awaitility.await;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.clearInvocations;import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.doReturn;import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;import static org.mockito.Mockito.reset;import static org.mockito.Mockito.spy;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;class Ec2CapacityReservationAssociationServiceTest {private static final String ACCOUNT = "111111111111";private static final String REGION = "us-east-1";private static final String FILE = "ec2-capacity-reservations.json";+ private static final ObjectMapper MAPPER = new ObjectMapper();@TempDirPath directory;@@ -453,11 +464,238 @@ class Ec2CapacityReservationAssociationServiceTest {}}+ @Test+ void repeatedAssociationReadsDoNotRescanInstancesWithoutGroupTargets() {+ try (Fixture fixture = new Fixture("memory", directory)) {+ fixture.create(4);+ fixture.launch();+ fixture.service.capacityReservationAssociations(REGION);+ clearInvocations(fixture.instanceStore, fixture.groups);+ fixture.service.capacityReservationAssociations(REGION);+ fixture.service.capacityReservationAssociations(REGION);+ verify(fixture.instanceStore, never()).scan(any());+ verify(fixture.groups, never()).capacityReservationMembers(any(), any(), any());+ }+ }++ @Test+ void groupLaunchRegistersMembershipUnderTheReconciliationLock() {+ try (Fixture fixture = new Fixture("memory", directory)) {+ fixture.create(2);+ String group = fixture.group();+ fixture.service.capacityReservationAssociations(REGION);+ Map<String, Set<String>> index = fixture.service.capacityGroupInstanceIdsForTest();+ Set<String> ids = spy(new HashSet<String>());+ doAnswer(invocation -> {+ assertTrue(fixture.service.holdsCapacityReservationLockForTest(),+ "Registration must share the pruning lock");+ return invocation.callRealMethod();+ }).when(ids).add(any());+ index.put(ACCOUNT + "::" + REGION, ids);+ Instance launched = fixture.launchGroup(group, null, 1, 1).getFirst();+ assertEquals(Set.of(launched.getInstanceId()), ids);+ fixture.service.describeInstances(REGION, List.of(launched.getInstanceId()), Map.of());+ CapacityReservation added = fixture.create(1);+ fixture.groups.add(REGION, ACCOUNT, MAPPER.valueToTree(Map.of(+ "Group", group, "ResourceArns", List.of(added.getCapacityReservationArn()))));+ assertEquals(added.getCapacityReservationId(), fixture.service.instanceCapacityReservationId(launched));+ }+ }++ @Test+ void groupIndexPrunesTerminatedFallbacksWithoutAvailableCapacity() {+ try (Fixture fixture = new Fixture("memory", directory)) {+ CapacityReservation capacity = fixture.create(1);+ String group = fixture.group(capacity);+ fixture.service.capacityReservationAssociations(REGION);+ fixture.service.cancelCapacityReservation(REGION, capacity.getCapacityReservationId());+ Instance launched = fixture.launchGroup(group, null, 1, 1).getFirst();+ Map<String, Set<String>> index = fixture.service.capacityGroupInstanceIdsForTest();+ Set<String> ids = index.get(ACCOUNT + "::" + REGION);+ assertTrue(ids.contains(launched.getInstanceId()));+ fixture.service.terminateInstances(REGION, List.of(launched.getInstanceId()));+ clearInvocations(fixture.instanceStore, fixture.groups);+ fixture.service.capacityReservationAssociations(REGION);+ assertTrue(ids.isEmpty(), "Terminated fallback IDs must be pruned even without capacity");+ verify(fixture.instanceStore, never()).scan(any());+ verify(fixture.groups, never()).capacityReservationMembers(any(), any(), any());+ }+ }++ @Test+ void groupMembershipIsReadOncePerReconcileAndRefreshedAfterChanges() {+ try (Fixture fixture = new Fixture("memory", directory)) {+ fixture.create(4);+ fixture.service.capacityReservationAssociations(REGION);+ String group = fixture.group();+ List<Instance> launched = fixture.launchGroup(group, null, 2, 2);+ fixture.service.describeInstances(REGION, launched.stream().map(Instance::getInstanceId).toList(), Map.of());+ clearInvocations(fixture.instanceStore, fixture.groups);+ assertTrue(fixture.service.capacityReservationAssociations(REGION).isEmpty());+ verify(fixture.groups, times(1)).capacityReservationMembers(REGION, ACCOUNT, group);+ verify(fixture.instanceStore, never()).scan(any());+ CapacityReservation added = fixture.create(2);+ fixture.groups.add(REGION, ACCOUNT, MAPPER.valueToTree(+ Map.of("Group", group, "ResourceArns", List.of(added.getCapacityReservationArn()))));+ assertEquals(2, fixture.service.capacityReservationAssociations(REGION).size());+ assertEquals(0, fixture.describe(added).getAvailableInstanceCount());+ }+ }++ @Test+ void groupTargetUsesOnlyMatchingMembersAndReturnsReleasedCapacity() {+ try (Fixture fixture = new Fixture("memory", directory)) {+ CapacityReservation outside = fixture.create(3);+ CapacityReservation first = fixture.create(1);+ CapacityReservation second = fixture.create(1);+ fixture.service.modifyCapacityReservation(REGION, first.getCapacityReservationId(),+ null, null, null, "targeted");+ String group = fixture.group(first, second);+ List<Instance> launched = fixture.launchGroup(group, "capacity-reservations-only", 2, 3);+ assertEquals(2, launched.size());+ assertEquals(3, fixture.describe(outside).getAvailableInstanceCount());+ assertEquals(0, fixture.describe(first).getAvailableInstanceCount());+ assertEquals(0, fixture.describe(second).getAvailableInstanceCount());+ assertTrue(launched.stream().allMatch(instance -> group.equals(instance.getCapacityReservationGroupArn())));+ fixture.service.terminateInstances(REGION, launched.stream().map(Instance::getInstanceId).toList());+ assertEquals(1, fixture.describe(first).getAvailableInstanceCount());+ assertEquals(1, fixture.describe(second).getAvailableInstanceCount());+ }+ }++ @Test+ void groupFallbackNeverConsumesOutsideCapacityAndAdoptsNewMembers() {+ try (Fixture fixture = new Fixture("memory", directory)) {+ CapacityReservation outside = fixture.create(4);+ String group = fixture.group();+ List<Instance> launched = fixture.launchGroup(group, null, 2, 2);+ fixture.service.describeInstances(REGION, launched.stream().map(Instance::getInstanceId).toList(), Map.of());+ assertTrue(fixture.service.capacityReservationAssociations(REGION).isEmpty());+ CapacityReservation added = fixture.create(2);+ assertEquals(2, fixture.describe(added).getAvailableInstanceCount());+ fixture.groups.add(REGION, ACCOUNT, MAPPER.valueToTree(+ Map.of("Group", group, "ResourceArns", List.of(added.getCapacityReservationArn()))));+ assertEquals(0, fixture.describe(added).getAvailableInstanceCount());+ assertEquals(4, fixture.describe(outside).getAvailableInstanceCount());+ }+ }++ @Test+ void cancelledGroupCapacityMovesToAnotherMemberWithoutChangingTheTarget() {+ try (Fixture fixture = new Fixture("memory", directory)) {+ CapacityReservation first = fixture.create(1);+ CapacityReservation second = fixture.create(1);+ String group = fixture.group(first, second);+ Instance instance = fixture.launchGroup(group, null, 1, 1).getFirst();+ fixture.service.describeInstances(REGION, List.of(instance.getInstanceId()), Map.of());+ String acquired = fixture.service.instanceCapacityReservationId(instance);+ fixture.service.cancelCapacityReservation(REGION, acquired);+ String other = acquired.equals(first.getCapacityReservationId())+ ? second.getCapacityReservationId() : first.getCapacityReservationId();+ assertEquals(other, fixture.service.instanceCapacityReservationId(instance));+ assertEquals(group, instance.getCapacityReservationGroupArn());+ }+ }++ @Test+ void deletingGroupLeavesAcquiredCapacityUntilTheInstanceStops() {+ try (Fixture fixture = new Fixture("memory", directory)) {+ CapacityReservation capacity = fixture.create(1);+ String group = fixture.group(capacity);+ Instance instance = fixture.launchGroup(group, null, 1, 1).getFirst();+ fixture.groups.delete(REGION, ACCOUNT, MAPPER.valueToTree(Map.of("Group", group)));+ assertEquals(capacity.getCapacityReservationId(), fixture.service.instanceCapacityReservationId(instance));+ fixture.service.stopInstances(REGION, List.of(instance.getInstanceId()));+ assertEquals(1, fixture.describe(capacity).getAvailableInstanceCount());+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void recoveredFallbackInstancesAdoptNewGroupCapacityWithoutRepeatedScans(String mode) {+ String group;+ String instanceId;+ try (Fixture fixture = new Fixture(mode, directory)) {+ group = fixture.group();+ instanceId = fixture.launchGroup(group, null, 1, 1).getFirst().getInstanceId();+ fixture.service.describeInstances(REGION, List.of(instanceId), Map.of());+ }+ try (Fixture fixture = new Fixture(mode, directory)) {+ CapacityReservation capacity = fixture.create(1);+ fixture.groups.add(REGION, ACCOUNT, MAPPER.valueToTree(Map.of(+ "Group", group, "ResourceArns", List.of(capacity.getCapacityReservationArn()))));+ assertEquals(Map.of(instanceId, capacity.getCapacityReservationId()),+ fixture.service.capacityReservationAssociations(REGION));+ clearInvocations(fixture.instanceStore);+ assertEquals(Map.of(instanceId, capacity.getCapacityReservationId()),+ fixture.service.capacityReservationAssociations(REGION));+ verify(fixture.instanceStore, never()).scan(any());+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void groupTargetAndMembershipSurviveRecovery(String mode) {+ String group;+ String instanceId;+ String reservationId;+ try (Fixture fixture = new Fixture(mode, directory)) {+ CapacityReservation capacity = fixture.create(1);+ reservationId = capacity.getCapacityReservationId();+ group = fixture.group(capacity);+ instanceId = fixture.launchGroup(group, null, 1, 1).getFirst().getInstanceId();+ }+ try (Fixture fixture = new Fixture(mode, directory)) {+ Instance instance = fixture.service.describeInstances(REGION, List.of(instanceId), Map.of())+ .getFirst().getInstances().getFirst();+ assertEquals(group, instance.getCapacityReservationGroupArn());+ assertEquals(reservationId, fixture.service.instanceCapacityReservationId(instance));+ fixture.service.stopInstances(REGION, List.of(instanceId));+ assertEquals(null, fixture.service.instanceCapacityReservationId(instance));+ fixture.service.startInstances(REGION, List.of(instanceId));+ assertEquals(reservationId, fixture.service.instanceCapacityReservationId(instance));+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"missing", "foreign-account", "foreign-region", "none", "insufficient"})+ void invalidOrInsufficientGroupRequestDoesNotPublishInstances(String condition) {+ try (Fixture fixture = new Fixture("memory", directory)) {+ fixture.create(3);+ String group = fixture.group();+ group = switch (condition) {+ case "missing" -> group + "-absent";+ case "foreign-account" -> group.replace(ACCOUNT, "222222222222");+ case "foreign-region" -> group.replace(REGION, "us-west-2");+ default -> group;+ };+ String selected = group;+ assertThrows(AwsException.class, () -> fixture.launchGroup(selected,+ "none".equals(condition) ? "none" : "capacity-reservations-only", 1, 1));+ assertTrue(fixture.service.describeInstances(REGION, List.of(), Map.of()).isEmpty());+ assertTrue(fixture.service.describeVolumes(REGION, List.of(), Map.of()).isEmpty());+ }+ }++ @Test+ void groupLaunchDoesNotWaitForTheResourceGroupsOperationMonitor() throws Exception {+ try (Fixture fixture = new Fixture("memory", directory);+ ExecutorService executor = Executors.newSingleThreadExecutor()) {+ String group = fixture.group(fixture.create(1));+ synchronized (fixture.groups) {+ Future<List<Instance>> launched = executor.submit(() -> fixture.launchGroup(group, null, 1, 1));+ assertEquals(1, launched.get(5, TimeUnit.SECONDS).size());+ }+ }+ }+private static final class Fixture implements AutoCloseable {private final EmulatorConfig config;private final StorageFactory factory;private final AccountAwareStorageBackend<CapacityReservation> store;private final Ec2Service service;+ private final AccountAwareStorageBackend<Instance> instanceStore;+ private final ResourceGroupsService groups;private Fixture(String mode, Path path) {config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);@@ -470,13 +708,49 @@ class Ec2CapacityReservationAssociationServiceTest {ServiceConfigAccess access = mock(ServiceConfigAccess.class);when(access.storageMode("ec2")).thenReturn(mode);when(access.storageFlushInterval("ec2")).thenReturn(3600000L);+ when(access.storageMode("resource-groups")).thenReturn(mode);+ when(access.storageFlushInterval("resource-groups")).thenReturn(3600000L);factory = new StorageFactory(config, access);store = spy(factory.create("ec2", FILE, new TypeReference<Map<String, CapacityReservation>>() {}));+ instanceStore = spy(factory.create("ec2", "ec2-instances.json",+ new TypeReference<Map<String, Instance>>() {}));StorageFactory serviceFactory = spy(factory);+ doReturn(instanceStore).when(serviceFactory).create(eq("ec2"), eq("ec2-instances.json"), any());doReturn(store).when(serviceFactory).create(eq("ec2"), eq(FILE), any());Ec2ImageCatalog catalog = new Ec2ImageCatalog();service = spy(new Ec2Service(config, null, null, new AmiImageResolver(catalog),catalog, new Ec2InstanceTypeCatalog(), serviceFactory));+ groups = spy(new ResourceGroupsService(factory, service));+ // CDI Instance clashes with the EC2 Instance model.+ service.resourceGroupsServices = mock(jakarta.enterprise.inject.Instance.class);+ when(service.resourceGroupsServices.isResolvable()).thenReturn(true);+ when(service.resourceGroupsServices.get()).thenReturn(groups);+ }++ private String group(CapacityReservation... reservations) {+ String arn = groups.create(REGION, ACCOUNT, MAPPER.valueToTree(Map.of(+ "Name", "capacity-pool", "Configuration", List.of(+ Map.of("Type", "AWS::EC2::CapacityReservationPool"),+ Map.of("Type", "AWS::ResourceGroups::Generic", "Parameters",+ List.of(Map.of("Name", "allowed-resource-types",+ "Values", List.of("AWS::EC2::CapacityReservation")))))))).arn();+ if (reservations.length > 0) {+ groups.add(REGION, ACCOUNT, MAPPER.valueToTree(Map.of("Group", arn, "ResourceArns",+ Arrays.stream(reservations).map(CapacityReservation::getCapacityReservationArn).toList())));+ }+ return arn;+ }++ private List<Instance> launchGroup(String group, String preference, int minimum, int maximum) {+ LaunchTemplateData.CapacityReservationSpecification specification =+ new LaunchTemplateData.CapacityReservationSpecification();+ specification.setCapacityReservationPreference(preference);+ LaunchTemplateData.CapacityReservationTarget target = new LaunchTemplateData.CapacityReservationTarget();+ target.setCapacityReservationResourceGroupArn(group);+ specification.setCapacityReservationTarget(target);+ return service.runInstances(REGION, "ami-test", "t3.micro", minimum, maximum,+ null, List.of(), null, null, List.of(), null, null, null, null, 0, null, null,+ null, null, false, null, null, null, false, null, specification).getInstances();}private Instance launch() {diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2TargetedCapacityErrorIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2TargetedCapacityErrorIntegrationTest.javaindex 056c818a7c..90a4d943a1 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2TargetedCapacityErrorIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2TargetedCapacityErrorIntegrationTest.java@@ -3,6 +3,8 @@ package io.github.hectorvent.floci.services.ec2;import io.quarkus.test.junit.QuarkusTest;import io.restassured.RestAssured;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;@@ -19,18 +21,33 @@ import java.util.ArrayList;import java.util.List;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;@QuarkusTestclass Ec2TargetedCapacityErrorIntegrationTest {+ @ParameterizedTest+ @ValueSource(strings = {"open", "none"})+ void untargetedInstancesRetainTheirPreference(String preference) {+ try (Ec2Client client = ec2Client("741623400003", Region.US_EAST_1)) {+ String id = client.runInstances(request -> request.imageId("ami-test").instanceType("t3.micro")+ .minCount(1).maxCount(1).capacityReservationSpecification(specification ->+ specification.capacityReservationPreference(preference)))+ .instances().getFirst().instanceId();+ try {+ assertEquals(preference, client.describeInstances(request -> request.instanceIds(id))+ .reservations().getFirst().instances().getFirst().capacityReservationSpecification()+ .capacityReservationPreferenceAsString());+ } finally {+ client.terminateInstances(request -> request.instanceIds(id));+ }+ }+ }+@Testvoid sdkReceivesCapacityErrorAndCanRetryAfterRelease() {- try (Ec2Client client = Ec2Client.builder()- .endpointOverride(URI.create("http://localhost:" + RestAssured.port))- .httpClientBuilder(UrlConnectionHttpClient.builder()).region(Region.US_EAST_1)- .credentialsProvider(StaticCredentialsProvider.create(- AwsBasicCredentials.create("741623400002", "test"))).build()) {+ try (Ec2Client client = ec2Client("741623400002", Region.US_EAST_1)) {Subnet subnet = client.describeSubnets().subnets().getFirst();String capacityId = client.createCapacityReservation(request -> request.instanceType("t3.micro").instancePlatform("Linux/UNIX").availabilityZone(subnet.availabilityZone())@@ -45,6 +62,9 @@ class Ec2TargetedCapacityErrorIntegrationTest {.capacityReservationTarget(CapacityReservationTarget.builder().capacityReservationId(capacityId).build()).build()).build();instances.add(client.runInstances(request).instances().getFirst().instanceId());+ assertNull(client.describeInstances(describe -> describe.instanceIds(instances)).reservations()+ .getFirst().instances().getFirst().capacityReservationSpecification()+ .capacityReservationPreferenceAsString());Ec2Exception failure = assertThrows(Ec2Exception.class, () -> client.runInstances(request));assertEquals("ReservationCapacityExceeded", failure.awsErrorDetails().errorCode());assertEquals(400, failure.statusCode());@@ -62,4 +82,12 @@ class Ec2TargetedCapacityErrorIntegrationTest {}}}++ private Ec2Client ec2Client(String account, Region region) {+ return Ec2Client.builder()+ .endpointOverride(URI.create("http://localhost:" + RestAssured.port))+ .httpClientBuilder(UrlConnectionHttpClient.builder()).region(region)+ .credentialsProvider(StaticCredentialsProvider.create(+ AwsBasicCredentials.create(account, "test"))).build();+ }}