Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2): protect subnets and VPCs that load balancers use (#5663)
Jackson Pricee52016617026
diff --git a/compatibility-tests/sdk-test-awscli/test/cloudformation.bats b/compatibility-tests/sdk-test-awscli/test/cloudformation.batsindex ace02ed7bf..4e0ad28012 100644--- a/compatibility-tests/sdk-test-awscli/test/cloudformation.bats+++ b/compatibility-tests/sdk-test-awscli/test/cloudformation.bats@@ -5,9 +5,15 @@ setup() {load 'test_helper/common-setup'STACK_NAME="bats-cfn-stack-$(unique_name)"TEMPLATE_FILE=$(mktemp /tmp/cfn-bats-XXXXXX.yaml)+ LB_ARN=""}teardown() {+ # A load balancer a test put on the stack's subnets goes first: EC2 refuses to delete a subnet+ # a load balancer still uses, so delete-stack could not remove the subnets.+ if [ -n "$LB_ARN" ]; then+ aws_cmd elbv2 delete-load-balancer --load-balancer-arn "$LB_ARN" >/dev/null 2>&1 || true+ fiaws_cmd cloudformation delete-stack --stack-name "$STACK_NAME" >/dev/null 2>&1 || true[ -n "$TEMPLATE_FILE" ] && rm -f "$TEMPLATE_FILE"}@@ -197,6 +203,7 @@ EOFassert_successlocal lb_arnlb_arn=$(json_get "$output" '.LoadBalancers[0].LoadBalancerArn')+ LB_ARN="$lb_arn"[ -n "$lb_arn" ][[ "$lb_arn" == *":loadbalancer/"* ]]}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/Ec2ResourceLifecycleTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/Ec2ResourceLifecycleTest.javanew file mode 100644index 0000000000..cc1dbc1d64--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/Ec2ResourceLifecycleTest.java@@ -0,0 +1,56 @@+package com.floci.test;++import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.ec2.Ec2Client;+import software.amazon.awssdk.services.ec2.model.Ec2Exception;+import software.amazon.awssdk.services.elasticloadbalancingv2.ElasticLoadBalancingV2Client;+import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerSchemeEnum;+import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerTypeEnum;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++@DisplayName("EC2 subnet and VPC lifecycle dependencies")+class Ec2ResourceLifecycleTest {++ @Test+ @DisplayName("DeleteSubnet and DeleteVpc are refused while a load balancer uses the subnet")+ void subnetAndVpcAreRefusedWhileALoadBalancerUsesThem() {+ String region = TestFixtures.region().id();+ try (Ec2Client ec2 = TestFixtures.ec2Client();+ ElasticLoadBalancingV2Client elb = TestFixtures.elbV2Client()) {+ String vpcId = ec2.createVpc(r -> r.cidrBlock("10.94.0.0/16")).vpc().vpcId();+ String first = ec2.createSubnet(r -> r.vpcId(vpcId).cidrBlock("10.94.1.0/24")+ .availabilityZone(region + "a")).subnet().subnetId();+ String second = ec2.createSubnet(r -> r.vpcId(vpcId).cidrBlock("10.94.2.0/24")+ .availabilityZone(region + "b")).subnet().subnetId();+ String loadBalancerArn = elb.createLoadBalancer(r -> r.name(TestFixtures.uniqueName("lifecycle"))+ .type(LoadBalancerTypeEnum.APPLICATION).scheme(LoadBalancerSchemeEnum.INTERNAL)+ .subnets(first, second)).loadBalancers().get(0).loadBalancerArn();+ try {+ assertThatThrownBy(() -> ec2.deleteSubnet(r -> r.subnetId(first)))+ .isInstanceOfSatisfying(Ec2Exception.class, e ->+ assertThat(e.awsErrorDetails().errorCode()).isEqualTo("DependencyViolation"));+ assertThatThrownBy(() -> ec2.deleteVpc(r -> r.vpcId(vpcId)))+ .isInstanceOfSatisfying(Ec2Exception.class, e ->+ assertThat(e.awsErrorDetails().errorCode()).isEqualTo("DependencyViolation"));+ } finally {+ // Cleaned up even when an assertion fails, so nothing leaks into the rest of the run.+ // A refusal that did not happen has already deleted the subnet, hence quietly.+ elb.deleteLoadBalancer(r -> r.loadBalancerArn(loadBalancerArn));+ quietly(() -> ec2.deleteSubnet(r -> r.subnetId(first)));+ quietly(() -> ec2.deleteSubnet(r -> r.subnetId(second)));+ quietly(() -> ec2.deleteVpc(r -> r.vpcId(vpcId)));+ }+ }+ }++ private static void quietly(Runnable cleanup) {+ try {+ cleanup.run();+ } catch (Ec2Exception ignored) {+ // already gone+ }+ }+}diff --git a/docs/services/ec2.md b/docs/services/ec2.mdindex 3ff026f98e..6608d8897b 100644--- a/docs/services/ec2.md+++ b/docs/services/ec2.md@@ -227,7 +227,7 @@ Floci seeds the following resources on first use in each region so Terraform, th|--------|-------------|| CreateVpc | Creates a VPC with the requested CIDR block. `InstanceTenancy` accepts `default` or `dedicated`; `host` is rejected with `InvalidParameterValue`, as on AWS. || DescribeVpcs | Lists or returns stored VPCs. |-| DeleteVpc | Deletes a VPC from the local EC2 store, together with its default security group and rules, main route table and default network ACL. Fails with `DependencyViolation` while the VPC still has a subnet, a security group, route table or network ACL other than those defaults, a VPC endpoint, an attached internet gateway or egress-only internet gateway, a transit gateway attachment, or anything still holding a network interface in one of its subnets: an instance that is not terminated, a NAT gateway that is not deleted or failed, a standalone network interface, or an interface endpoint. |+| DeleteVpc | Deletes a VPC from the local EC2 store, together with its default security group and rules, main route table and default network ACL. Fails with `DependencyViolation` while the VPC still has a subnet, a security group, route table or network ACL other than those defaults, a VPC endpoint, an attached internet gateway or egress-only internet gateway, a transit gateway attachment, an Elastic Load Balancing v2 load balancer, or anything still holding a network interface in one of its subnets: an instance that is not terminated, a NAT gateway that is not deleted or failed, a standalone network interface, or an interface endpoint. || ModifyVpcAttribute | Updates supported VPC attributes. || DescribeVpcAttribute | Returns a supported VPC attribute. || DescribeVpcEndpointServices | Lists common AWS interface endpoint services and S3 (gateway and interface) in every availability zone, named per partition (see [Partitions](../configuration/partitions.md)). A `ServiceName` filter is echoed back as asked. |@@ -249,7 +249,7 @@ private gateway lifecycles are not modelled.|--------|-------------|| CreateSubnet | Creates a subnet in a VPC. || DescribeSubnets | Lists or returns stored subnets. |-| DeleteSubnet | Deletes a subnet from the local EC2 store. Fails with `DependencyViolation` while the subnet still has a transit gateway attachment or anything holding a network interface in it: an instance that is not terminated, a NAT gateway that is not deleted or failed, a standalone network interface, or an interface endpoint. |+| DeleteSubnet | Deletes a subnet from the local EC2 store. Fails with `DependencyViolation` while the subnet still has a transit gateway attachment, an Elastic Load Balancing v2 load balancer, or anything holding a network interface in it: an instance that is not terminated, a NAT gateway that is not deleted or failed, a standalone network interface, or an interface endpoint. || ModifySubnetAttribute | Updates supported subnet attributes. |### Security Groupsdiff --git a/docs/services/elb.md b/docs/services/elb.mdindex d493277c07..78064a5a7c 100644--- a/docs/services/elb.md+++ b/docs/services/elb.md@@ -88,6 +88,7 @@ HTTPS listeners require a default certificate. Supply one when creating a listen- Load balancer, target group, listener, rule, and tag state is persisted through Floci storage and rebuilt on service startup.- Load balancers are created in `active` state.+- EC2 refuses deletion of subnets and VPCs still referenced by load balancers with `DependencyViolation`. These account- and region-scoped dependencies follow persisted load balancer state; changing subnets or deleting the load balancer releases its old references. A CloudFormation update that changes `Subnets` or `SubnetMappings` moves the load balancer the way `SetSubnets` does, and a rolled-back update moves it back. An independent target group does not prevent subnet or VPC deletion.- HTTP listener sockets are preserved when listener actions change and are restarted only when socket-level settings such as port change.- Instance targets are resolved through EC2 instance private addresses so local load balancer traffic can reach containers.- Target health starts in `initial` state with reason `Elb.RegistrationInProgress` and is updated by Floci's health checker when monitoring is active.diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.javaindex 71a0a8c213..3afc5c2635 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java@@ -2276,11 +2276,21 @@ public class CloudFormationService implements ResourceProvider {// resources are deleted during post-update cleanup.persistStack(stack);}- // Both cleanup paths feed the single final status/reason writer.- List<UpdateCleanupFailure> cleanupFailures =- new ArrayList<>(deleteRemovedOrConditionFalseResources(- stack, resources, conditions, region));- cleanupFailures.addAll(finishCommittedResourceCleanup(stack, region));+ // Both cleanup paths feed the single final status/reason writer. A removed resource+ // whose delete failed is tried once more after the replacement cleanup, as an entity+ // a replacement displaced can still use it: an old load balancer on a subnet the same+ // update removed.+ List<StackResource> retried = deleteRemovedOrConditionFalseResources(+ stack, resources, conditions, region);+ List<UpdateCleanupFailure> displacedFailures = finishCommittedResourceCleanup(stack, region);+ List<UpdateCleanupFailure> cleanupFailures = new ArrayList<>();+ for (StackResource resource : retried) {+ UpdateCleanupFailure failure = deleteRemovedResource(stack, resource, region, true);+ if (failure != null) {+ cleanupFailures.add(failure);+ }+ }+ cleanupFailures.addAll(displacedFailures);finishCommittedStackUpdate(stack, cleanupFailures);return;}@@ -2802,8 +2812,8 @@ public class CloudFormationService implements ResourceProvider {String region) {// Dependents first, in the template's order rather than the map's, as the update cleanup// walks it: a resource the update added sits after the resources that depend on it, so- // reversing the map would delete a subnet the update added before an instance or NAT- // gateway the update put on it, and EC2 refuses that delete.+ // reversing the map would delete a subnet the update moved a load balancer onto before the+ // load balancer is moved back, and EC2 refuses that delete.List<StackResource> resources = resourcesInCreationOrder(stack, region);Collections.reverse(resources);List<String> failures = new ArrayList<>();@@ -3066,16 +3076,21 @@ public class CloudFormationService implements ResourceProvider {* as {@code DELETE_FAILED}, so a later {@code DeleteStack} or {@code UpdateStack} can retry the cleanup* instead of orphaning the backing resource.*- * @return one {@link UpdateCleanupFailure} per resource whose physical deletion failed+ * <p>Dependents go first, in the order {@link #resourcesInCreationOrder} gives rather than the map's:+ * a subnet an earlier update added sits after the load balancer it moved onto it, and the template+ * that named what a removed resource uses is gone. A failed delete is not reported here, since the+ * caller tries it again after the replacement cleanup.+ *+ * @return the resources whose physical deletion failed, for {@link #deleteRemovedResource} to try again*/- private List<UpdateCleanupFailure> deleteRemovedOrConditionFalseResources(Stack stack, JsonNode resources,+ private List<StackResource> deleteRemovedOrConditionFalseResources(Stack stack, JsonNode resources,Map<String, Boolean> conditions, String region) {if (!resources.isObject()) {return List.of();}- List<UpdateCleanupFailure> failures = new ArrayList<>();- List<StackResource> ordered = new ArrayList<>(stack.resourcesSnapshot().values());+ List<StackResource> failed = new ArrayList<>();+ List<StackResource> ordered = resourcesInCreationOrder(stack, region);Collections.reverse(ordered);for (StackResource resource : ordered) {JsonNode resDef = resources.get(resource.getLogicalId());@@ -3093,31 +3108,49 @@ public class CloudFormationService implements ResourceProvider {addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_IN_PROGRESS", null);- try {- deleteResourcePhysically(resource, region, ownerAccount(stack));- addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),- resource.getResourceType(), "DELETE_COMPLETE", null);- stack.getResources().remove(resource.getLogicalId());- } catch (Exception e) {- String reason = e.getMessage() != null- ? e.getMessage()- : "Resource deletion failed during update cleanup";- failures.add(new UpdateCleanupFailure(- resource.getLogicalId(), resource.getPhysicalId(), reason));- // Keep the resource under stack management as DELETE_FAILED. Removing it here would- // drop it from DescribeStackResources while the backing resource still exists,- // orphaning it and preventing a later DeleteStack/UpdateStack from retrying cleanup.- resource.setStatus("DELETE_FAILED");- resource.setStatusReason(reason);- addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),- resource.getResourceType(), "DELETE_FAILED", reason);- LOG.warnv("Failed to delete removed or condition-disabled {0} ({1}) in stack {2}: {3}",- resource.getResourceType(), resource.getPhysicalId(),- stack.getStackName(), reason);+ if (deleteRemovedResource(stack, resource, region, false) != null) {+ failed.add(resource);}}- return failures;+ return failed;+ }++ /**+ * Deletes one removed or condition-disabled resource, and reports a failure only on the last+ * attempt.+ *+ * @return the failure, or null once the resource is deleted+ */+ private UpdateCleanupFailure deleteRemovedResource(Stack stack, StackResource resource, String region,+ boolean lastAttempt) {+ try {+ deleteResourcePhysically(resource, region, ownerAccount(stack));+ addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),+ resource.getResourceType(), "DELETE_COMPLETE", null);+ stack.getResources().remove(resource.getLogicalId());+ return null;+ } catch (Exception e) {+ String reason = e.getMessage() != null+ ? e.getMessage()+ : "Resource deletion failed during update cleanup";+ UpdateCleanupFailure failure = new UpdateCleanupFailure(+ resource.getLogicalId(), resource.getPhysicalId(), reason);+ if (!lastAttempt) {+ return failure;+ }+ // Keep the resource under stack management as DELETE_FAILED. Removing it here would+ // drop it from DescribeStackResources while the backing resource still exists,+ // orphaning it and preventing a later DeleteStack/UpdateStack from retrying cleanup.+ resource.setStatus("DELETE_FAILED");+ resource.setStatusReason(reason);+ addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),+ resource.getResourceType(), "DELETE_FAILED", reason);+ LOG.warnv("Failed to delete removed or condition-disabled {0} ({1}) in stack {2}: {3}",+ resource.getResourceType(), resource.getPhysicalId(),+ stack.getStackName(), reason);+ return failure;+ }}/**diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisioner.javaindex 6726e5bd95..37295343b1 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisioner.java@@ -36,6 +36,8 @@ public class ElbV2CfnProvisioner implements CfnResourceProvisioner {private static final String TARGET_GROUP = "AWS::ElasticLoadBalancingV2::TargetGroup";private static final String LISTENER = "AWS::ElasticLoadBalancingV2::Listener";private static final String LISTENER_RULE = "AWS::ElasticLoadBalancingV2::ListenerRule";+ /** Prefixes one uses attribute per subnet a load balancer is on, numbered from 1. */+ private static final String USED_SUBNET_ATTR_PREFIX = CfnRollback.USES_ATTR_PREFIX + "Subnet.";private final ElbV2Service elbV2Service;@@ -141,8 +143,9 @@ public class ElbV2CfnProvisioner implements CfnResourceProvisioner {return true;}String arn = resource.getPhysicalId();- elbV2Service.setSubnets(AwsArnUtils.parse(arn).region(), arn,- prior.isEmpty() ? List.of() : List.of(prior.split(",")));+ List<String> priorSubnets = prior.isEmpty() ? List.of() : List.of(prior.split(","));+ elbV2Service.setSubnets(AwsArnUtils.parse(arn).region(), arn, priorSubnets);+ recordUsedSubnets(resource, priorSubnets);resource.getAttributes().remove(CfnRollback.LOAD_BALANCER_SUBNETS_UPDATE_SNAPSHOT_ATTR);return true;}@@ -165,17 +168,19 @@ public class ElbV2CfnProvisioner implements CfnResourceProvisioner {Map<String, String> tags = ctx.resolveTags(props, "Tags");LoadBalancer lb;+ List<String> onSubnets;try {lb = elbV2Service.createLoadBalancer(ctx.region(), name, scheme, type, ipAddressType,subnets, securityGroups, tags);+ onSubnets = subnetIds(lb);} catch (AwsException e) {if (!"DuplicateLoadBalancerName".equals(e.getErrorCode())) {throw e;}lb = elbV2Service.describeLoadBalancers(ctx.region(), null, List.of(name), null, null).get(0);- if (ctx.reusesPriorEntity(lb.getLoadBalancerArn())) {- moveToTemplateSubnets(r, lb, subnets, ctx);- }+ onSubnets = ctx.reusesPriorEntity(lb.getLoadBalancerArn())+ ? moveToTemplateSubnets(r, lb, subnets, ctx)+ : subnetIds(lb);}r.setPhysicalId(lb.getLoadBalancerArn());@@ -184,6 +189,23 @@ public class ElbV2CfnProvisioner implements CfnResourceProvisioner {r.getAttributes().put("CanonicalHostedZoneID", lb.getCanonicalHostedZoneId());r.getAttributes().put("LoadBalancerName", lb.getLoadBalancerName());r.getAttributes().put("LoadBalancerFullName", loadBalancerFullName(lb.getLoadBalancerArn()));+ recordUsedSubnets(r, onSubnets);+ }++ private static List<String> subnetIds(LoadBalancer lb) {+ return lb.getAvailabilityZones().stream().map(AvailabilityZone::getSubnetId).toList();+ }++ /**+ * Records the subnets the balancer is on as uses attributes. EC2 refuses to delete a subnet a+ * load balancer still uses, and once the template no longer names the balancer, as when an+ * update removes it together with a subnet it is on, these are what orders it before them.+ */+ private static void recordUsedSubnets(StackResource r, List<String> subnetIds) {+ r.getAttributes().keySet().removeIf(key -> key.startsWith(USED_SUBNET_ATTR_PREFIX));+ for (int i = 0; i < subnetIds.size(); i++) {+ r.getAttributes().put(USED_SUBNET_ATTR_PREFIX + (i + 1), subnetIds.get(i));+ }}/**@@ -222,18 +244,21 @@ public class ElbV2CfnProvisioner implements CfnResourceProvisioner {}/**- * {@code Subnets} is not create-only, so an update that keeps the balancer moves it onto the- * template's subnets in place, as SetSubnets does, rather than leaving it on a subnet the template- * dropped or replaced and the update cleanup then deletes. The prior list is kept for a rollback.+ * {@code Subnets} and {@code SubnetMappings} are not create-only, so an update that keeps the+ * balancer moves it onto the template's subnets in place, as SetSubnets does. That also releases+ * the subnets it leaves, which EC2 refuses to delete while a load balancer still uses them, so the+ * update cleanup can delete a subnet the template dropped or replaced. The prior list is kept for+ * a rollback. Returns the subnets the balancer is on afterwards.*/- private void moveToTemplateSubnets(StackResource r, LoadBalancer lb, List<String> subnets,- ProvisionContext ctx) {- List<String> current = lb.getAvailabilityZones().stream().map(AvailabilityZone::getSubnetId).toList();+ private List<String> moveToTemplateSubnets(StackResource r, LoadBalancer lb, List<String> subnets,+ ProvisionContext ctx) {+ List<String> current = subnetIds(lb);if (subnets.isEmpty() || Set.copyOf(subnets).equals(Set.copyOf(current))) {- return;+ return current;}elbV2Service.setSubnets(ctx.region(), lb.getLoadBalancerArn(), subnets);r.getAttributes().put(CfnRollback.LOAD_BALANCER_SUBNETS_UPDATE_SNAPSHOT_ATTR, String.join(",", current));+ return subnets;}private void provisionTargetGroup(StackResource r, JsonNode props, ProvisionContext ctx) {diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.javaindex 286f58a3b8..ad074f7a0b 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java@@ -135,6 +135,7 @@ import java.util.concurrent.atomic.AtomicInteger;import java.util.concurrent.locks.ReentrantLock;import java.util.function.BiPredicate;import java.util.function.Function;+import java.util.function.Predicate;import java.util.function.Supplier;import java.util.regex.Pattern;import java.util.stream.Collectors;@@ -313,6 +314,9 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabprivate final List<VpcRouteTableListener> routeTableListeners = new CopyOnWriteArrayList<>();private jakarta.enterprise.inject.Instance<Ec2InstanceLaunchListener> instanceLaunchListenersInstance;private final List<Ec2InstanceLaunchListener> instanceLaunchListeners = new CopyOnWriteArrayList<>();+ // Fully qualified to tell the CDI Instance from the EC2 model Instance. Only the CDI constructor+ // receives it; null on the hand-built constructors, which reads as "nothing else uses a subnet".+ private jakarta.enterprise.inject.Instance<VpcDependencyProvider> vpcDependencyProviders;public void addRouteTableListener(VpcRouteTableListener listener) {if (listener != null && !this.routeTableListeners.contains(listener)) {@@ -444,7 +448,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, ResettabEc2VolumeBlockDeviceManager volumeBlockDeviceManager,jakarta.enterprise.inject.Instance<VpcRouteTableListener> routeTableListenersInstance,jakarta.enterprise.inject.Instance<Ec2InstanceLaunchListener> instanceLaunchListenersInstance,- Ec2EbsEncryptionService ebsEncryptionService) {+ Ec2EbsEncryptionService ebsEncryptionService,+ jakarta.enterprise.inject.Instance<VpcDependencyProvider> vpcDependencyProviders) {this(config, containerManager, portForwardManager, amiImageResolver, imageCatalog,instanceTypeCatalog, storageFactory, requestContextInstance, iamService, volumeBlockDeviceManager);this.vpcNetworkManager = vpcNetworkManager;@@ -452,6 +457,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabthis.routeTableListenersInstance = routeTableListenersInstance;this.instanceLaunchListenersInstance = instanceLaunchListenersInstance;this.ebsEncryptionService = ebsEncryptionService;+ this.vpcDependencyProviders = vpcDependencyProviders;}public Ec2Service(EmulatorConfig config, Ec2ContainerManager containerManager,@@ -2439,6 +2445,32 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabreturn lockFor(key(region, "transit-gateway-attachments"));}+ /**+ * Runs an action that checks subnets and then records a reference to them, holding the lock+ * {@link #deleteSubnet} and {@link #deleteVpc} take while they ask each+ * {@link VpcDependencyProvider} whether anything still uses what they delete. Without it the+ * check and the write straddle the delete: the delete sees no reference because the referring+ * resource is not saved yet, removes the subnet, and the resource is then saved pointing at a+ * subnet that is gone.+ *+ * <p>Safe to call from inside a provider's own monitor, because the ordering only ever runs one+ * way. The deletes hold this lock while calling the providers, and their dependency checks are+ * plain reads over the provider's own store that take no provider monitor, so EC2 never waits on+ * a provider while a provider waits on EC2.+ */+ public <T> T supplyWithSubnetReferencesHeld(String region, Supplier<T> action) {+ synchronized (attachmentTopologyLock(region)) {+ return action.get();+ }+ }++ /** {@link #supplyWithSubnetReferencesHeld} for an action with no result. */+ public void runWithSubnetReferencesHeld(String region, Runnable action) {+ synchronized (attachmentTopologyLock(region)) {+ action.run();+ }+ }+private TransitGatewayVpcAttachmentOptions resolveAttachmentOptions(TransitGatewayVpcAttachmentOptions requested) {TransitGatewayVpcAttachmentOptions options = new TransitGatewayVpcAttachmentOptions();@@ -5346,6 +5378,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, ResettabrequireNoTransitGatewayAttachment(region,attachment -> vpcId.equals(attachment.getVpcId()),"The vpc '" + vpcId + "' has dependencies and cannot be deleted.");+ requireNoProviderDependency(provider -> provider.hasVpcDependency(region, vpcId),+ "The vpc '" + vpcId + "' has dependencies and cannot be deleted.");requireNoVpcDependents(region, vpcId);deleteVpcDefaultResources(region, vpcId);vpcs.delete(key(region, vpcId));@@ -6566,6 +6600,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab"The subnet '" + subnetId + "' has dependencies and cannot be deleted.");requireNoSubnetResidents(region, Set.of(subnetId),"The subnet '" + subnetId + "' has dependencies and cannot be deleted.");+ requireNoProviderDependency(provider -> provider.hasSubnetDependency(region, subnetId),+ "The subnet '" + subnetId + "' has dependencies and cannot be deleted.");subnets.delete(key(region, subnetId));}if (vpcNetworkManager != null) {@@ -6588,6 +6624,18 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab}}+ /**+ * Refuses to delete a subnet or VPC that another service still uses, such as a load balancer.+ * Called under {@link #attachmentTopologyLock}, which those services hold through+ * {@link #supplyWithSubnetReferencesHeld} while they record a reference, so no reference can+ * appear between this check and the delete.+ */+ private void requireNoProviderDependency(Predicate<VpcDependencyProvider> dependsOnIt, String message) {+ if (vpcDependencyProviders != null && vpcDependencyProviders.stream().anyMatch(dependsOnIt)) {+ throw new AwsException("DependencyViolation", message, 400);+ }+ }+public void modifySubnetAttribute(String region, String subnetId, String attribute, String value) {ensureDefaultResources(region);Subnet subnet = requireSubnet(region, subnetId);diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/VpcDependencyProvider.java b/src/main/java/io/github/hectorvent/floci/services/ec2/VpcDependencyProvider.javanew file mode 100644index 0000000000..564784c7fd--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/VpcDependencyProvider.java@@ -0,0 +1,16 @@+package io.github.hectorvent.floci.services.ec2;++/**+ * Implemented by services whose resources reference EC2 subnets or VPCs, so DeleteSubnet and+ * DeleteVpc refuse to remove one that is still in use. Answers come from the owning service's+ * current, account-scoped state, as plain reads that take no lock of that service.+ *+ * <p>A service records a reference inside {@link Ec2Service#supplyWithSubnetReferencesHeld}, which+ * holds the lock EC2 holds while it asks these questions and deletes, so a delete cannot pass its+ * check while a reference is being published.+ */+public interface VpcDependencyProvider {+ boolean hasSubnetDependency(String region, String subnetId);++ boolean hasVpcDependency(String region, String vpcId);+}diff --git a/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java b/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.javaindex b65a32a5a3..2c1aeb972c 100644--- a/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.core.resource.SupportedResourceType;import io.github.hectorvent.floci.core.storage.StorageBackedMap;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.services.ec2.VpcDependencyProvider;import io.github.hectorvent.floci.services.ec2.model.Subnet;import io.github.hectorvent.floci.services.elbv2.model.*;import io.github.hectorvent.floci.services.iam.ServerCertificateReferenceProvider;@@ -34,7 +35,7 @@ import java.util.concurrent.ConcurrentHashMap;import java.util.stream.Collectors;@ApplicationScoped-public class ElbV2Service implements ResourceProvider, ServerCertificateReferenceProvider {+public class ElbV2Service implements ResourceProvider, ServerCertificateReferenceProvider, VpcDependencyProvider {@InjectElbV2DataPlane dataPlane;@@ -195,10 +196,23 @@ public class ElbV2Service implements ResourceProvider, ServerCertificateReferenc// ── Load Balancers ────────────────────────────────────────────────────────+ /**+ * Resolving the subnets and publishing the balancer happen under EC2's subnet lock, so a+ * DeleteSubnet cannot pass its dependency check in between and leave the balancer on a subnet+ * that is gone.+ */public LoadBalancer createLoadBalancer(String region, String name, String scheme,String type, String ipAddressType,List<String> subnets, List<String> securityGroups,Map<String, String> initialTags) {+ return ec2Service.supplyWithSubnetReferencesHeld(region, () -> createLoadBalancerWithSubnetsHeld(+ region, name, scheme, type, ipAddressType, subnets, securityGroups, initialTags));+ }++ private LoadBalancer createLoadBalancerWithSubnetsHeld(String region, String name, String scheme,+ String type, String ipAddressType,+ List<String> subnets, List<String> securityGroups,+ Map<String, String> initialTags) {validateName(name, "load balancer");Map<String, LoadBalancer> regionLbs = loadBalancers.computeIfAbsent(region, k -> new ConcurrentHashMap<>());boolean duplicate = regionLbs.values().stream()@@ -241,6 +255,23 @@ public class ElbV2Service implements ResourceProvider, ServerCertificateReferencreturn lb;}+ /**+ * A subnet is in use while a load balancer is configured with it. Account-scoped through the+ * request-scoped storage, and a target group does not count: it pins nothing in AWS either.+ */+ @Override+ public boolean hasSubnetDependency(String region, String subnetId) {+ return loadBalancers.getOrDefault(region, Map.of()).values().stream()+ .anyMatch(lb -> lb.getAvailabilityZones().stream()+ .anyMatch(zone -> subnetId.equals(zone.getSubnetId())));+ }++ @Override+ public boolean hasVpcDependency(String region, String vpcId) {+ return loadBalancers.getOrDefault(region, Map.of()).values().stream()+ .anyMatch(lb -> vpcId.equals(lb.getVpcId()));+ }+private String loadBalancerDnsSuffix() {if (config == null) {return EmbeddedDnsServer.DEFAULT_SUFFIX;@@ -336,18 +367,21 @@ public class ElbV2Service implements ResourceProvider, ServerCertificateReferencpersistRegion(loadBalancers, region);}+ /** Under EC2's subnet lock for the same reason as {@link #createLoadBalancer}. */public void setSubnets(String region, String arn, List<String> subnets) {- LoadBalancer lb = requireLoadBalancer(region, arn);- String vpcId = resolveSubnetVpcId(region, lb.getType(), subnets);- if (vpcId != null && lb.getVpcId() != null && !lb.getVpcId().equals(vpcId)) {- throw new AwsException("InvalidConfigurationRequest",- "All subnets must belong to the load balancer VPC.", 400);- }- lb.setAvailabilityZones(resolveAvailabilityZones(region, subnets));- if (vpcId != null) {- lb.setVpcId(vpcId);- }- persistRegion(loadBalancers, region);+ ec2Service.runWithSubnetReferencesHeld(region, () -> {+ LoadBalancer lb = requireLoadBalancer(region, arn);+ String vpcId = resolveSubnetVpcId(region, lb.getType(), subnets);+ if (vpcId != null && lb.getVpcId() != null && !lb.getVpcId().equals(vpcId)) {+ throw new AwsException("InvalidConfigurationRequest",+ "All subnets must belong to the load balancer VPC.", 400);+ }+ lb.setAvailabilityZones(resolveAvailabilityZones(region, subnets));+ if (vpcId != null) {+ lb.setVpcId(vpcId);+ }+ persistRegion(loadBalancers, region);+ });}public void setIpAddressType(String region, String arn, String ipAddressType) {diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationElbV2SubnetUpdateIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationElbV2SubnetUpdateIntegrationTest.javanew file mode 100644index 0000000000..657165a780--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationElbV2SubnetUpdateIntegrationTest.java@@ -0,0 +1,687 @@+package io.github.hectorvent.floci.services.cloudformation;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.XmlParser;+import io.github.hectorvent.floci.services.elbv2.ElbV2Service;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.mockito.InjectSpy;+import io.restassured.response.ValidatableResponse;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.Set;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.anyList;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.doCallRealMethod;+import static org.mockito.Mockito.reset;++/**+ * Stack updates that move an ELBv2 load balancer between subnets, through the whole engine. EC2+ * refuses to delete a subnet a load balancer still uses, so an update has to move the balancer off+ * a subnet the template dropped or replaced before its cleanup deletes that subnet, and a rolled-back+ * update has to move the balancer back before it deletes a subnet the update added.+ */+@QuarkusTest+class CloudFormationElbV2SubnetUpdateIntegrationTest {++ private static final String CFN_AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20261013/us-east-1/cloudformation/aws4_request";+ private static final String EC2_AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20261013/us-east-1/ec2/aws4_request";+ private static final String ELB_AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20261013/us-east-1/elasticloadbalancing/aws4_request";+ private static final String CIDR_A = "10.93.1.0/24";+ private static final String CIDR_B = "10.93.2.0/24";+ private static final String CIDR_C = "10.93.3.0/24";++ @InjectSpy+ ElbV2Service elbService;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void anUpdateThatDropsASubnetMovesTheLoadBalancerOffItSoTheCleanupDeletesIt() {+ String stack = "alb-drop-subnet-" + Long.toString(System.nanoTime(), 36);+ try {+ Map<String, String> created = createStack(stack,+ template(CIDR_B, true, List.of("SubnetA", "SubnetB", "SubnetC"), ""));++ Map<String, String> updated = updateStack(stack,+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""), "UPDATE_COMPLETE");++ assertEquals(Set.of(updated.get("SubnetA"), updated.get("SubnetB")),+ Set.copyOf(loadBalancerSubnets(updated.get("Alb"))));+ ec2("DescribeSubnets", "SubnetId.1", created.get("SubnetC")).statusCode(400)+ .body(containsString("InvalidSubnetID.NotFound"));+ } finally {+ deleteStack(stack);+ }+ }++ @Test+ void anUpdateThatReplacesASubnetMovesTheLoadBalancerOntoTheReplacement() {+ String stack = "alb-replace-subnet-" + Long.toString(System.nanoTime(), 36);+ try {+ String oldSubnetB = createStack(stack,+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), "")).get("SubnetB");++ // A new CidrBlock is create-only, so SubnetB is replaced and the old one is deleted on commit.+ Map<String, String> updated = updateStack(stack,+ template("10.93.4.0/24", false, List.of("SubnetA", "SubnetB"), ""), "UPDATE_COMPLETE");++ String newSubnetB = updated.get("SubnetB");+ assertNotEquals(oldSubnetB, newSubnetB);+ assertEquals(Set.of(updated.get("SubnetA"), newSubnetB),+ Set.copyOf(loadBalancerSubnets(updated.get("Alb"))));+ ec2("DescribeSubnets", "SubnetId.1", oldSubnetB).statusCode(400)+ .body(containsString("InvalidSubnetID.NotFound"));+ } finally {+ deleteStack(stack);+ }+ }++ /**+ * An earlier update added SubnetC and moved the load balancer onto it, so SubnetC was recorded+ * after the load balancer. An update that removes both has to delete the load balancer first,+ * though the template that named what it uses is gone.+ */+ @Test+ void anUpdateThatRemovesTheLoadBalancerAndASubnetItUsesDeletesTheLoadBalancerFirst() {+ String stack = "alb-remove-both-" + Long.toString(System.nanoTime(), 36);+ try {+ createStack(stack, template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));+ Map<String, String> moved = updateStack(stack,+ template(CIDR_B, true, List.of("SubnetA", "SubnetB", "SubnetC"), ""), "UPDATE_COMPLETE");+ assertEquals(3, loadBalancerSubnets(moved.get("Alb")).size());++ String withoutEither = """+ {+ "Resources": {+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.93.0.0/16"}},+ "SubnetA": {+ "Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1a"}+ },+ "SubnetB": {+ "Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1b"}+ }+ }+ }+ """.formatted(CIDR_A, CIDR_B);+ request(CFN_AUTH, "UpdateStack", "StackName", stack, "TemplateBody", withoutEither).statusCode(200);+ CfnStackWaits.StackState updated = CfnStackWaits.awaitStatus(stack, "UPDATE_COMPLETE");++ assertTrue(updated.reason() == null || updated.reason().isEmpty(), updated.reason());+ List<String> events = events(stack);+ assertTrue(events.contains("Alb DELETE_COMPLETE"), String.join(", ", events));+ assertTrue(events.indexOf("Alb DELETE_COMPLETE") < events.indexOf("SubnetC DELETE_IN_PROGRESS"),+ String.join(", ", events));+ ec2("DescribeSubnets", "SubnetId.1", moved.get("SubnetC")).statusCode(400)+ .body(containsString("InvalidSubnetID.NotFound"));+ request(ELB_AUTH, "DescribeLoadBalancers", "LoadBalancerArns.member.1", moved.get("Alb")).statusCode(400)+ .body(containsString("LoadBalancerNotFound"));+ } finally {+ deleteStack(stack);+ }+ }++ /**+ * A new name replaces the load balancer, and the old one stays on SubnetC until the update's+ * cleanup deletes it. The same update drops SubnetC, so the cleanup has to delete the old load+ * balancer before SubnetC.+ */+ @Test+ void anUpdateThatReplacesTheLoadBalancerAndDropsASubnetTheOldOneUsesDeletesTheOldOneFirst() {+ String stack = "alb-rename-drop-" + Long.toString(System.nanoTime(), 36);+ String suffix = stack.substring(stack.length() - 8);+ try {+ Map<String, String> created = createStack(stack, named(+ template(CIDR_B, true, List.of("SubnetA", "SubnetB", "SubnetC"), ""), "old-" + suffix));++ request(CFN_AUTH, "UpdateStack", "StackName", stack, "TemplateBody",+ named(template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""), "new-" + suffix))+ .statusCode(200);+ CfnStackWaits.StackState updated = CfnStackWaits.awaitStatus(stack, "UPDATE_COMPLETE");++ assertTrue(updated.reason() == null || updated.reason().isEmpty(), updated.reason());+ List<String> events = events(stack);+ assertTrue(events.contains("Alb DELETE_COMPLETE") && events.contains("SubnetC DELETE_COMPLETE"),+ String.join(", ", events));+ assertTrue(events.lastIndexOf("Alb DELETE_COMPLETE") < events.indexOf("SubnetC DELETE_COMPLETE"),+ String.join(", ", events));+ Map<String, String> outputs = outputs(stack);+ assertNotEquals(created.get("Alb"), outputs.get("Alb"));+ request(ELB_AUTH, "DescribeLoadBalancers", "LoadBalancerArns.member.1", created.get("Alb")).statusCode(400)+ .body(containsString("LoadBalancerNotFound"));+ ec2("DescribeSubnets", "SubnetId.1", created.get("SubnetC")).statusCode(400)+ .body(containsString("InvalidSubnetID.NotFound"));+ } finally {+ deleteStack(stack);+ }+ }++ @Test+ void aFailedUpdateMovesTheLoadBalancerBackToItsSubnets() {+ String stack = "alb-subnets-rollback-" + Long.toString(System.nanoTime(), 36);+ try {+ Map<String, String> created = createStack(stack,+ template(CIDR_B, true, List.of("SubnetA", "SubnetB", "SubnetC"), ""));+ Set<String> before = Set.copyOf(loadBalancerSubnets(created.get("Alb")));++ Map<String, String> rolledBack = updateStack(stack, template(CIDR_B, true,+ List.of("SubnetA", "SubnetB"), failingSecret(stack, "Alb")), "UPDATE_ROLLBACK_COMPLETE");++ assertEquals(3, before.size());+ assertEquals(before, Set.copyOf(loadBalancerSubnets(rolledBack.get("Alb"))));+ } finally {+ deleteStack(stack);+ }+ }++ /**+ * The update adds SubnetC and moves the load balancer onto it before a later resource fails.+ * The rollback has to move the load balancer back before it deletes the subnet it added, or EC2+ * refuses the delete and the rollback fails.+ */+ @Test+ void aFailedUpdateMovesTheLoadBalancerOffASubnetItAddedBeforeDeletingIt() {+ String stack = "alb-added-subnet-rollback-" + Long.toString(System.nanoTime(), 36);+ try {+ Map<String, String> created = createStack(stack,+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));+ Set<String> before = Set.of(created.get("SubnetA"), created.get("SubnetB"));++ Map<String, String> rolledBack = updateStack(stack, template(CIDR_B, true,+ List.of("SubnetA", "SubnetC"), failingSecret(stack, "Alb")), "UPDATE_ROLLBACK_COMPLETE");++ assertEquals(before, Set.copyOf(loadBalancerSubnets(rolledBack.get("Alb"))));+ assertEquals(before, Set.copyOf(vpcSubnets(created.get("Vpc"))));+ } finally {+ deleteStack(stack);+ }+ }++ /**+ * The first rollback's move back fails, which leaves the load balancer on the subnet the update+ * added, so that subnet cannot be deleted either and the stack ends UPDATE_ROLLBACK_FAILED with+ * both owed. ContinueUpdateRollback walks them again in dependency order: the load balancer moves+ * back first, from the subnets it kept for the retry, and then the subnet goes.+ */+ @Test+ void continueUpdateRollbackMovesTheLoadBalancerBackBeforeDeletingTheSubnetItAdded() {+ String stack = "alb-continue-rollback-" + Long.toString(System.nanoTime(), 36);+ try {+ Map<String, String> created = createStack(stack,+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));+ Set<String> before = Set.of(created.get("SubnetA"), created.get("SubnetB"));+ failTheFirstMoveBack();++ failUpdateOntoAnAddedSubnet(stack);++ continueUpdateRollback(stack);+ CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_COMPLETE");+ assertEquals(before, Set.copyOf(loadBalancerSubnets(created.get("Alb"))));+ assertEquals(before, Set.copyOf(vpcSubnets(created.get("Vpc"))));+ } finally {+ reset(elbService);+ deleteStack(stack);+ }+ }++ /**+ * ResourcesToSkip leaves a skipped resource as the failed update left it. Skipping only the load+ * balancer keeps it on the added subnet, so that subnet still cannot be deleted. Skipping the+ * subnet as well completes the rollback with both left in place, and the stack still deletes+ * cleanly, load balancer first.+ */+ @Test+ void continueUpdateRollbackWithResourcesToSkipLeavesTheSkippedLoadBalancerOnTheAddedSubnet() {+ String stack = "alb-continue-skip-" + Long.toString(System.nanoTime(), 36);+ try {+ Map<String, String> created = createStack(stack,+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));+ failTheFirstMoveBack();+ failUpdateOntoAnAddedSubnet(stack);+ Set<String> addedSubnet = new HashSet<>(vpcSubnets(created.get("Vpc")));+ addedSubnet.removeAll(Set.of(created.get("SubnetA"), created.get("SubnetB")));+ assertEquals(1, addedSubnet.size());++ continueUpdateRollback(stack, "Alb");+ CfnStackWaits.StackState stillFailed = CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_FAILED");+ assertTrue(stillFailed.reason().contains("SubnetC"), stillFailed.reason());++ continueUpdateRollback(stack, "SubnetC");+ CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_COMPLETE");+ assertEquals(Set.of(created.get("SubnetA"), addedSubnet.iterator().next()),+ Set.copyOf(loadBalancerSubnets(created.get("Alb"))));+ assertEquals(3, vpcSubnets(created.get("Vpc")).size());+ } finally {+ reset(elbService);+ deleteStack(stack);+ }+ }++ /**+ * The update's own move onto the new subnets goes through, the first move back fails as if its+ * subnet had been deleted meanwhile, and any later call goes through again.+ */+ private void failTheFirstMoveBack() {+ doCallRealMethod()+ .doThrow(new AwsException("SubnetNotFound", "The subnet ID was not found", 400))+ .doCallRealMethod()+ .when(elbService).setSubnets(anyString(), anyString(), anyList());+ }++ /** Moves the load balancer onto SubnetA and an added SubnetC in an update a later resource fails. */+ private static void failUpdateOntoAnAddedSubnet(String stack) {+ updateStack(stack, template(CIDR_B, true, List.of("SubnetA", "SubnetC"), failingSecret(stack, "Alb")),+ "UPDATE_ROLLBACK_FAILED");+ String reason = CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_FAILED").reason();+ assertTrue(reason.contains("Alb") && reason.contains("SubnetC"), reason);+ }++ private static void continueUpdateRollback(String stack, String... resourcesToSkip) {+ String[] values = new String[2 + resourcesToSkip.length * 2];+ values[0] = "StackName";+ values[1] = stack;+ for (int i = 0; i < resourcesToSkip.length; i++) {+ values[2 + i * 2] = "ResourcesToSkip.member." + (i + 1);+ values[3 + i * 2] = resourcesToSkip[i];+ }+ request(CFN_AUTH, "ContinueUpdateRollback", values).statusCode(200);+ }++ /**+ * A failed rollback restores the parameters but can leave a resource on the branch the failed+ * update chose: here the load balancer stays on the subnets an Fn::If picked for the new+ * parameter value. Deleting the stack has to delete it before those subnets too, whichever+ * branch the restored parameters now select.+ */+ @Test+ void deleteStackAfterAFailedRollbackDeletesTheLoadBalancerBeforeTheSubnetsOfEitherBranch() {+ String stack = "alb-fnif-branch-" + Long.toString(System.nanoTime(), 36);+ try {+ Map<String, String> created = createStack(stack, branchTemplate(""), "first");+ failTheFirstMoveBack();+ request(CFN_AUTH, "UpdateStack", "StackName", stack, "TemplateBody",+ branchTemplate(failingSecret(stack, "Alb")),+ "Parameters.member.1.ParameterKey", "Placement",+ "Parameters.member.1.ParameterValue", "second").statusCode(200);+ String reason = CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_FAILED").reason();+ assertTrue(reason.contains("Alb"), reason);+ assertEquals(Set.of(created.get("SubnetC"), created.get("SubnetD")),+ Set.copyOf(loadBalancerSubnets(created.get("Alb"))), "the move back failed");+ reset(elbService);++ deleteStack(stack);++ ec2("DescribeVpcs", "VpcId.1", created.get("Vpc")).statusCode(400)+ .body(containsString("InvalidVpcID.NotFound"));+ } finally {+ reset(elbService);+ if (request(CFN_AUTH, "DescribeStacks", "StackName", stack).extract().statusCode() == 200) {+ request(CFN_AUTH, "DeleteStack", "StackName", stack);+ }+ }+ }++ /** CloudFormation deletes a VPC too, so the load balancer has to go before its subnets and they before the VPC. */+ @Test+ void deleteStackRemovesTheLoadBalancerBeforeTheSubnetsAndTheVpcUnderIt() {+ String stack = "alb-vpc-delete-" + Long.toString(System.nanoTime(), 36);+ Map<String, String> created = createStack(stack, template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));++ deleteStack(stack);++ ec2("DescribeVpcs", "VpcId.1", created.get("Vpc")).statusCode(400)+ .body(containsString("InvalidVpcID.NotFound"));+ request(ELB_AUTH, "DescribeLoadBalancers", "LoadBalancerArns.member.1", created.get("Alb")).statusCode(400)+ .body(containsString("LoadBalancerNotFound"));+ }++ /**+ * A load balancer outside the stack on the stack's subnets pins them, and through them the VPC,+ * so DeleteStack fails until that load balancer is gone, and then deletes everything.+ */+ @Test+ void aLoadBalancerOutsideTheStackKeepsItsSubnetsAndVpcUntilItIsDeleted() {+ String stack = "alb-vpc-outside-" + Long.toString(System.nanoTime(), 36);+ Map<String, String> created = createStack(stack, template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));+ try {+ String outside = request(ELB_AUTH, "CreateLoadBalancer",+ "Name", "outside-" + stack.substring(stack.length() - 8), "Type", "application",+ "Scheme", "internal",+ "Subnets.member.1", created.get("SubnetA"), "Subnets.member.2", created.get("SubnetB"))+ .statusCode(200).extract()+ .path("CreateLoadBalancerResponse.CreateLoadBalancerResult.LoadBalancers.member.LoadBalancerArn");+ try {+ request(CFN_AUTH, "DeleteStack", "StackName", stack).statusCode(200);+ CfnStackWaits.StackState failed = CfnStackWaits.awaitTerminal(stack, CFN_AUTH);+ assertEquals("DELETE_FAILED", failed.status());+ String reason = failed.reason();+ assertEquals(Set.of("SubnetA", "SubnetB", "Vpc"),+ Set.of(reason.substring(reason.indexOf('[') + 1, reason.indexOf(']')).split(", ")), reason);+ } finally {+ request(ELB_AUTH, "DeleteLoadBalancer", "LoadBalancerArn", outside).statusCode(200);+ }+ } finally {+ deleteStack(stack);+ }++ ec2("DescribeVpcs", "VpcId.1", created.get("Vpc")).statusCode(400)+ .body(containsString("InvalidVpcID.NotFound"));+ }++ /**+ * An update that adds a VPC, subnets in it and a load balancer on them, then fails, rolls them+ * back dependents first: the load balancer before its subnets, and the subnets before the VPC,+ * so nothing the update added is left behind.+ */+ @Test+ void aFailedUpdateDeletesTheLoadBalancerItAddedBeforeTheSubnetsAndVpcUnderIt() {+ String stack = "alb-vpc-rollback-" + Long.toString(System.nanoTime(), 36);+ try {+ Map<String, String> created = createStack(stack,+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));+ String added = """+ ,+ "Vpc2": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.95.0.0/16",+ "Tags": [{"Key": "floci-it", "Value": "%1$s"}]}},+ "SubnetX": {+ "Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc2"}, "CidrBlock": "10.95.1.0/24", "AvailabilityZone": "us-east-1a"}+ },+ "SubnetY": {+ "Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc2"}, "CidrBlock": "10.95.2.0/24", "AvailabilityZone": "us-east-1b"}+ },+ "Alb2": {+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",+ "Properties": {"Type": "application", "Scheme": "internal",+ "Subnets": [{"Ref": "SubnetX"}, {"Ref": "SubnetY"}]}+ }""".formatted(stack);++ updateStack(stack, template(CIDR_B, false, List.of("SubnetA", "SubnetB"),+ added + failingSecret(stack, "Alb2")), "UPDATE_ROLLBACK_COMPLETE");++ String vpcs = ec2("DescribeVpcs", "Filter.1.Name", "tag:floci-it", "Filter.1.Value.1", stack)+ .statusCode(200).extract().asString();+ assertEquals(List.of(), XmlParser.extractAll(vpcs, "vpcId"));+ assertEquals(Set.of(created.get("SubnetA"), created.get("SubnetB")),+ Set.copyOf(loadBalancerSubnets(created.get("Alb"))));+ } finally {+ deleteStack(stack);+ }+ }++ /**+ * The rollback orders resources by the template's conditions, evaluated with the SSM-resolved+ * parameter value the update itself used. With the raw value, the parameter's name, the load+ * balancer's condition reads false and the subnet it uses would be deleted first.+ */+ @Test+ void aRollbackOrdersAResourceUnderAnSsmResolvedConditionBeforeTheSubnetItUses() {+ String stack = "alb-ssm-condition-rollback-" + Long.toString(System.nanoTime(), 36);+ String parameterName = "/" + stack + "/env";+ ssm("PutParameter", "{\"Name\": \"" + parameterName + "\", \"Value\": \"prod\", \"Type\": \"String\"}");+ try {+ Map<String, String> created = createStack(stack, ssmConditionTemplate(parameterName, ""));+ Set<String> before = Set.of(created.get("SubnetA"), created.get("SubnetB"));++ String added = """+ ,+ "SubnetC": {+ "Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1c"}+ },+ "ProdAlb": {+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",+ "Condition": "IsProd",+ "Properties": {"Type": "application", "Scheme": "internal",+ "Subnets": [{"Ref": "SubnetA"}, {"Ref": "SubnetC"}]}+ },+ "BadSecret": {+ "Type": "AWS::SecretsManager::Secret",+ "Condition": "IsProd",+ "DependsOn": "ProdAlb",+ "Properties": {+ "Name": "%s-secret",+ "SecretString": "explicit",+ "GenerateSecretString": {"PasswordLength": 32}+ }+ }""".formatted(CIDR_C, stack);+ updateStack(stack, ssmConditionTemplate(parameterName, added), "UPDATE_ROLLBACK_COMPLETE");++ assertEquals(before, Set.copyOf(vpcSubnets(created.get("Vpc"))));+ } finally {+ deleteStack(stack);+ ssm("DeleteParameter", "{\"Name\": \"" + parameterName + "\"}");+ }+ }++ /** A secret that sets both secret properties, which fails the update once what it depends on is done. */+ private static String failingSecret(String stack, String dependsOn) {+ return """+ ,+ "BadSecret": {+ "Type": "AWS::SecretsManager::Secret",+ "DependsOn": "%s",+ "Properties": {+ "Name": "%s-secret",+ "SecretString": "explicit",+ "GenerateSecretString": {"PasswordLength": 32}+ }+ }""".formatted(dependsOn, stack);+ }++ /**+ * A VPC with subnets in three zones, and an internal ALB on the named ones. SubnetC, its+ * output and SubnetB's CIDR vary between the versions a test applies.+ */+ private static String template(String cidrB, boolean withSubnetC, List<String> albSubnets, String extraResources) {+ StringBuilder refs = new StringBuilder();+ for (String subnet : albSubnets) {+ refs.append(refs.isEmpty() ? "" : ", ").append("{\"Ref\": \"").append(subnet).append("\"}");+ }+ String subnetC = withSubnetC ? """+ "SubnetC": {+ "Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1c"}+ },""".formatted(CIDR_C) : "";+ String subnetCOutput = withSubnetC ? """+ "SubnetC": {"Value": {"Ref": "SubnetC"}},""" : "";+ return """+ {+ "Resources": {+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.93.0.0/16"}},+ "SubnetA": {+ "Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1a"}+ },+ "SubnetB": {+ "Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1b"}+ },%s+ "Alb": {+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",+ "Properties": {"Type": "application", "Scheme": "internal", "Subnets": [%s]}+ }%s+ },+ "Outputs": {%s+ "SubnetA": {"Value": {"Ref": "SubnetA"}},+ "SubnetB": {"Value": {"Ref": "SubnetB"}},+ "Vpc": {"Value": {"Ref": "Vpc"}},+ "Alb": {"Value": {"Ref": "Alb"}}+ }+ }+ """.formatted(CIDR_A, cidrB, subnetC, refs, extraResources, subnetCOutput);+ }++ /** A {@link #template} whose ALB has the given name, which is create-only. */+ private static String named(String template, String name) {+ return template.replace("\"Scheme\": \"internal\", \"Subnets\"",+ "\"Name\": \"" + name + "\", \"Scheme\": \"internal\", \"Subnets\"");+ }++ /**+ * A VPC with four subnets and an ALB whose subnets an Fn::If picks by the Placement parameter:+ * SubnetA and SubnetB for "first", SubnetC and SubnetD for "second". The ALB is declared before+ * the subnets, and SubnetC is created after SubnetD after SubnetB, so whichever branch the+ * ordering reads, SubnetC is not created before the ALB unless the ALB depends on it.+ */+ private static String branchTemplate(String extraResources) {+ return """+ {+ "Parameters": {"Placement": {"Type": "String", "Default": "first"}},+ "Conditions": {"Second": {"Fn::Equals": [{"Ref": "Placement"}, "second"]}},+ "Resources": {+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.93.0.0/16"}},+ "Alb": {+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",+ "Properties": {"Type": "application", "Scheme": "internal",+ "Subnets": {"Fn::If": ["Second", [{"Ref": "SubnetC"}, {"Ref": "SubnetD"}],+ [{"Ref": "SubnetA"}, {"Ref": "SubnetB"}]]}}+ },+ "SubnetA": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": {"Ref": "Vpc"},+ "CidrBlock": "10.93.1.0/24", "AvailabilityZone": "us-east-1a"}},+ "SubnetB": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": {"Ref": "Vpc"},+ "CidrBlock": "10.93.2.0/24", "AvailabilityZone": "us-east-1b"}},+ "SubnetC": {"Type": "AWS::EC2::Subnet", "DependsOn": "SubnetD", "Properties": {"VpcId": {"Ref": "Vpc"},+ "CidrBlock": "10.93.3.0/24", "AvailabilityZone": "us-east-1a"}},+ "SubnetD": {"Type": "AWS::EC2::Subnet", "DependsOn": "SubnetB", "Properties": {"VpcId": {"Ref": "Vpc"},+ "CidrBlock": "10.93.4.0/24", "AvailabilityZone": "us-east-1b"}}%s+ },+ "Outputs": {+ "Vpc": {"Value": {"Ref": "Vpc"}},+ "SubnetC": {"Value": {"Ref": "SubnetC"}},+ "SubnetD": {"Value": {"Ref": "SubnetD"}},+ "Alb": {"Value": {"Ref": "Alb"}}+ }+ }+ """.formatted(extraResources);+ }++ /** A VPC and two subnets, with an IsProd condition on an SSM-typed parameter. */+ private static String ssmConditionTemplate(String parameterName, String extraResources) {+ return """+ {+ "Parameters": {+ "Env": {"Type": "AWS::SSM::Parameter::Value<String>", "Default": "%s"}+ },+ "Conditions": {+ "IsProd": {"Fn::Equals": [{"Ref": "Env"}, "prod"]}+ },+ "Resources": {+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.93.0.0/16"}},+ "SubnetA": {+ "Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1a"}+ },+ "SubnetB": {+ "Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1b"}+ }%s+ },+ "Outputs": {+ "SubnetA": {"Value": {"Ref": "SubnetA"}},+ "SubnetB": {"Value": {"Ref": "SubnetB"}},+ "Vpc": {"Value": {"Ref": "Vpc"}}+ }+ }+ """.formatted(parameterName, CIDR_A, CIDR_B, extraResources);+ }++ private static List<String> loadBalancerSubnets(String arn) {+ assertNotNull(arn, "the stack outputs the load balancer");+ String xml = request(ELB_AUTH, "DescribeLoadBalancers", "LoadBalancerArns.member.1", arn)+ .statusCode(200).extract().asString();+ return XmlParser.extractAll(xml, "SubnetId");+ }++ private static List<String> vpcSubnets(String vpcId) {+ String xml = ec2("DescribeSubnets", "Filter.1.Name", "vpc-id", "Filter.1.Value.1", vpcId)+ .statusCode(200).extract().asString();+ return XmlParser.extractAll(xml, "subnetId");+ }++ private static Map<String, String> createStack(String stack, String templateBody, String placement) {+ request(CFN_AUTH, "CreateStack", "StackName", stack, "TemplateBody", templateBody,+ "Parameters.member.1.ParameterKey", "Placement",+ "Parameters.member.1.ParameterValue", placement).statusCode(200);+ CfnStackWaits.awaitStatus(stack, "CREATE_COMPLETE");+ return outputs(stack);+ }++ private static Map<String, String> createStack(String stack, String templateBody) {+ request(CFN_AUTH, "CreateStack", "StackName", stack, "TemplateBody", templateBody).statusCode(200);+ CfnStackWaits.awaitStatus(stack, "CREATE_COMPLETE");+ return outputs(stack);+ }++ private static Map<String, String> updateStack(String stack, String templateBody, String expectedStatus) {+ request(CFN_AUTH, "UpdateStack", "StackName", stack, "TemplateBody", templateBody).statusCode(200);+ CfnStackWaits.awaitStatus(stack, expectedStatus);+ return outputs(stack);+ }++ /** The stack's events, oldest first, each as its logical id and status. */+ private static List<String> events(String stack) {+ String xml = request(CFN_AUTH, "DescribeStackEvents", "StackName", stack).statusCode(200)+ .extract().asString();+ List<String> ids = XmlParser.extractAll(xml, "LogicalResourceId");+ List<String> statuses = XmlParser.extractAll(xml, "ResourceStatus");+ assertEquals(ids.size(), statuses.size());+ List<String> events = new ArrayList<>();+ for (int i = ids.size() - 1; i >= 0; i--) {+ events.add(ids.get(i) + " " + statuses.get(i));+ }+ return events;+ }++ private static Map<String, String> outputs(String stack) {+ String xml = request(CFN_AUTH, "DescribeStacks", "StackName", stack).statusCode(200).extract().asString();+ return XmlParser.extractPairs(xml, "member", "OutputKey", "OutputValue");+ }++ /** DeleteStack answers before the stack is gone, so this waits and fails on DELETE_FAILED. */+ private static void deleteStack(String stack) {+ request(CFN_AUTH, "DeleteStack", "StackName", stack).statusCode(200);+ CfnStackWaits.awaitStackDeleted(stack, CFN_AUTH);+ }++ private static void ssm(String action, String body) {+ given().header("X-Amz-Target", "AmazonSSM." + action).contentType("application/x-amz-json-1.1")+ .body(body).when().post("/").then().statusCode(200);+ }++ private static ValidatableResponse ec2(String action, String... values) {+ return request(EC2_AUTH, action, values);+ }++ private static ValidatableResponse request(String auth, String action, String... values) {+ Map<String, String> parameters = new LinkedHashMap<>();+ parameters.put("Action", action);+ for (int i = 0; i < values.length; i += 2) {+ parameters.put(values[i], values[i + 1]);+ }+ return given().header("Authorization", auth).formParams(parameters).when().post("/").then()+ .log().ifValidationFails();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisionerTest.javaindex af7b97d853..e08d1eac3e 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisionerTest.java@@ -22,9 +22,11 @@ import org.mockito.ArgumentCaptor;import org.mockito.InOrder;import java.util.HashMap;+import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Set;+import java.util.stream.Collectors;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -133,7 +135,7 @@ class ElbV2CfnProvisionerTest {void loadBalancerExposesExactlyTheSchemaAttributes() {when(elb.createLoadBalancer(eq(REGION), eq("web"), eq("internet-facing"), eq("application"), isNull(),eq(List.of("subnet-1", "subnet-2")), eq(List.of("sg-1")), eq(Map.of("env", "test"))))- .thenReturn(loadBalancer("web"));+ .thenReturn(loadBalancerOn("web", "subnet-1", "subnet-2"));ObjectNode props = mapper.createObjectNode().put("Name", "web").put("Scheme", "internet-facing").put("Type", "application");props.putArray("Subnets").add("subnet-1").add("subnet-2");@@ -144,8 +146,10 @@ class ElbV2CfnProvisionerTest {provisioner.provision(r, props, ctx());assertEquals(LB_ARN, r.getPhysicalId());+ // Fn::GetAtt resolves no __Floci key, so those are the hidden bookkeeping, not attributes.assertEquals(Set.of("LoadBalancerArn", "DNSName", "CanonicalHostedZoneID", "LoadBalancerName",- "LoadBalancerFullName"), r.getAttributes().keySet());+ "LoadBalancerFullName"), r.getAttributes().keySet().stream()+ .filter(key -> !key.startsWith("__Floci")).collect(Collectors.toSet()));assertEquals("app/web/50dc6c495c0c9188", r.getAttributes().get("LoadBalancerFullName"));assertEquals("web", r.getAttributes().get("LoadBalancerName"));assertEquals("Z35SXDOTRQ7X7K", r.getAttributes().get("CanonicalHostedZoneID"));@@ -452,6 +456,36 @@ class ElbV2CfnProvisionerTest {verify(elb).setSubnets(REGION, LB_ARN, List.of("subnet-1", "subnet-2"));}+ /** The subnets a load balancer is on, as its hidden uses attributes record them. */+ private static Set<String> usedSubnets(StackResource lb) {+ Set<String> used = new HashSet<>();+ lb.getAttributes().forEach((key, value) -> {+ if (key.startsWith(CfnRollback.USES_ATTR_PREFIX)) {+ used.add(value);+ }+ });+ return used;+ }++ @Test+ void theLoadBalancerRecordsTheSubnetsItIsOnAcrossAMoveAndAMoveBack() {+ when(elb.createLoadBalancer(eq(REGION), eq("web"), isNull(), isNull(), isNull(),+ eq(List.of("subnet-1", "subnet-2")), anyList(), any()))+ .thenReturn(loadBalancerOn("web", "subnet-1", "subnet-2"));+ ObjectNode props = mapper.createObjectNode().put("Name", "web");+ props.putArray("Subnets").add("subnet-1").add("subnet-2");+ StackResource created = resource("AWS::ElasticLoadBalancingV2::LoadBalancer", "Alb");+ provisioner.provision(created, props, ctx());+ assertEquals(Set.of("subnet-1", "subnet-2"), usedSubnets(created));++ StackResource moved = keptLoadBalancerUpdatedTo(loadBalancerOn("web", "subnet-1", "subnet-2"),+ "subnet-3");+ assertEquals(Set.of("subnet-3"), usedSubnets(moved));++ assertTrue(provisioner.rollbackUpdate(moved));+ assertEquals(Set.of("subnet-1", "subnet-2"), usedSubnets(moved));+ }+@Testvoid theSameSubnetsInAnotherOrderLeaveTheKeptLoadBalancerAlone() {StackResource lb = keptLoadBalancerUpdatedTo(loadBalancerOn("web", "subnet-1", "subnet-2"),diff --git a/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.javaindex cbfbdc234b..815a3d1733 100644--- a/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.CertificateReferenceResolver;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.PersistentStorage;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.ec2.Ec2Service;@@ -22,6 +23,7 @@ import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;+import org.junit.jupiter.api.io.TempDir;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.NullAndEmptySource;import org.junit.jupiter.params.provider.ValueSource;@@ -29,6 +31,7 @@ import org.mockito.ArgumentCaptor;import org.mockito.Mock;import org.mockito.junit.jupiter.MockitoExtension;+import java.nio.file.Path;import java.util.ArrayList;import java.util.HashMap;import java.util.List;@@ -39,6 +42,7 @@ import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;+import java.util.function.Supplier;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -640,6 +644,12 @@ class ElbV2ServiceTest {}private static void stubAlbSubnets(Ec2Service ec2Service) {+ lenient().when(ec2Service.supplyWithSubnetReferencesHeld(anyString(), any()))+ .thenAnswer(invocation -> invocation.<Supplier<?>>getArgument(1).get());+ lenient().doAnswer(invocation -> {+ invocation.<Runnable>getArgument(1).run();+ return null;+ }).when(ec2Service).runWithSubnetReferencesHeld(anyString(), any());Subnet subnetA = subnet("subnet-a", REGION + "a");Subnet subnetB = subnet("subnet-b", REGION + "b");lenient().when(ec2Service.requireSubnet(REGION, "subnet-a")).thenReturn(subnetA);@@ -656,6 +666,25 @@ class ElbV2ServiceTest {return subnet;}+ /** Files in a directory, so a second service reads back what the first wrote as JSON. */+ private static final class PersistentStorageFactory extends StorageFactory {+ private final Path dir;++ private PersistentStorageFactory(Path dir) {+ super(null, null);+ this.dir = dir;+ }++ @Override+ public <V> AccountAwareStorageBackend<V> create(String serviceName,+ String fileName,+ TypeReference<Map<String, V>> typeReference) {+ PersistentStorage<String, V> files = new PersistentStorage<>(dir.resolve(fileName), typeReference);+ files.load();+ return new AccountAwareStorageBackend<>(files, null, "000000000000");+ }+ }+private static final class SharedStorageFactory extends StorageFactory {private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();@@ -697,4 +726,27 @@ class ElbV2ServiceTest {reloaded.describeLoadBalancers(REGION, null, List.of("stored-nlb"), null, null).getFirst().getCanonicalHostedZoneId());}++ @Test+ void subnetAndVpcDependenciesComeFromPersistedLoadBalancersOnly(@TempDir Path dir) {+ ElbV2Service first = serviceWithStorage(new PersistentStorageFactory(dir), dataPlane, healthChecker);+ String arn = first.createLoadBalancer(REGION, "stored-dependency", "internal", "application", "ipv4",+ ALB_SUBNETS, List.of(), Map.of()).getLoadBalancerArn();+ String targetGroup = first.createTargetGroup(REGION, "independent-group", "HTTP", "HTTP1", 80,+ "vpc-a", "instance", "HTTP", "traffic-port", true, "/", 30, 5, 5, 2, "200",+ "ipv4", Map.of()).getTargetGroupArn();+ ElbV2Service restored = serviceWithStorage(new PersistentStorageFactory(dir), dataPlane, healthChecker);+ assertTrue(restored.hasSubnetDependency(REGION, "subnet-a"));+ assertTrue(restored.hasSubnetDependency(REGION, "subnet-b"));+ assertTrue(restored.hasVpcDependency(REGION, "vpc-a"));+ assertFalse(restored.hasSubnetDependency("us-east-1", "subnet-a"));+ assertFalse(restored.hasVpcDependency("us-east-1", "vpc-a"));+ assertFalse(restored.hasSubnetDependency(REGION, "subnet-unrelated"));+ restored.deleteLoadBalancer(REGION, arn);+ ElbV2Service afterDelete = serviceWithStorage(new PersistentStorageFactory(dir), dataPlane, healthChecker);+ assertFalse(afterDelete.hasSubnetDependency(REGION, "subnet-a"));+ assertFalse(afterDelete.hasVpcDependency(REGION, "vpc-a"));+ assertEquals(targetGroup, afterDelete.describeTargetGroups(REGION, null,+ List.of(targetGroup), null).getFirst().getTargetGroupArn());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2SubnetDependencyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2SubnetDependencyIntegrationTest.javanew file mode 100644index 0000000000..a629d998b3--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2SubnetDependencyIntegrationTest.java@@ -0,0 +1,282 @@+package io.github.hectorvent.floci.services.elbv2;++import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.services.ec2.model.Subnet;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.mockito.InjectSpy;+import io.restassured.response.ValidatableResponse;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.UUID;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.ThreadLocalRandom;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.concurrent.atomic.AtomicBoolean;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasItems;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyList;+import static org.mockito.ArgumentMatchers.anyMap;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.doReturn;+import static org.mockito.Mockito.reset;++@QuarkusTest+class ElbV2SubnetDependencyIntegrationTest {+ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "111122223333";++ @InjectSpy+ Ec2Service ec2Service;++ @InjectSpy+ ElbV2Service elbService;++ @ParameterizedTest+ @ValueSource(strings = {"application", "network"})+ void loadBalancerPinsSubnetsUntilDeletedButTargetGroupDoesNot(String type) {+ String vpc = vpc();+ String first = subnet(vpc, "10.92.1.0/24", "a");+ String second = subnet(vpc, "10.92.2.0/24", "b");+ String unused = subnet(vpc, "10.92.3.0/24", "c");+ String arn = loadBalancer(type, first, second);+ String targetGroup = elb("CreateTargetGroup", "Name", name(), "Protocol", "HTTP", "Port", "80",+ "VpcId", vpc).statusCode(200).extract()+ .path("CreateTargetGroupResponse.CreateTargetGroupResult.TargetGroups.member.TargetGroupArn");+ try {+ ec2("DeleteSubnet", "SubnetId", first).statusCode(400)+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));+ ec2("DeleteSubnet", "SubnetId", second).statusCode(400)+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));+ ec2("DeleteVpc", "VpcId", vpc).statusCode(400)+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));+ elb("DescribeLoadBalancers", "LoadBalancerArns.member.1", arn).statusCode(200)+ .body("DescribeLoadBalancersResponse.DescribeLoadBalancersResult.LoadBalancers.member"+ + ".AvailabilityZones.member.SubnetId", hasItems(first, second));+ ec2("DeleteSubnet", "SubnetId", unused).statusCode(200);+ } finally {+ elb("DeleteLoadBalancer", "LoadBalancerArn", arn).statusCode(200);+ }+ ec2("DeleteSubnet", "SubnetId", first).statusCode(200);+ ec2("DeleteSubnet", "SubnetId", second).statusCode(200);+ ec2("DeleteVpc", "VpcId", vpc).statusCode(200);+ elb("DescribeTargetGroups", "TargetGroupArns.member.1", targetGroup).statusCode(200);+ elb("DeleteTargetGroup", "TargetGroupArn", targetGroup).statusCode(200);+ }++ @ParameterizedTest+ @ValueSource(strings = {"application", "network"})+ void changingSubnetsReleasesOnlyTheOldReferences(String type) {+ String vpc = vpc();+ String first = subnet(vpc, "10.92.1.0/24", "a");+ String second = subnet(vpc, "10.92.2.0/24", "b");+ String replacement = subnet(vpc, "10.92.3.0/24", "c");+ String arn = loadBalancer(type, first, second);+ try {+ elb("SetSubnets", "LoadBalancerArn", arn, "Subnets.member.1", first,+ "Subnets.member.2", replacement).statusCode(200);+ ec2("DeleteSubnet", "SubnetId", second).statusCode(200);+ ec2("DeleteSubnet", "SubnetId", replacement).statusCode(400)+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));+ } finally {+ elb("DeleteLoadBalancer", "LoadBalancerArn", arn).statusCode(200);+ }+ ec2("DeleteSubnet", "SubnetId", first).statusCode(200);+ ec2("DeleteSubnet", "SubnetId", replacement).statusCode(200);+ ec2("DeleteVpc", "VpcId", vpc).statusCode(200);+ }++ /**+ * The other account is fresh for each run, since deleting its default subnet is permanent for+ * the rest of the suite's shared application.+ */+ @Test+ void referencesInAnotherAccountDoNotPinTheSameDefaultSubnetId() {+ String otherAccount = Long.toString(ThreadLocalRandom.current().nextLong(100_000_000_000L, 1_000_000_000_000L));+ String first = Ec2Service.defaultSubnetId(REGION, "a");+ String second = Ec2Service.defaultSubnetId(REGION, "b");+ String arn = loadBalancer("application", first, second);+ try {+ request("ec2", otherAccount, REGION, "DeleteSubnet", "SubnetId", first).statusCode(200);+ ec2("DeleteSubnet", "SubnetId", first).statusCode(400)+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));+ } finally {+ elb("DeleteLoadBalancer", "LoadBalancerArn", arn).statusCode(200);+ }+ }++ /**+ * A load balancer's subnets keep its VPC from being deleted anyway, so the VPC check is driven+ * directly: it stands for state such as a balancer stored before subnets were protected, whose+ * subnets could already be gone.+ */+ @Test+ void aVpcStillReferencedByALoadBalancerCannotBeDeletedEvenWithoutSubnets() {+ String vpc = vpc();+ doReturn(true).when(elbService).hasVpcDependency(REGION, vpc);+ try {+ ec2("DeleteVpc", "VpcId", vpc).statusCode(400)+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));+ } finally {+ reset(elbService);+ }+ ec2("DeleteVpc", "VpcId", vpc).statusCode(200);+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void deletionWaitsForCreateOrSetSubnetsToPublishItsReferences(boolean reassign) throws Exception {+ String vpc = vpc();+ String first = subnet(vpc, "10.92.1.0/24", "a");+ String second = subnet(vpc, "10.92.2.0/24", "b");+ String replacement = subnet(vpc, "10.92.3.0/24", "c");+ String existing = reassign ? loadBalancer("application", first, second) : null;+ CountDownLatch validated = new CountDownLatch(1);+ CountDownLatch publish = new CountDownLatch(1);+ CountDownLatch deleting = new CountDownLatch(1);+ AtomicBoolean pauseOnce = new AtomicBoolean(true);+ doAnswer(invocation -> {+ List<Subnet> found = (List<Subnet>) invocation.callRealMethod();+ if (((List<?>) invocation.getArgument(1)).contains(replacement) && pauseOnce.getAndSet(false)) {+ validated.countDown();+ assertTrue(publish.await(10, TimeUnit.SECONDS));+ }+ return found;+ }).when(ec2Service).describeSubnets(eq(REGION), anyList(), anyMap());+ doAnswer(invocation -> {+ deleting.countDown();+ return invocation.callRealMethod();+ }).when(ec2Service).deleteSubnet(REGION, replacement);+ String arn = existing;+ try (ExecutorService executor = Executors.newFixedThreadPool(2)) {+ Future<String> attach = executor.submit(() -> {+ if (reassign) {+ elb("SetSubnets", "LoadBalancerArn", existing, "Subnets.member.1", first,+ "Subnets.member.2", replacement).statusCode(200);+ return existing;+ }+ return loadBalancer("application", first, replacement);+ });+ try {+ assertTrue(validated.await(10, TimeUnit.SECONDS));+ Future<ValidatableResponse> deletion = executor.submit(+ () -> ec2("DeleteSubnet", "SubnetId", replacement));+ assertTrue(deleting.await(10, TimeUnit.SECONDS));+ assertThrows(TimeoutException.class, () -> deletion.get(200, TimeUnit.MILLISECONDS));+ publish.countDown();+ arn = attach.get(10, TimeUnit.SECONDS);+ deletion.get(10, TimeUnit.SECONDS).statusCode(400)+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));+ } finally {+ publish.countDown();+ }+ } finally {+ reset(ec2Service);+ if (arn != null) {+ elb("DeleteLoadBalancer", "LoadBalancerArn", arn).statusCode(200);+ }+ }+ ec2("DeleteSubnet", "SubnetId", first).statusCode(200);+ ec2("DeleteSubnet", "SubnetId", second).statusCode(200);+ ec2("DeleteSubnet", "SubnetId", replacement).statusCode(200);+ ec2("DeleteVpc", "VpcId", vpc).statusCode(200);+ }++ @Test+ void createCannotPublishAReferenceAfterSubnetDeletionWins() throws Exception {+ String vpc = vpc();+ String first = subnet(vpc, "10.92.1.0/24", "a");+ String second = subnet(vpc, "10.92.2.0/24", "b");+ CountDownLatch checked = new CountDownLatch(1);+ CountDownLatch delete = new CountDownLatch(1);+ CountDownLatch creating = new CountDownLatch(1);+ doAnswer(invocation -> {+ boolean dependent = (boolean) invocation.callRealMethod();+ checked.countDown();+ assertTrue(delete.await(10, TimeUnit.SECONDS));+ return dependent;+ }).when(elbService).hasSubnetDependency(REGION, first);+ doAnswer(invocation -> {+ creating.countDown();+ return invocation.callRealMethod();+ }).when(elbService).createLoadBalancer(eq(REGION), anyString(),+ any(), eq("application"), any(),+ anyList(), any(), anyMap());+ try (ExecutorService executor = Executors.newFixedThreadPool(2)) {+ Future<ValidatableResponse> deletion = executor.submit(() -> ec2("DeleteSubnet", "SubnetId", first));+ try {+ assertTrue(checked.await(10, TimeUnit.SECONDS));+ Future<ValidatableResponse> creation = executor.submit(() -> elb("CreateLoadBalancer", "Name", name(),+ "Type", "application", "Subnets.member.1", first, "Subnets.member.2", second));+ assertTrue(creating.await(10, TimeUnit.SECONDS));+ assertThrows(TimeoutException.class, () -> creation.get(200, TimeUnit.MILLISECONDS));+ delete.countDown();+ deletion.get(10, TimeUnit.SECONDS).statusCode(200);+ creation.get(10, TimeUnit.SECONDS).statusCode(400)+ .body("ErrorResponse.Error.Code", equalTo("SubnetNotFound"));+ } finally {+ delete.countDown();+ }+ } finally {+ reset(elbService);+ }+ ec2("DeleteSubnet", "SubnetId", second).statusCode(200);+ ec2("DeleteVpc", "VpcId", vpc).statusCode(200);+ }++ private String vpc() {+ return ec2("CreateVpc", "CidrBlock", "10.92.0.0/16").statusCode(200).extract()+ .path("CreateVpcResponse.vpc.vpcId");+ }++ private String subnet(String vpc, String cidr, String zone) {+ return ec2("CreateSubnet", "VpcId", vpc, "CidrBlock", cidr,+ "AvailabilityZone", REGION + zone).statusCode(200).extract()+ .path("CreateSubnetResponse.subnet.subnetId");+ }++ private String loadBalancer(String type, String first, String second) {+ return elb("CreateLoadBalancer", "Name", name(), "Type", type,+ "Subnets.member.1", first, "Subnets.member.2", second).statusCode(200).extract()+ .path("CreateLoadBalancerResponse.CreateLoadBalancerResult.LoadBalancers.member.LoadBalancerArn");+ }++ private static String name() {+ return "subnet-" + UUID.randomUUID().toString().substring(0, 20);+ }++ private ValidatableResponse ec2(String action, String... values) {+ return request("ec2", ACCOUNT, REGION, action, values);+ }++ private ValidatableResponse elb(String action, String... values) {+ return request("elasticloadbalancing", ACCOUNT, REGION, action, values);+ }++ private ValidatableResponse request(String service, String account, String region,+ String action, String... values) {+ Map<String, String> parameters = new LinkedHashMap<>();+ parameters.put("Action", action);+ for (int i = 0; i < values.length; i += 2) {+ parameters.put(values[i], values[i + 1]);+ }+ return given().header("Authorization", "AWS4-HMAC-SHA256 Credential=" + account+ + "/20261011/" + region + "/" + service + "/aws4_request")+ .formParams(parameters).when().post("/").then();+ }+}