Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ec2): protect subnets and VPCs that load balancers use (#5663)

Jackson Pricee52016617026
diff --git a/compatibility-tests/sdk-test-awscli/test/cloudformation.bats b/compatibility-tests/sdk-test-awscli/test/cloudformation.bats
index ace02ed7bf..4e0ad28012 100644
--- a/compatibility-tests/sdk-test-awscli/test/cloudformation.bats
+++ b/compatibility-tests/sdk-test-awscli/test/cloudformation.bats
@@ -5,9 +5,15 @@ setup() {
load 'test_helper/common-setup'
STACK_NAME="bats-cfn-stack-$(unique_name)"
TEMPLATE_FILE=$(mktemp /tmp/cfn-bats-XXXXXX.yaml)
+ LB_ARN=""
}
teardown() {
+ # A load balancer a test put on the stack's subnets goes first: EC2 refuses to delete a subnet
+ # a load balancer still uses, so delete-stack could not remove the subnets.
+ if [ -n "$LB_ARN" ]; then
+ aws_cmd elbv2 delete-load-balancer --load-balancer-arn "$LB_ARN" >/dev/null 2>&1 || true
+ fi
aws_cmd cloudformation delete-stack --stack-name "$STACK_NAME" >/dev/null 2>&1 || true
[ -n "$TEMPLATE_FILE" ] && rm -f "$TEMPLATE_FILE"
}
@@ -197,6 +203,7 @@ EOF
assert_success
local lb_arn
lb_arn=$(json_get "$output" '.LoadBalancers[0].LoadBalancerArn')
+ LB_ARN="$lb_arn"
[ -n "$lb_arn" ]
[[ "$lb_arn" == *":loadbalancer/"* ]]
}
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/Ec2ResourceLifecycleTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/Ec2ResourceLifecycleTest.java
new file mode 100644
index 0000000000..cc1dbc1d64
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/Ec2ResourceLifecycleTest.java
@@ -0,0 +1,56 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.ec2.Ec2Client;
+import software.amazon.awssdk.services.ec2.model.Ec2Exception;
+import software.amazon.awssdk.services.elasticloadbalancingv2.ElasticLoadBalancingV2Client;
+import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerSchemeEnum;
+import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerTypeEnum;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+@DisplayName("EC2 subnet and VPC lifecycle dependencies")
+class Ec2ResourceLifecycleTest {
+
+ @Test
+ @DisplayName("DeleteSubnet and DeleteVpc are refused while a load balancer uses the subnet")
+ void subnetAndVpcAreRefusedWhileALoadBalancerUsesThem() {
+ String region = TestFixtures.region().id();
+ try (Ec2Client ec2 = TestFixtures.ec2Client();
+ ElasticLoadBalancingV2Client elb = TestFixtures.elbV2Client()) {
+ String vpcId = ec2.createVpc(r -> r.cidrBlock("10.94.0.0/16")).vpc().vpcId();
+ String first = ec2.createSubnet(r -> r.vpcId(vpcId).cidrBlock("10.94.1.0/24")
+ .availabilityZone(region + "a")).subnet().subnetId();
+ String second = ec2.createSubnet(r -> r.vpcId(vpcId).cidrBlock("10.94.2.0/24")
+ .availabilityZone(region + "b")).subnet().subnetId();
+ String loadBalancerArn = elb.createLoadBalancer(r -> r.name(TestFixtures.uniqueName("lifecycle"))
+ .type(LoadBalancerTypeEnum.APPLICATION).scheme(LoadBalancerSchemeEnum.INTERNAL)
+ .subnets(first, second)).loadBalancers().get(0).loadBalancerArn();
+ try {
+ assertThatThrownBy(() -> ec2.deleteSubnet(r -> r.subnetId(first)))
+ .isInstanceOfSatisfying(Ec2Exception.class, e ->
+ assertThat(e.awsErrorDetails().errorCode()).isEqualTo("DependencyViolation"));
+ assertThatThrownBy(() -> ec2.deleteVpc(r -> r.vpcId(vpcId)))
+ .isInstanceOfSatisfying(Ec2Exception.class, e ->
+ assertThat(e.awsErrorDetails().errorCode()).isEqualTo("DependencyViolation"));
+ } finally {
+ // Cleaned up even when an assertion fails, so nothing leaks into the rest of the run.
+ // A refusal that did not happen has already deleted the subnet, hence quietly.
+ elb.deleteLoadBalancer(r -> r.loadBalancerArn(loadBalancerArn));
+ quietly(() -> ec2.deleteSubnet(r -> r.subnetId(first)));
+ quietly(() -> ec2.deleteSubnet(r -> r.subnetId(second)));
+ quietly(() -> ec2.deleteVpc(r -> r.vpcId(vpcId)));
+ }
+ }
+ }
+
+ private static void quietly(Runnable cleanup) {
+ try {
+ cleanup.run();
+ } catch (Ec2Exception ignored) {
+ // already gone
+ }
+ }
+}
diff --git a/docs/services/ec2.md b/docs/services/ec2.md
index 3ff026f98e..6608d8897b 100644
--- a/docs/services/ec2.md
+++ b/docs/services/ec2.md
@@ -227,7 +227,7 @@ Floci seeds the following resources on first use in each region so Terraform, th
|--------|-------------|
| CreateVpc | Creates a VPC with the requested CIDR block. `InstanceTenancy` accepts `default` or `dedicated`; `host` is rejected with `InvalidParameterValue`, as on AWS. |
| DescribeVpcs | Lists or returns stored VPCs. |
-| DeleteVpc | Deletes a VPC from the local EC2 store, together with its default security group and rules, main route table and default network ACL. Fails with `DependencyViolation` while the VPC still has a subnet, a security group, route table or network ACL other than those defaults, a VPC endpoint, an attached internet gateway or egress-only internet gateway, a transit gateway attachment, or anything still holding a network interface in one of its subnets: an instance that is not terminated, a NAT gateway that is not deleted or failed, a standalone network interface, or an interface endpoint. |
+| DeleteVpc | Deletes a VPC from the local EC2 store, together with its default security group and rules, main route table and default network ACL. Fails with `DependencyViolation` while the VPC still has a subnet, a security group, route table or network ACL other than those defaults, a VPC endpoint, an attached internet gateway or egress-only internet gateway, a transit gateway attachment, an Elastic Load Balancing v2 load balancer, or anything still holding a network interface in one of its subnets: an instance that is not terminated, a NAT gateway that is not deleted or failed, a standalone network interface, or an interface endpoint. |
| ModifyVpcAttribute | Updates supported VPC attributes. |
| DescribeVpcAttribute | Returns a supported VPC attribute. |
| DescribeVpcEndpointServices | Lists common AWS interface endpoint services and S3 (gateway and interface) in every availability zone, named per partition (see [Partitions](../configuration/partitions.md)). A `ServiceName` filter is echoed back as asked. |
@@ -249,7 +249,7 @@ private gateway lifecycles are not modelled.
|--------|-------------|
| CreateSubnet | Creates a subnet in a VPC. |
| DescribeSubnets | Lists or returns stored subnets. |
-| DeleteSubnet | Deletes a subnet from the local EC2 store. Fails with `DependencyViolation` while the subnet still has a transit gateway attachment or anything holding a network interface in it: an instance that is not terminated, a NAT gateway that is not deleted or failed, a standalone network interface, or an interface endpoint. |
+| DeleteSubnet | Deletes a subnet from the local EC2 store. Fails with `DependencyViolation` while the subnet still has a transit gateway attachment, an Elastic Load Balancing v2 load balancer, or anything holding a network interface in it: an instance that is not terminated, a NAT gateway that is not deleted or failed, a standalone network interface, or an interface endpoint. |
| ModifySubnetAttribute | Updates supported subnet attributes. |
### Security Groups
diff --git a/docs/services/elb.md b/docs/services/elb.md
index d493277c07..78064a5a7c 100644
--- a/docs/services/elb.md
+++ b/docs/services/elb.md
@@ -88,6 +88,7 @@ HTTPS listeners require a default certificate. Supply one when creating a listen
- Load balancer, target group, listener, rule, and tag state is persisted through Floci storage and rebuilt on service startup.
- Load balancers are created in `active` state.
+- EC2 refuses deletion of subnets and VPCs still referenced by load balancers with `DependencyViolation`. These account- and region-scoped dependencies follow persisted load balancer state; changing subnets or deleting the load balancer releases its old references. A CloudFormation update that changes `Subnets` or `SubnetMappings` moves the load balancer the way `SetSubnets` does, and a rolled-back update moves it back. An independent target group does not prevent subnet or VPC deletion.
- HTTP listener sockets are preserved when listener actions change and are restarted only when socket-level settings such as port change.
- Instance targets are resolved through EC2 instance private addresses so local load balancer traffic can reach containers.
- Target health starts in `initial` state with reason `Elb.RegistrationInProgress` and is updated by Floci's health checker when monitoring is active.
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
index 71a0a8c213..3afc5c2635 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
@@ -2276,11 +2276,21 @@ public class CloudFormationService implements ResourceProvider {
// resources are deleted during post-update cleanup.
persistStack(stack);
}
- // Both cleanup paths feed the single final status/reason writer.
- List<UpdateCleanupFailure> cleanupFailures =
- new ArrayList<>(deleteRemovedOrConditionFalseResources(
- stack, resources, conditions, region));
- cleanupFailures.addAll(finishCommittedResourceCleanup(stack, region));
+ // Both cleanup paths feed the single final status/reason writer. A removed resource
+ // whose delete failed is tried once more after the replacement cleanup, as an entity
+ // a replacement displaced can still use it: an old load balancer on a subnet the same
+ // update removed.
+ List<StackResource> retried = deleteRemovedOrConditionFalseResources(
+ stack, resources, conditions, region);
+ List<UpdateCleanupFailure> displacedFailures = finishCommittedResourceCleanup(stack, region);
+ List<UpdateCleanupFailure> cleanupFailures = new ArrayList<>();
+ for (StackResource resource : retried) {
+ UpdateCleanupFailure failure = deleteRemovedResource(stack, resource, region, true);
+ if (failure != null) {
+ cleanupFailures.add(failure);
+ }
+ }
+ cleanupFailures.addAll(displacedFailures);
finishCommittedStackUpdate(stack, cleanupFailures);
return;
}
@@ -2802,8 +2812,8 @@ public class CloudFormationService implements ResourceProvider {
String region) {
// Dependents first, in the template's order rather than the map's, as the update cleanup
// walks it: a resource the update added sits after the resources that depend on it, so
- // reversing the map would delete a subnet the update added before an instance or NAT
- // gateway the update put on it, and EC2 refuses that delete.
+ // reversing the map would delete a subnet the update moved a load balancer onto before the
+ // load balancer is moved back, and EC2 refuses that delete.
List<StackResource> resources = resourcesInCreationOrder(stack, region);
Collections.reverse(resources);
List<String> failures = new ArrayList<>();
@@ -3066,16 +3076,21 @@ public class CloudFormationService implements ResourceProvider {
* as {@code DELETE_FAILED}, so a later {@code DeleteStack} or {@code UpdateStack} can retry the cleanup
* instead of orphaning the backing resource.
*
- * @return one {@link UpdateCleanupFailure} per resource whose physical deletion failed
+ * <p>Dependents go first, in the order {@link #resourcesInCreationOrder} gives rather than the map's:
+ * a subnet an earlier update added sits after the load balancer it moved onto it, and the template
+ * that named what a removed resource uses is gone. A failed delete is not reported here, since the
+ * caller tries it again after the replacement cleanup.
+ *
+ * @return the resources whose physical deletion failed, for {@link #deleteRemovedResource} to try again
*/
- private List<UpdateCleanupFailure> deleteRemovedOrConditionFalseResources(Stack stack, JsonNode resources,
+ private List<StackResource> deleteRemovedOrConditionFalseResources(Stack stack, JsonNode resources,
Map<String, Boolean> conditions, String region) {
if (!resources.isObject()) {
return List.of();
}
- List<UpdateCleanupFailure> failures = new ArrayList<>();
- List<StackResource> ordered = new ArrayList<>(stack.resourcesSnapshot().values());
+ List<StackResource> failed = new ArrayList<>();
+ List<StackResource> ordered = resourcesInCreationOrder(stack, region);
Collections.reverse(ordered);
for (StackResource resource : ordered) {
JsonNode resDef = resources.get(resource.getLogicalId());
@@ -3093,31 +3108,49 @@ public class CloudFormationService implements ResourceProvider {
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_IN_PROGRESS", null);
- try {
- deleteResourcePhysically(resource, region, ownerAccount(stack));
- addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
- resource.getResourceType(), "DELETE_COMPLETE", null);
- stack.getResources().remove(resource.getLogicalId());
- } catch (Exception e) {
- String reason = e.getMessage() != null
- ? e.getMessage()
- : "Resource deletion failed during update cleanup";
- failures.add(new UpdateCleanupFailure(
- resource.getLogicalId(), resource.getPhysicalId(), reason));
- // Keep the resource under stack management as DELETE_FAILED. Removing it here would
- // drop it from DescribeStackResources while the backing resource still exists,
- // orphaning it and preventing a later DeleteStack/UpdateStack from retrying cleanup.
- resource.setStatus("DELETE_FAILED");
- resource.setStatusReason(reason);
- addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
- resource.getResourceType(), "DELETE_FAILED", reason);
- LOG.warnv("Failed to delete removed or condition-disabled {0} ({1}) in stack {2}: {3}",
- resource.getResourceType(), resource.getPhysicalId(),
- stack.getStackName(), reason);
+ if (deleteRemovedResource(stack, resource, region, false) != null) {
+ failed.add(resource);
}
}
- return failures;
+ return failed;
+ }
+
+ /**
+ * Deletes one removed or condition-disabled resource, and reports a failure only on the last
+ * attempt.
+ *
+ * @return the failure, or null once the resource is deleted
+ */
+ private UpdateCleanupFailure deleteRemovedResource(Stack stack, StackResource resource, String region,
+ boolean lastAttempt) {
+ try {
+ deleteResourcePhysically(resource, region, ownerAccount(stack));
+ addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
+ resource.getResourceType(), "DELETE_COMPLETE", null);
+ stack.getResources().remove(resource.getLogicalId());
+ return null;
+ } catch (Exception e) {
+ String reason = e.getMessage() != null
+ ? e.getMessage()
+ : "Resource deletion failed during update cleanup";
+ UpdateCleanupFailure failure = new UpdateCleanupFailure(
+ resource.getLogicalId(), resource.getPhysicalId(), reason);
+ if (!lastAttempt) {
+ return failure;
+ }
+ // Keep the resource under stack management as DELETE_FAILED. Removing it here would
+ // drop it from DescribeStackResources while the backing resource still exists,
+ // orphaning it and preventing a later DeleteStack/UpdateStack from retrying cleanup.
+ resource.setStatus("DELETE_FAILED");
+ resource.setStatusReason(reason);
+ addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
+ resource.getResourceType(), "DELETE_FAILED", reason);
+ LOG.warnv("Failed to delete removed or condition-disabled {0} ({1}) in stack {2}: {3}",
+ resource.getResourceType(), resource.getPhysicalId(),
+ stack.getStackName(), reason);
+ return failure;
+ }
}
/**
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisioner.java
index 6726e5bd95..37295343b1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisioner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisioner.java
@@ -36,6 +36,8 @@ public class ElbV2CfnProvisioner implements CfnResourceProvisioner {
private static final String TARGET_GROUP = "AWS::ElasticLoadBalancingV2::TargetGroup";
private static final String LISTENER = "AWS::ElasticLoadBalancingV2::Listener";
private static final String LISTENER_RULE = "AWS::ElasticLoadBalancingV2::ListenerRule";
+ /** Prefixes one uses attribute per subnet a load balancer is on, numbered from 1. */
+ private static final String USED_SUBNET_ATTR_PREFIX = CfnRollback.USES_ATTR_PREFIX + "Subnet.";
private final ElbV2Service elbV2Service;
@@ -141,8 +143,9 @@ public class ElbV2CfnProvisioner implements CfnResourceProvisioner {
return true;
}
String arn = resource.getPhysicalId();
- elbV2Service.setSubnets(AwsArnUtils.parse(arn).region(), arn,
- prior.isEmpty() ? List.of() : List.of(prior.split(",")));
+ List<String> priorSubnets = prior.isEmpty() ? List.of() : List.of(prior.split(","));
+ elbV2Service.setSubnets(AwsArnUtils.parse(arn).region(), arn, priorSubnets);
+ recordUsedSubnets(resource, priorSubnets);
resource.getAttributes().remove(CfnRollback.LOAD_BALANCER_SUBNETS_UPDATE_SNAPSHOT_ATTR);
return true;
}
@@ -165,17 +168,19 @@ public class ElbV2CfnProvisioner implements CfnResourceProvisioner {
Map<String, String> tags = ctx.resolveTags(props, "Tags");
LoadBalancer lb;
+ List<String> onSubnets;
try {
lb = elbV2Service.createLoadBalancer(ctx.region(), name, scheme, type, ipAddressType,
subnets, securityGroups, tags);
+ onSubnets = subnetIds(lb);
} catch (AwsException e) {
if (!"DuplicateLoadBalancerName".equals(e.getErrorCode())) {
throw e;
}
lb = elbV2Service.describeLoadBalancers(ctx.region(), null, List.of(name), null, null).get(0);
- if (ctx.reusesPriorEntity(lb.getLoadBalancerArn())) {
- moveToTemplateSubnets(r, lb, subnets, ctx);
- }
+ onSubnets = ctx.reusesPriorEntity(lb.getLoadBalancerArn())
+ ? moveToTemplateSubnets(r, lb, subnets, ctx)
+ : subnetIds(lb);
}
r.setPhysicalId(lb.getLoadBalancerArn());
@@ -184,6 +189,23 @@ public class ElbV2CfnProvisioner implements CfnResourceProvisioner {
r.getAttributes().put("CanonicalHostedZoneID", lb.getCanonicalHostedZoneId());
r.getAttributes().put("LoadBalancerName", lb.getLoadBalancerName());
r.getAttributes().put("LoadBalancerFullName", loadBalancerFullName(lb.getLoadBalancerArn()));
+ recordUsedSubnets(r, onSubnets);
+ }
+
+ private static List<String> subnetIds(LoadBalancer lb) {
+ return lb.getAvailabilityZones().stream().map(AvailabilityZone::getSubnetId).toList();
+ }
+
+ /**
+ * Records the subnets the balancer is on as uses attributes. EC2 refuses to delete a subnet a
+ * load balancer still uses, and once the template no longer names the balancer, as when an
+ * update removes it together with a subnet it is on, these are what orders it before them.
+ */
+ private static void recordUsedSubnets(StackResource r, List<String> subnetIds) {
+ r.getAttributes().keySet().removeIf(key -> key.startsWith(USED_SUBNET_ATTR_PREFIX));
+ for (int i = 0; i < subnetIds.size(); i++) {
+ r.getAttributes().put(USED_SUBNET_ATTR_PREFIX + (i + 1), subnetIds.get(i));
+ }
}
/**
@@ -222,18 +244,21 @@ public class ElbV2CfnProvisioner implements CfnResourceProvisioner {
}
/**
- * {@code Subnets} is not create-only, so an update that keeps the balancer moves it onto the
- * template's subnets in place, as SetSubnets does, rather than leaving it on a subnet the template
- * dropped or replaced and the update cleanup then deletes. The prior list is kept for a rollback.
+ * {@code Subnets} and {@code SubnetMappings} are not create-only, so an update that keeps the
+ * balancer moves it onto the template's subnets in place, as SetSubnets does. That also releases
+ * the subnets it leaves, which EC2 refuses to delete while a load balancer still uses them, so the
+ * update cleanup can delete a subnet the template dropped or replaced. The prior list is kept for
+ * a rollback. Returns the subnets the balancer is on afterwards.
*/
- private void moveToTemplateSubnets(StackResource r, LoadBalancer lb, List<String> subnets,
- ProvisionContext ctx) {
- List<String> current = lb.getAvailabilityZones().stream().map(AvailabilityZone::getSubnetId).toList();
+ private List<String> moveToTemplateSubnets(StackResource r, LoadBalancer lb, List<String> subnets,
+ ProvisionContext ctx) {
+ List<String> current = subnetIds(lb);
if (subnets.isEmpty() || Set.copyOf(subnets).equals(Set.copyOf(current))) {
- return;
+ return current;
}
elbV2Service.setSubnets(ctx.region(), lb.getLoadBalancerArn(), subnets);
r.getAttributes().put(CfnRollback.LOAD_BALANCER_SUBNETS_UPDATE_SNAPSHOT_ATTR, String.join(",", current));
+ return subnets;
}
private void provisionTargetGroup(StackResource r, JsonNode props, ProvisionContext ctx) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
index 286f58a3b8..ad074f7a0b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
@@ -135,6 +135,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.BiPredicate;
import java.util.function.Function;
+import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -313,6 +314,9 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
private final List<VpcRouteTableListener> routeTableListeners = new CopyOnWriteArrayList<>();
private jakarta.enterprise.inject.Instance<Ec2InstanceLaunchListener> instanceLaunchListenersInstance;
private final List<Ec2InstanceLaunchListener> instanceLaunchListeners = new CopyOnWriteArrayList<>();
+ // Fully qualified to tell the CDI Instance from the EC2 model Instance. Only the CDI constructor
+ // receives it; null on the hand-built constructors, which reads as "nothing else uses a subnet".
+ private jakarta.enterprise.inject.Instance<VpcDependencyProvider> vpcDependencyProviders;
public void addRouteTableListener(VpcRouteTableListener listener) {
if (listener != null && !this.routeTableListeners.contains(listener)) {
@@ -444,7 +448,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
Ec2VolumeBlockDeviceManager volumeBlockDeviceManager,
jakarta.enterprise.inject.Instance<VpcRouteTableListener> routeTableListenersInstance,
jakarta.enterprise.inject.Instance<Ec2InstanceLaunchListener> instanceLaunchListenersInstance,
- Ec2EbsEncryptionService ebsEncryptionService) {
+ Ec2EbsEncryptionService ebsEncryptionService,
+ jakarta.enterprise.inject.Instance<VpcDependencyProvider> vpcDependencyProviders) {
this(config, containerManager, portForwardManager, amiImageResolver, imageCatalog,
instanceTypeCatalog, storageFactory, requestContextInstance, iamService, volumeBlockDeviceManager);
this.vpcNetworkManager = vpcNetworkManager;
@@ -452,6 +457,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
this.routeTableListenersInstance = routeTableListenersInstance;
this.instanceLaunchListenersInstance = instanceLaunchListenersInstance;
this.ebsEncryptionService = ebsEncryptionService;
+ this.vpcDependencyProviders = vpcDependencyProviders;
}
public Ec2Service(EmulatorConfig config, Ec2ContainerManager containerManager,
@@ -2439,6 +2445,32 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
return lockFor(key(region, "transit-gateway-attachments"));
}
+ /**
+ * Runs an action that checks subnets and then records a reference to them, holding the lock
+ * {@link #deleteSubnet} and {@link #deleteVpc} take while they ask each
+ * {@link VpcDependencyProvider} whether anything still uses what they delete. Without it the
+ * check and the write straddle the delete: the delete sees no reference because the referring
+ * resource is not saved yet, removes the subnet, and the resource is then saved pointing at a
+ * subnet that is gone.
+ *
+ * <p>Safe to call from inside a provider's own monitor, because the ordering only ever runs one
+ * way. The deletes hold this lock while calling the providers, and their dependency checks are
+ * plain reads over the provider's own store that take no provider monitor, so EC2 never waits on
+ * a provider while a provider waits on EC2.
+ */
+ public <T> T supplyWithSubnetReferencesHeld(String region, Supplier<T> action) {
+ synchronized (attachmentTopologyLock(region)) {
+ return action.get();
+ }
+ }
+
+ /** {@link #supplyWithSubnetReferencesHeld} for an action with no result. */
+ public void runWithSubnetReferencesHeld(String region, Runnable action) {
+ synchronized (attachmentTopologyLock(region)) {
+ action.run();
+ }
+ }
+
private TransitGatewayVpcAttachmentOptions resolveAttachmentOptions(
TransitGatewayVpcAttachmentOptions requested) {
TransitGatewayVpcAttachmentOptions options = new TransitGatewayVpcAttachmentOptions();
@@ -5346,6 +5378,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
requireNoTransitGatewayAttachment(region,
attachment -> vpcId.equals(attachment.getVpcId()),
"The vpc '" + vpcId + "' has dependencies and cannot be deleted.");
+ requireNoProviderDependency(provider -> provider.hasVpcDependency(region, vpcId),
+ "The vpc '" + vpcId + "' has dependencies and cannot be deleted.");
requireNoVpcDependents(region, vpcId);
deleteVpcDefaultResources(region, vpcId);
vpcs.delete(key(region, vpcId));
@@ -6566,6 +6600,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
"The subnet '" + subnetId + "' has dependencies and cannot be deleted.");
requireNoSubnetResidents(region, Set.of(subnetId),
"The subnet '" + subnetId + "' has dependencies and cannot be deleted.");
+ requireNoProviderDependency(provider -> provider.hasSubnetDependency(region, subnetId),
+ "The subnet '" + subnetId + "' has dependencies and cannot be deleted.");
subnets.delete(key(region, subnetId));
}
if (vpcNetworkManager != null) {
@@ -6588,6 +6624,18 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
}
}
+ /**
+ * Refuses to delete a subnet or VPC that another service still uses, such as a load balancer.
+ * Called under {@link #attachmentTopologyLock}, which those services hold through
+ * {@link #supplyWithSubnetReferencesHeld} while they record a reference, so no reference can
+ * appear between this check and the delete.
+ */
+ private void requireNoProviderDependency(Predicate<VpcDependencyProvider> dependsOnIt, String message) {
+ if (vpcDependencyProviders != null && vpcDependencyProviders.stream().anyMatch(dependsOnIt)) {
+ throw new AwsException("DependencyViolation", message, 400);
+ }
+ }
+
public void modifySubnetAttribute(String region, String subnetId, String attribute, String value) {
ensureDefaultResources(region);
Subnet subnet = requireSubnet(region, subnetId);
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/VpcDependencyProvider.java b/src/main/java/io/github/hectorvent/floci/services/ec2/VpcDependencyProvider.java
new file mode 100644
index 0000000000..564784c7fd
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/VpcDependencyProvider.java
@@ -0,0 +1,16 @@
+package io.github.hectorvent.floci.services.ec2;
+
+/**
+ * Implemented by services whose resources reference EC2 subnets or VPCs, so DeleteSubnet and
+ * DeleteVpc refuse to remove one that is still in use. Answers come from the owning service's
+ * current, account-scoped state, as plain reads that take no lock of that service.
+ *
+ * <p>A service records a reference inside {@link Ec2Service#supplyWithSubnetReferencesHeld}, which
+ * holds the lock EC2 holds while it asks these questions and deletes, so a delete cannot pass its
+ * check while a reference is being published.
+ */
+public interface VpcDependencyProvider {
+ boolean hasSubnetDependency(String region, String subnetId);
+
+ boolean hasVpcDependency(String region, String vpcId);
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java b/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java
index b65a32a5a3..2c1aeb972c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java
@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.core.resource.SupportedResourceType;
import io.github.hectorvent.floci.core.storage.StorageBackedMap;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.ec2.VpcDependencyProvider;
import io.github.hectorvent.floci.services.ec2.model.Subnet;
import io.github.hectorvent.floci.services.elbv2.model.*;
import io.github.hectorvent.floci.services.iam.ServerCertificateReferenceProvider;
@@ -34,7 +35,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@ApplicationScoped
-public class ElbV2Service implements ResourceProvider, ServerCertificateReferenceProvider {
+public class ElbV2Service implements ResourceProvider, ServerCertificateReferenceProvider, VpcDependencyProvider {
@Inject
ElbV2DataPlane dataPlane;
@@ -195,10 +196,23 @@ public class ElbV2Service implements ResourceProvider, ServerCertificateReferenc
// ── Load Balancers ────────────────────────────────────────────────────────
+ /**
+ * Resolving the subnets and publishing the balancer happen under EC2's subnet lock, so a
+ * DeleteSubnet cannot pass its dependency check in between and leave the balancer on a subnet
+ * that is gone.
+ */
public LoadBalancer createLoadBalancer(String region, String name, String scheme,
String type, String ipAddressType,
List<String> subnets, List<String> securityGroups,
Map<String, String> initialTags) {
+ return ec2Service.supplyWithSubnetReferencesHeld(region, () -> createLoadBalancerWithSubnetsHeld(
+ region, name, scheme, type, ipAddressType, subnets, securityGroups, initialTags));
+ }
+
+ private LoadBalancer createLoadBalancerWithSubnetsHeld(String region, String name, String scheme,
+ String type, String ipAddressType,
+ List<String> subnets, List<String> securityGroups,
+ Map<String, String> initialTags) {
validateName(name, "load balancer");
Map<String, LoadBalancer> regionLbs = loadBalancers.computeIfAbsent(region, k -> new ConcurrentHashMap<>());
boolean duplicate = regionLbs.values().stream()
@@ -241,6 +255,23 @@ public class ElbV2Service implements ResourceProvider, ServerCertificateReferenc
return lb;
}
+ /**
+ * A subnet is in use while a load balancer is configured with it. Account-scoped through the
+ * request-scoped storage, and a target group does not count: it pins nothing in AWS either.
+ */
+ @Override
+ public boolean hasSubnetDependency(String region, String subnetId) {
+ return loadBalancers.getOrDefault(region, Map.of()).values().stream()
+ .anyMatch(lb -> lb.getAvailabilityZones().stream()
+ .anyMatch(zone -> subnetId.equals(zone.getSubnetId())));
+ }
+
+ @Override
+ public boolean hasVpcDependency(String region, String vpcId) {
+ return loadBalancers.getOrDefault(region, Map.of()).values().stream()
+ .anyMatch(lb -> vpcId.equals(lb.getVpcId()));
+ }
+
private String loadBalancerDnsSuffix() {
if (config == null) {
return EmbeddedDnsServer.DEFAULT_SUFFIX;
@@ -336,18 +367,21 @@ public class ElbV2Service implements ResourceProvider, ServerCertificateReferenc
persistRegion(loadBalancers, region);
}
+ /** Under EC2's subnet lock for the same reason as {@link #createLoadBalancer}. */
public void setSubnets(String region, String arn, List<String> subnets) {
- LoadBalancer lb = requireLoadBalancer(region, arn);
- String vpcId = resolveSubnetVpcId(region, lb.getType(), subnets);
- if (vpcId != null && lb.getVpcId() != null && !lb.getVpcId().equals(vpcId)) {
- throw new AwsException("InvalidConfigurationRequest",
- "All subnets must belong to the load balancer VPC.", 400);
- }
- lb.setAvailabilityZones(resolveAvailabilityZones(region, subnets));
- if (vpcId != null) {
- lb.setVpcId(vpcId);
- }
- persistRegion(loadBalancers, region);
+ ec2Service.runWithSubnetReferencesHeld(region, () -> {
+ LoadBalancer lb = requireLoadBalancer(region, arn);
+ String vpcId = resolveSubnetVpcId(region, lb.getType(), subnets);
+ if (vpcId != null && lb.getVpcId() != null && !lb.getVpcId().equals(vpcId)) {
+ throw new AwsException("InvalidConfigurationRequest",
+ "All subnets must belong to the load balancer VPC.", 400);
+ }
+ lb.setAvailabilityZones(resolveAvailabilityZones(region, subnets));
+ if (vpcId != null) {
+ lb.setVpcId(vpcId);
+ }
+ persistRegion(loadBalancers, region);
+ });
}
public void setIpAddressType(String region, String arn, String ipAddressType) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationElbV2SubnetUpdateIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationElbV2SubnetUpdateIntegrationTest.java
new file mode 100644
index 0000000000..657165a780
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationElbV2SubnetUpdateIntegrationTest.java
@@ -0,0 +1,687 @@
+package io.github.hectorvent.floci.services.cloudformation;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.XmlParser;
+import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.mockito.InjectSpy;
+import io.restassured.response.ValidatableResponse;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.reset;
+
+/**
+ * Stack updates that move an ELBv2 load balancer between subnets, through the whole engine. EC2
+ * refuses to delete a subnet a load balancer still uses, so an update has to move the balancer off
+ * a subnet the template dropped or replaced before its cleanup deletes that subnet, and a rolled-back
+ * update has to move the balancer back before it deletes a subnet the update added.
+ */
+@QuarkusTest
+class CloudFormationElbV2SubnetUpdateIntegrationTest {
+
+ private static final String CFN_AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20261013/us-east-1/cloudformation/aws4_request";
+ private static final String EC2_AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20261013/us-east-1/ec2/aws4_request";
+ private static final String ELB_AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20261013/us-east-1/elasticloadbalancing/aws4_request";
+ private static final String CIDR_A = "10.93.1.0/24";
+ private static final String CIDR_B = "10.93.2.0/24";
+ private static final String CIDR_C = "10.93.3.0/24";
+
+ @InjectSpy
+ ElbV2Service elbService;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void anUpdateThatDropsASubnetMovesTheLoadBalancerOffItSoTheCleanupDeletesIt() {
+ String stack = "alb-drop-subnet-" + Long.toString(System.nanoTime(), 36);
+ try {
+ Map<String, String> created = createStack(stack,
+ template(CIDR_B, true, List.of("SubnetA", "SubnetB", "SubnetC"), ""));
+
+ Map<String, String> updated = updateStack(stack,
+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""), "UPDATE_COMPLETE");
+
+ assertEquals(Set.of(updated.get("SubnetA"), updated.get("SubnetB")),
+ Set.copyOf(loadBalancerSubnets(updated.get("Alb"))));
+ ec2("DescribeSubnets", "SubnetId.1", created.get("SubnetC")).statusCode(400)
+ .body(containsString("InvalidSubnetID.NotFound"));
+ } finally {
+ deleteStack(stack);
+ }
+ }
+
+ @Test
+ void anUpdateThatReplacesASubnetMovesTheLoadBalancerOntoTheReplacement() {
+ String stack = "alb-replace-subnet-" + Long.toString(System.nanoTime(), 36);
+ try {
+ String oldSubnetB = createStack(stack,
+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), "")).get("SubnetB");
+
+ // A new CidrBlock is create-only, so SubnetB is replaced and the old one is deleted on commit.
+ Map<String, String> updated = updateStack(stack,
+ template("10.93.4.0/24", false, List.of("SubnetA", "SubnetB"), ""), "UPDATE_COMPLETE");
+
+ String newSubnetB = updated.get("SubnetB");
+ assertNotEquals(oldSubnetB, newSubnetB);
+ assertEquals(Set.of(updated.get("SubnetA"), newSubnetB),
+ Set.copyOf(loadBalancerSubnets(updated.get("Alb"))));
+ ec2("DescribeSubnets", "SubnetId.1", oldSubnetB).statusCode(400)
+ .body(containsString("InvalidSubnetID.NotFound"));
+ } finally {
+ deleteStack(stack);
+ }
+ }
+
+ /**
+ * An earlier update added SubnetC and moved the load balancer onto it, so SubnetC was recorded
+ * after the load balancer. An update that removes both has to delete the load balancer first,
+ * though the template that named what it uses is gone.
+ */
+ @Test
+ void anUpdateThatRemovesTheLoadBalancerAndASubnetItUsesDeletesTheLoadBalancerFirst() {
+ String stack = "alb-remove-both-" + Long.toString(System.nanoTime(), 36);
+ try {
+ createStack(stack, template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));
+ Map<String, String> moved = updateStack(stack,
+ template(CIDR_B, true, List.of("SubnetA", "SubnetB", "SubnetC"), ""), "UPDATE_COMPLETE");
+ assertEquals(3, loadBalancerSubnets(moved.get("Alb")).size());
+
+ String withoutEither = """
+ {
+ "Resources": {
+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.93.0.0/16"}},
+ "SubnetA": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1a"}
+ },
+ "SubnetB": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1b"}
+ }
+ }
+ }
+ """.formatted(CIDR_A, CIDR_B);
+ request(CFN_AUTH, "UpdateStack", "StackName", stack, "TemplateBody", withoutEither).statusCode(200);
+ CfnStackWaits.StackState updated = CfnStackWaits.awaitStatus(stack, "UPDATE_COMPLETE");
+
+ assertTrue(updated.reason() == null || updated.reason().isEmpty(), updated.reason());
+ List<String> events = events(stack);
+ assertTrue(events.contains("Alb DELETE_COMPLETE"), String.join(", ", events));
+ assertTrue(events.indexOf("Alb DELETE_COMPLETE") < events.indexOf("SubnetC DELETE_IN_PROGRESS"),
+ String.join(", ", events));
+ ec2("DescribeSubnets", "SubnetId.1", moved.get("SubnetC")).statusCode(400)
+ .body(containsString("InvalidSubnetID.NotFound"));
+ request(ELB_AUTH, "DescribeLoadBalancers", "LoadBalancerArns.member.1", moved.get("Alb")).statusCode(400)
+ .body(containsString("LoadBalancerNotFound"));
+ } finally {
+ deleteStack(stack);
+ }
+ }
+
+ /**
+ * A new name replaces the load balancer, and the old one stays on SubnetC until the update's
+ * cleanup deletes it. The same update drops SubnetC, so the cleanup has to delete the old load
+ * balancer before SubnetC.
+ */
+ @Test
+ void anUpdateThatReplacesTheLoadBalancerAndDropsASubnetTheOldOneUsesDeletesTheOldOneFirst() {
+ String stack = "alb-rename-drop-" + Long.toString(System.nanoTime(), 36);
+ String suffix = stack.substring(stack.length() - 8);
+ try {
+ Map<String, String> created = createStack(stack, named(
+ template(CIDR_B, true, List.of("SubnetA", "SubnetB", "SubnetC"), ""), "old-" + suffix));
+
+ request(CFN_AUTH, "UpdateStack", "StackName", stack, "TemplateBody",
+ named(template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""), "new-" + suffix))
+ .statusCode(200);
+ CfnStackWaits.StackState updated = CfnStackWaits.awaitStatus(stack, "UPDATE_COMPLETE");
+
+ assertTrue(updated.reason() == null || updated.reason().isEmpty(), updated.reason());
+ List<String> events = events(stack);
+ assertTrue(events.contains("Alb DELETE_COMPLETE") && events.contains("SubnetC DELETE_COMPLETE"),
+ String.join(", ", events));
+ assertTrue(events.lastIndexOf("Alb DELETE_COMPLETE") < events.indexOf("SubnetC DELETE_COMPLETE"),
+ String.join(", ", events));
+ Map<String, String> outputs = outputs(stack);
+ assertNotEquals(created.get("Alb"), outputs.get("Alb"));
+ request(ELB_AUTH, "DescribeLoadBalancers", "LoadBalancerArns.member.1", created.get("Alb")).statusCode(400)
+ .body(containsString("LoadBalancerNotFound"));
+ ec2("DescribeSubnets", "SubnetId.1", created.get("SubnetC")).statusCode(400)
+ .body(containsString("InvalidSubnetID.NotFound"));
+ } finally {
+ deleteStack(stack);
+ }
+ }
+
+ @Test
+ void aFailedUpdateMovesTheLoadBalancerBackToItsSubnets() {
+ String stack = "alb-subnets-rollback-" + Long.toString(System.nanoTime(), 36);
+ try {
+ Map<String, String> created = createStack(stack,
+ template(CIDR_B, true, List.of("SubnetA", "SubnetB", "SubnetC"), ""));
+ Set<String> before = Set.copyOf(loadBalancerSubnets(created.get("Alb")));
+
+ Map<String, String> rolledBack = updateStack(stack, template(CIDR_B, true,
+ List.of("SubnetA", "SubnetB"), failingSecret(stack, "Alb")), "UPDATE_ROLLBACK_COMPLETE");
+
+ assertEquals(3, before.size());
+ assertEquals(before, Set.copyOf(loadBalancerSubnets(rolledBack.get("Alb"))));
+ } finally {
+ deleteStack(stack);
+ }
+ }
+
+ /**
+ * The update adds SubnetC and moves the load balancer onto it before a later resource fails.
+ * The rollback has to move the load balancer back before it deletes the subnet it added, or EC2
+ * refuses the delete and the rollback fails.
+ */
+ @Test
+ void aFailedUpdateMovesTheLoadBalancerOffASubnetItAddedBeforeDeletingIt() {
+ String stack = "alb-added-subnet-rollback-" + Long.toString(System.nanoTime(), 36);
+ try {
+ Map<String, String> created = createStack(stack,
+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));
+ Set<String> before = Set.of(created.get("SubnetA"), created.get("SubnetB"));
+
+ Map<String, String> rolledBack = updateStack(stack, template(CIDR_B, true,
+ List.of("SubnetA", "SubnetC"), failingSecret(stack, "Alb")), "UPDATE_ROLLBACK_COMPLETE");
+
+ assertEquals(before, Set.copyOf(loadBalancerSubnets(rolledBack.get("Alb"))));
+ assertEquals(before, Set.copyOf(vpcSubnets(created.get("Vpc"))));
+ } finally {
+ deleteStack(stack);
+ }
+ }
+
+ /**
+ * The first rollback's move back fails, which leaves the load balancer on the subnet the update
+ * added, so that subnet cannot be deleted either and the stack ends UPDATE_ROLLBACK_FAILED with
+ * both owed. ContinueUpdateRollback walks them again in dependency order: the load balancer moves
+ * back first, from the subnets it kept for the retry, and then the subnet goes.
+ */
+ @Test
+ void continueUpdateRollbackMovesTheLoadBalancerBackBeforeDeletingTheSubnetItAdded() {
+ String stack = "alb-continue-rollback-" + Long.toString(System.nanoTime(), 36);
+ try {
+ Map<String, String> created = createStack(stack,
+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));
+ Set<String> before = Set.of(created.get("SubnetA"), created.get("SubnetB"));
+ failTheFirstMoveBack();
+
+ failUpdateOntoAnAddedSubnet(stack);
+
+ continueUpdateRollback(stack);
+ CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_COMPLETE");
+ assertEquals(before, Set.copyOf(loadBalancerSubnets(created.get("Alb"))));
+ assertEquals(before, Set.copyOf(vpcSubnets(created.get("Vpc"))));
+ } finally {
+ reset(elbService);
+ deleteStack(stack);
+ }
+ }
+
+ /**
+ * ResourcesToSkip leaves a skipped resource as the failed update left it. Skipping only the load
+ * balancer keeps it on the added subnet, so that subnet still cannot be deleted. Skipping the
+ * subnet as well completes the rollback with both left in place, and the stack still deletes
+ * cleanly, load balancer first.
+ */
+ @Test
+ void continueUpdateRollbackWithResourcesToSkipLeavesTheSkippedLoadBalancerOnTheAddedSubnet() {
+ String stack = "alb-continue-skip-" + Long.toString(System.nanoTime(), 36);
+ try {
+ Map<String, String> created = createStack(stack,
+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));
+ failTheFirstMoveBack();
+ failUpdateOntoAnAddedSubnet(stack);
+ Set<String> addedSubnet = new HashSet<>(vpcSubnets(created.get("Vpc")));
+ addedSubnet.removeAll(Set.of(created.get("SubnetA"), created.get("SubnetB")));
+ assertEquals(1, addedSubnet.size());
+
+ continueUpdateRollback(stack, "Alb");
+ CfnStackWaits.StackState stillFailed = CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_FAILED");
+ assertTrue(stillFailed.reason().contains("SubnetC"), stillFailed.reason());
+
+ continueUpdateRollback(stack, "SubnetC");
+ CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_COMPLETE");
+ assertEquals(Set.of(created.get("SubnetA"), addedSubnet.iterator().next()),
+ Set.copyOf(loadBalancerSubnets(created.get("Alb"))));
+ assertEquals(3, vpcSubnets(created.get("Vpc")).size());
+ } finally {
+ reset(elbService);
+ deleteStack(stack);
+ }
+ }
+
+ /**
+ * The update's own move onto the new subnets goes through, the first move back fails as if its
+ * subnet had been deleted meanwhile, and any later call goes through again.
+ */
+ private void failTheFirstMoveBack() {
+ doCallRealMethod()
+ .doThrow(new AwsException("SubnetNotFound", "The subnet ID was not found", 400))
+ .doCallRealMethod()
+ .when(elbService).setSubnets(anyString(), anyString(), anyList());
+ }
+
+ /** Moves the load balancer onto SubnetA and an added SubnetC in an update a later resource fails. */
+ private static void failUpdateOntoAnAddedSubnet(String stack) {
+ updateStack(stack, template(CIDR_B, true, List.of("SubnetA", "SubnetC"), failingSecret(stack, "Alb")),
+ "UPDATE_ROLLBACK_FAILED");
+ String reason = CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_FAILED").reason();
+ assertTrue(reason.contains("Alb") && reason.contains("SubnetC"), reason);
+ }
+
+ private static void continueUpdateRollback(String stack, String... resourcesToSkip) {
+ String[] values = new String[2 + resourcesToSkip.length * 2];
+ values[0] = "StackName";
+ values[1] = stack;
+ for (int i = 0; i < resourcesToSkip.length; i++) {
+ values[2 + i * 2] = "ResourcesToSkip.member." + (i + 1);
+ values[3 + i * 2] = resourcesToSkip[i];
+ }
+ request(CFN_AUTH, "ContinueUpdateRollback", values).statusCode(200);
+ }
+
+ /**
+ * A failed rollback restores the parameters but can leave a resource on the branch the failed
+ * update chose: here the load balancer stays on the subnets an Fn::If picked for the new
+ * parameter value. Deleting the stack has to delete it before those subnets too, whichever
+ * branch the restored parameters now select.
+ */
+ @Test
+ void deleteStackAfterAFailedRollbackDeletesTheLoadBalancerBeforeTheSubnetsOfEitherBranch() {
+ String stack = "alb-fnif-branch-" + Long.toString(System.nanoTime(), 36);
+ try {
+ Map<String, String> created = createStack(stack, branchTemplate(""), "first");
+ failTheFirstMoveBack();
+ request(CFN_AUTH, "UpdateStack", "StackName", stack, "TemplateBody",
+ branchTemplate(failingSecret(stack, "Alb")),
+ "Parameters.member.1.ParameterKey", "Placement",
+ "Parameters.member.1.ParameterValue", "second").statusCode(200);
+ String reason = CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_FAILED").reason();
+ assertTrue(reason.contains("Alb"), reason);
+ assertEquals(Set.of(created.get("SubnetC"), created.get("SubnetD")),
+ Set.copyOf(loadBalancerSubnets(created.get("Alb"))), "the move back failed");
+ reset(elbService);
+
+ deleteStack(stack);
+
+ ec2("DescribeVpcs", "VpcId.1", created.get("Vpc")).statusCode(400)
+ .body(containsString("InvalidVpcID.NotFound"));
+ } finally {
+ reset(elbService);
+ if (request(CFN_AUTH, "DescribeStacks", "StackName", stack).extract().statusCode() == 200) {
+ request(CFN_AUTH, "DeleteStack", "StackName", stack);
+ }
+ }
+ }
+
+ /** CloudFormation deletes a VPC too, so the load balancer has to go before its subnets and they before the VPC. */
+ @Test
+ void deleteStackRemovesTheLoadBalancerBeforeTheSubnetsAndTheVpcUnderIt() {
+ String stack = "alb-vpc-delete-" + Long.toString(System.nanoTime(), 36);
+ Map<String, String> created = createStack(stack, template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));
+
+ deleteStack(stack);
+
+ ec2("DescribeVpcs", "VpcId.1", created.get("Vpc")).statusCode(400)
+ .body(containsString("InvalidVpcID.NotFound"));
+ request(ELB_AUTH, "DescribeLoadBalancers", "LoadBalancerArns.member.1", created.get("Alb")).statusCode(400)
+ .body(containsString("LoadBalancerNotFound"));
+ }
+
+ /**
+ * A load balancer outside the stack on the stack's subnets pins them, and through them the VPC,
+ * so DeleteStack fails until that load balancer is gone, and then deletes everything.
+ */
+ @Test
+ void aLoadBalancerOutsideTheStackKeepsItsSubnetsAndVpcUntilItIsDeleted() {
+ String stack = "alb-vpc-outside-" + Long.toString(System.nanoTime(), 36);
+ Map<String, String> created = createStack(stack, template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));
+ try {
+ String outside = request(ELB_AUTH, "CreateLoadBalancer",
+ "Name", "outside-" + stack.substring(stack.length() - 8), "Type", "application",
+ "Scheme", "internal",
+ "Subnets.member.1", created.get("SubnetA"), "Subnets.member.2", created.get("SubnetB"))
+ .statusCode(200).extract()
+ .path("CreateLoadBalancerResponse.CreateLoadBalancerResult.LoadBalancers.member.LoadBalancerArn");
+ try {
+ request(CFN_AUTH, "DeleteStack", "StackName", stack).statusCode(200);
+ CfnStackWaits.StackState failed = CfnStackWaits.awaitTerminal(stack, CFN_AUTH);
+ assertEquals("DELETE_FAILED", failed.status());
+ String reason = failed.reason();
+ assertEquals(Set.of("SubnetA", "SubnetB", "Vpc"),
+ Set.of(reason.substring(reason.indexOf('[') + 1, reason.indexOf(']')).split(", ")), reason);
+ } finally {
+ request(ELB_AUTH, "DeleteLoadBalancer", "LoadBalancerArn", outside).statusCode(200);
+ }
+ } finally {
+ deleteStack(stack);
+ }
+
+ ec2("DescribeVpcs", "VpcId.1", created.get("Vpc")).statusCode(400)
+ .body(containsString("InvalidVpcID.NotFound"));
+ }
+
+ /**
+ * An update that adds a VPC, subnets in it and a load balancer on them, then fails, rolls them
+ * back dependents first: the load balancer before its subnets, and the subnets before the VPC,
+ * so nothing the update added is left behind.
+ */
+ @Test
+ void aFailedUpdateDeletesTheLoadBalancerItAddedBeforeTheSubnetsAndVpcUnderIt() {
+ String stack = "alb-vpc-rollback-" + Long.toString(System.nanoTime(), 36);
+ try {
+ Map<String, String> created = createStack(stack,
+ template(CIDR_B, false, List.of("SubnetA", "SubnetB"), ""));
+ String added = """
+ ,
+ "Vpc2": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.95.0.0/16",
+ "Tags": [{"Key": "floci-it", "Value": "%1$s"}]}},
+ "SubnetX": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc2"}, "CidrBlock": "10.95.1.0/24", "AvailabilityZone": "us-east-1a"}
+ },
+ "SubnetY": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc2"}, "CidrBlock": "10.95.2.0/24", "AvailabilityZone": "us-east-1b"}
+ },
+ "Alb2": {
+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
+ "Properties": {"Type": "application", "Scheme": "internal",
+ "Subnets": [{"Ref": "SubnetX"}, {"Ref": "SubnetY"}]}
+ }""".formatted(stack);
+
+ updateStack(stack, template(CIDR_B, false, List.of("SubnetA", "SubnetB"),
+ added + failingSecret(stack, "Alb2")), "UPDATE_ROLLBACK_COMPLETE");
+
+ String vpcs = ec2("DescribeVpcs", "Filter.1.Name", "tag:floci-it", "Filter.1.Value.1", stack)
+ .statusCode(200).extract().asString();
+ assertEquals(List.of(), XmlParser.extractAll(vpcs, "vpcId"));
+ assertEquals(Set.of(created.get("SubnetA"), created.get("SubnetB")),
+ Set.copyOf(loadBalancerSubnets(created.get("Alb"))));
+ } finally {
+ deleteStack(stack);
+ }
+ }
+
+ /**
+ * The rollback orders resources by the template's conditions, evaluated with the SSM-resolved
+ * parameter value the update itself used. With the raw value, the parameter's name, the load
+ * balancer's condition reads false and the subnet it uses would be deleted first.
+ */
+ @Test
+ void aRollbackOrdersAResourceUnderAnSsmResolvedConditionBeforeTheSubnetItUses() {
+ String stack = "alb-ssm-condition-rollback-" + Long.toString(System.nanoTime(), 36);
+ String parameterName = "/" + stack + "/env";
+ ssm("PutParameter", "{\"Name\": \"" + parameterName + "\", \"Value\": \"prod\", \"Type\": \"String\"}");
+ try {
+ Map<String, String> created = createStack(stack, ssmConditionTemplate(parameterName, ""));
+ Set<String> before = Set.of(created.get("SubnetA"), created.get("SubnetB"));
+
+ String added = """
+ ,
+ "SubnetC": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1c"}
+ },
+ "ProdAlb": {
+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
+ "Condition": "IsProd",
+ "Properties": {"Type": "application", "Scheme": "internal",
+ "Subnets": [{"Ref": "SubnetA"}, {"Ref": "SubnetC"}]}
+ },
+ "BadSecret": {
+ "Type": "AWS::SecretsManager::Secret",
+ "Condition": "IsProd",
+ "DependsOn": "ProdAlb",
+ "Properties": {
+ "Name": "%s-secret",
+ "SecretString": "explicit",
+ "GenerateSecretString": {"PasswordLength": 32}
+ }
+ }""".formatted(CIDR_C, stack);
+ updateStack(stack, ssmConditionTemplate(parameterName, added), "UPDATE_ROLLBACK_COMPLETE");
+
+ assertEquals(before, Set.copyOf(vpcSubnets(created.get("Vpc"))));
+ } finally {
+ deleteStack(stack);
+ ssm("DeleteParameter", "{\"Name\": \"" + parameterName + "\"}");
+ }
+ }
+
+ /** A secret that sets both secret properties, which fails the update once what it depends on is done. */
+ private static String failingSecret(String stack, String dependsOn) {
+ return """
+ ,
+ "BadSecret": {
+ "Type": "AWS::SecretsManager::Secret",
+ "DependsOn": "%s",
+ "Properties": {
+ "Name": "%s-secret",
+ "SecretString": "explicit",
+ "GenerateSecretString": {"PasswordLength": 32}
+ }
+ }""".formatted(dependsOn, stack);
+ }
+
+ /**
+ * A VPC with subnets in three zones, and an internal ALB on the named ones. SubnetC, its
+ * output and SubnetB's CIDR vary between the versions a test applies.
+ */
+ private static String template(String cidrB, boolean withSubnetC, List<String> albSubnets, String extraResources) {
+ StringBuilder refs = new StringBuilder();
+ for (String subnet : albSubnets) {
+ refs.append(refs.isEmpty() ? "" : ", ").append("{\"Ref\": \"").append(subnet).append("\"}");
+ }
+ String subnetC = withSubnetC ? """
+ "SubnetC": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1c"}
+ },""".formatted(CIDR_C) : "";
+ String subnetCOutput = withSubnetC ? """
+ "SubnetC": {"Value": {"Ref": "SubnetC"}},""" : "";
+ return """
+ {
+ "Resources": {
+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.93.0.0/16"}},
+ "SubnetA": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1a"}
+ },
+ "SubnetB": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1b"}
+ },%s
+ "Alb": {
+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
+ "Properties": {"Type": "application", "Scheme": "internal", "Subnets": [%s]}
+ }%s
+ },
+ "Outputs": {%s
+ "SubnetA": {"Value": {"Ref": "SubnetA"}},
+ "SubnetB": {"Value": {"Ref": "SubnetB"}},
+ "Vpc": {"Value": {"Ref": "Vpc"}},
+ "Alb": {"Value": {"Ref": "Alb"}}
+ }
+ }
+ """.formatted(CIDR_A, cidrB, subnetC, refs, extraResources, subnetCOutput);
+ }
+
+ /** A {@link #template} whose ALB has the given name, which is create-only. */
+ private static String named(String template, String name) {
+ return template.replace("\"Scheme\": \"internal\", \"Subnets\"",
+ "\"Name\": \"" + name + "\", \"Scheme\": \"internal\", \"Subnets\"");
+ }
+
+ /**
+ * A VPC with four subnets and an ALB whose subnets an Fn::If picks by the Placement parameter:
+ * SubnetA and SubnetB for "first", SubnetC and SubnetD for "second". The ALB is declared before
+ * the subnets, and SubnetC is created after SubnetD after SubnetB, so whichever branch the
+ * ordering reads, SubnetC is not created before the ALB unless the ALB depends on it.
+ */
+ private static String branchTemplate(String extraResources) {
+ return """
+ {
+ "Parameters": {"Placement": {"Type": "String", "Default": "first"}},
+ "Conditions": {"Second": {"Fn::Equals": [{"Ref": "Placement"}, "second"]}},
+ "Resources": {
+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.93.0.0/16"}},
+ "Alb": {
+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
+ "Properties": {"Type": "application", "Scheme": "internal",
+ "Subnets": {"Fn::If": ["Second", [{"Ref": "SubnetC"}, {"Ref": "SubnetD"}],
+ [{"Ref": "SubnetA"}, {"Ref": "SubnetB"}]]}}
+ },
+ "SubnetA": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": {"Ref": "Vpc"},
+ "CidrBlock": "10.93.1.0/24", "AvailabilityZone": "us-east-1a"}},
+ "SubnetB": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": {"Ref": "Vpc"},
+ "CidrBlock": "10.93.2.0/24", "AvailabilityZone": "us-east-1b"}},
+ "SubnetC": {"Type": "AWS::EC2::Subnet", "DependsOn": "SubnetD", "Properties": {"VpcId": {"Ref": "Vpc"},
+ "CidrBlock": "10.93.3.0/24", "AvailabilityZone": "us-east-1a"}},
+ "SubnetD": {"Type": "AWS::EC2::Subnet", "DependsOn": "SubnetB", "Properties": {"VpcId": {"Ref": "Vpc"},
+ "CidrBlock": "10.93.4.0/24", "AvailabilityZone": "us-east-1b"}}%s
+ },
+ "Outputs": {
+ "Vpc": {"Value": {"Ref": "Vpc"}},
+ "SubnetC": {"Value": {"Ref": "SubnetC"}},
+ "SubnetD": {"Value": {"Ref": "SubnetD"}},
+ "Alb": {"Value": {"Ref": "Alb"}}
+ }
+ }
+ """.formatted(extraResources);
+ }
+
+ /** A VPC and two subnets, with an IsProd condition on an SSM-typed parameter. */
+ private static String ssmConditionTemplate(String parameterName, String extraResources) {
+ return """
+ {
+ "Parameters": {
+ "Env": {"Type": "AWS::SSM::Parameter::Value<String>", "Default": "%s"}
+ },
+ "Conditions": {
+ "IsProd": {"Fn::Equals": [{"Ref": "Env"}, "prod"]}
+ },
+ "Resources": {
+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.93.0.0/16"}},
+ "SubnetA": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1a"}
+ },
+ "SubnetB": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "%s", "AvailabilityZone": "us-east-1b"}
+ }%s
+ },
+ "Outputs": {
+ "SubnetA": {"Value": {"Ref": "SubnetA"}},
+ "SubnetB": {"Value": {"Ref": "SubnetB"}},
+ "Vpc": {"Value": {"Ref": "Vpc"}}
+ }
+ }
+ """.formatted(parameterName, CIDR_A, CIDR_B, extraResources);
+ }
+
+ private static List<String> loadBalancerSubnets(String arn) {
+ assertNotNull(arn, "the stack outputs the load balancer");
+ String xml = request(ELB_AUTH, "DescribeLoadBalancers", "LoadBalancerArns.member.1", arn)
+ .statusCode(200).extract().asString();
+ return XmlParser.extractAll(xml, "SubnetId");
+ }
+
+ private static List<String> vpcSubnets(String vpcId) {
+ String xml = ec2("DescribeSubnets", "Filter.1.Name", "vpc-id", "Filter.1.Value.1", vpcId)
+ .statusCode(200).extract().asString();
+ return XmlParser.extractAll(xml, "subnetId");
+ }
+
+ private static Map<String, String> createStack(String stack, String templateBody, String placement) {
+ request(CFN_AUTH, "CreateStack", "StackName", stack, "TemplateBody", templateBody,
+ "Parameters.member.1.ParameterKey", "Placement",
+ "Parameters.member.1.ParameterValue", placement).statusCode(200);
+ CfnStackWaits.awaitStatus(stack, "CREATE_COMPLETE");
+ return outputs(stack);
+ }
+
+ private static Map<String, String> createStack(String stack, String templateBody) {
+ request(CFN_AUTH, "CreateStack", "StackName", stack, "TemplateBody", templateBody).statusCode(200);
+ CfnStackWaits.awaitStatus(stack, "CREATE_COMPLETE");
+ return outputs(stack);
+ }
+
+ private static Map<String, String> updateStack(String stack, String templateBody, String expectedStatus) {
+ request(CFN_AUTH, "UpdateStack", "StackName", stack, "TemplateBody", templateBody).statusCode(200);
+ CfnStackWaits.awaitStatus(stack, expectedStatus);
+ return outputs(stack);
+ }
+
+ /** The stack's events, oldest first, each as its logical id and status. */
+ private static List<String> events(String stack) {
+ String xml = request(CFN_AUTH, "DescribeStackEvents", "StackName", stack).statusCode(200)
+ .extract().asString();
+ List<String> ids = XmlParser.extractAll(xml, "LogicalResourceId");
+ List<String> statuses = XmlParser.extractAll(xml, "ResourceStatus");
+ assertEquals(ids.size(), statuses.size());
+ List<String> events = new ArrayList<>();
+ for (int i = ids.size() - 1; i >= 0; i--) {
+ events.add(ids.get(i) + " " + statuses.get(i));
+ }
+ return events;
+ }
+
+ private static Map<String, String> outputs(String stack) {
+ String xml = request(CFN_AUTH, "DescribeStacks", "StackName", stack).statusCode(200).extract().asString();
+ return XmlParser.extractPairs(xml, "member", "OutputKey", "OutputValue");
+ }
+
+ /** DeleteStack answers before the stack is gone, so this waits and fails on DELETE_FAILED. */
+ private static void deleteStack(String stack) {
+ request(CFN_AUTH, "DeleteStack", "StackName", stack).statusCode(200);
+ CfnStackWaits.awaitStackDeleted(stack, CFN_AUTH);
+ }
+
+ private static void ssm(String action, String body) {
+ given().header("X-Amz-Target", "AmazonSSM." + action).contentType("application/x-amz-json-1.1")
+ .body(body).when().post("/").then().statusCode(200);
+ }
+
+ private static ValidatableResponse ec2(String action, String... values) {
+ return request(EC2_AUTH, action, values);
+ }
+
+ private static ValidatableResponse request(String auth, String action, String... values) {
+ Map<String, String> parameters = new LinkedHashMap<>();
+ parameters.put("Action", action);
+ for (int i = 0; i < values.length; i += 2) {
+ parameters.put(values[i], values[i + 1]);
+ }
+ return given().header("Authorization", auth).formParams(parameters).when().post("/").then()
+ .log().ifValidationFails();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisionerTest.java
index af7b97d853..e08d1eac3e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisionerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ElbV2CfnProvisionerTest.java
@@ -22,9 +22,11 @@ import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -133,7 +135,7 @@ class ElbV2CfnProvisionerTest {
void loadBalancerExposesExactlyTheSchemaAttributes() {
when(elb.createLoadBalancer(eq(REGION), eq("web"), eq("internet-facing"), eq("application"), isNull(),
eq(List.of("subnet-1", "subnet-2")), eq(List.of("sg-1")), eq(Map.of("env", "test"))))
- .thenReturn(loadBalancer("web"));
+ .thenReturn(loadBalancerOn("web", "subnet-1", "subnet-2"));
ObjectNode props = mapper.createObjectNode().put("Name", "web").put("Scheme", "internet-facing")
.put("Type", "application");
props.putArray("Subnets").add("subnet-1").add("subnet-2");
@@ -144,8 +146,10 @@ class ElbV2CfnProvisionerTest {
provisioner.provision(r, props, ctx());
assertEquals(LB_ARN, r.getPhysicalId());
+ // Fn::GetAtt resolves no __Floci key, so those are the hidden bookkeeping, not attributes.
assertEquals(Set.of("LoadBalancerArn", "DNSName", "CanonicalHostedZoneID", "LoadBalancerName",
- "LoadBalancerFullName"), r.getAttributes().keySet());
+ "LoadBalancerFullName"), r.getAttributes().keySet().stream()
+ .filter(key -> !key.startsWith("__Floci")).collect(Collectors.toSet()));
assertEquals("app/web/50dc6c495c0c9188", r.getAttributes().get("LoadBalancerFullName"));
assertEquals("web", r.getAttributes().get("LoadBalancerName"));
assertEquals("Z35SXDOTRQ7X7K", r.getAttributes().get("CanonicalHostedZoneID"));
@@ -452,6 +456,36 @@ class ElbV2CfnProvisionerTest {
verify(elb).setSubnets(REGION, LB_ARN, List.of("subnet-1", "subnet-2"));
}
+ /** The subnets a load balancer is on, as its hidden uses attributes record them. */
+ private static Set<String> usedSubnets(StackResource lb) {
+ Set<String> used = new HashSet<>();
+ lb.getAttributes().forEach((key, value) -> {
+ if (key.startsWith(CfnRollback.USES_ATTR_PREFIX)) {
+ used.add(value);
+ }
+ });
+ return used;
+ }
+
+ @Test
+ void theLoadBalancerRecordsTheSubnetsItIsOnAcrossAMoveAndAMoveBack() {
+ when(elb.createLoadBalancer(eq(REGION), eq("web"), isNull(), isNull(), isNull(),
+ eq(List.of("subnet-1", "subnet-2")), anyList(), any()))
+ .thenReturn(loadBalancerOn("web", "subnet-1", "subnet-2"));
+ ObjectNode props = mapper.createObjectNode().put("Name", "web");
+ props.putArray("Subnets").add("subnet-1").add("subnet-2");
+ StackResource created = resource("AWS::ElasticLoadBalancingV2::LoadBalancer", "Alb");
+ provisioner.provision(created, props, ctx());
+ assertEquals(Set.of("subnet-1", "subnet-2"), usedSubnets(created));
+
+ StackResource moved = keptLoadBalancerUpdatedTo(loadBalancerOn("web", "subnet-1", "subnet-2"),
+ "subnet-3");
+ assertEquals(Set.of("subnet-3"), usedSubnets(moved));
+
+ assertTrue(provisioner.rollbackUpdate(moved));
+ assertEquals(Set.of("subnet-1", "subnet-2"), usedSubnets(moved));
+ }
+
@Test
void theSameSubnetsInAnotherOrderLeaveTheKeptLoadBalancerAlone() {
StackResource lb = keptLoadBalancerUpdatedTo(loadBalancerOn("web", "subnet-1", "subnet-2"),
diff --git a/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java
index cbfbdc234b..815a3d1733 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.CertificateReferenceResolver;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.PersistentStorage;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.ec2.Ec2Service;
@@ -22,6 +23,7 @@ import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.NullAndEmptySource;
import org.junit.jupiter.params.provider.ValueSource;
@@ -29,6 +31,7 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -39,6 +42,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -640,6 +644,12 @@ class ElbV2ServiceTest {
}
private static void stubAlbSubnets(Ec2Service ec2Service) {
+ lenient().when(ec2Service.supplyWithSubnetReferencesHeld(anyString(), any()))
+ .thenAnswer(invocation -> invocation.<Supplier<?>>getArgument(1).get());
+ lenient().doAnswer(invocation -> {
+ invocation.<Runnable>getArgument(1).run();
+ return null;
+ }).when(ec2Service).runWithSubnetReferencesHeld(anyString(), any());
Subnet subnetA = subnet("subnet-a", REGION + "a");
Subnet subnetB = subnet("subnet-b", REGION + "b");
lenient().when(ec2Service.requireSubnet(REGION, "subnet-a")).thenReturn(subnetA);
@@ -656,6 +666,25 @@ class ElbV2ServiceTest {
return subnet;
}
+ /** Files in a directory, so a second service reads back what the first wrote as JSON. */
+ private static final class PersistentStorageFactory extends StorageFactory {
+ private final Path dir;
+
+ private PersistentStorageFactory(Path dir) {
+ super(null, null);
+ this.dir = dir;
+ }
+
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ PersistentStorage<String, V> files = new PersistentStorage<>(dir.resolve(fileName), typeReference);
+ files.load();
+ return new AccountAwareStorageBackend<>(files, null, "000000000000");
+ }
+ }
+
private static final class SharedStorageFactory extends StorageFactory {
private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
@@ -697,4 +726,27 @@ class ElbV2ServiceTest {
reloaded.describeLoadBalancers(REGION, null, List.of("stored-nlb"), null, null)
.getFirst().getCanonicalHostedZoneId());
}
+
+ @Test
+ void subnetAndVpcDependenciesComeFromPersistedLoadBalancersOnly(@TempDir Path dir) {
+ ElbV2Service first = serviceWithStorage(new PersistentStorageFactory(dir), dataPlane, healthChecker);
+ String arn = first.createLoadBalancer(REGION, "stored-dependency", "internal", "application", "ipv4",
+ ALB_SUBNETS, List.of(), Map.of()).getLoadBalancerArn();
+ String targetGroup = first.createTargetGroup(REGION, "independent-group", "HTTP", "HTTP1", 80,
+ "vpc-a", "instance", "HTTP", "traffic-port", true, "/", 30, 5, 5, 2, "200",
+ "ipv4", Map.of()).getTargetGroupArn();
+ ElbV2Service restored = serviceWithStorage(new PersistentStorageFactory(dir), dataPlane, healthChecker);
+ assertTrue(restored.hasSubnetDependency(REGION, "subnet-a"));
+ assertTrue(restored.hasSubnetDependency(REGION, "subnet-b"));
+ assertTrue(restored.hasVpcDependency(REGION, "vpc-a"));
+ assertFalse(restored.hasSubnetDependency("us-east-1", "subnet-a"));
+ assertFalse(restored.hasVpcDependency("us-east-1", "vpc-a"));
+ assertFalse(restored.hasSubnetDependency(REGION, "subnet-unrelated"));
+ restored.deleteLoadBalancer(REGION, arn);
+ ElbV2Service afterDelete = serviceWithStorage(new PersistentStorageFactory(dir), dataPlane, healthChecker);
+ assertFalse(afterDelete.hasSubnetDependency(REGION, "subnet-a"));
+ assertFalse(afterDelete.hasVpcDependency(REGION, "vpc-a"));
+ assertEquals(targetGroup, afterDelete.describeTargetGroups(REGION, null,
+ List.of(targetGroup), null).getFirst().getTargetGroupArn());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2SubnetDependencyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2SubnetDependencyIntegrationTest.java
new file mode 100644
index 0000000000..a629d998b3
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2SubnetDependencyIntegrationTest.java
@@ -0,0 +1,282 @@
+package io.github.hectorvent.floci.services.elbv2;
+
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.ec2.model.Subnet;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.mockito.InjectSpy;
+import io.restassured.response.ValidatableResponse;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItems;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.reset;
+
+@QuarkusTest
+class ElbV2SubnetDependencyIntegrationTest {
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT = "111122223333";
+
+ @InjectSpy
+ Ec2Service ec2Service;
+
+ @InjectSpy
+ ElbV2Service elbService;
+
+ @ParameterizedTest
+ @ValueSource(strings = {"application", "network"})
+ void loadBalancerPinsSubnetsUntilDeletedButTargetGroupDoesNot(String type) {
+ String vpc = vpc();
+ String first = subnet(vpc, "10.92.1.0/24", "a");
+ String second = subnet(vpc, "10.92.2.0/24", "b");
+ String unused = subnet(vpc, "10.92.3.0/24", "c");
+ String arn = loadBalancer(type, first, second);
+ String targetGroup = elb("CreateTargetGroup", "Name", name(), "Protocol", "HTTP", "Port", "80",
+ "VpcId", vpc).statusCode(200).extract()
+ .path("CreateTargetGroupResponse.CreateTargetGroupResult.TargetGroups.member.TargetGroupArn");
+ try {
+ ec2("DeleteSubnet", "SubnetId", first).statusCode(400)
+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));
+ ec2("DeleteSubnet", "SubnetId", second).statusCode(400)
+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));
+ ec2("DeleteVpc", "VpcId", vpc).statusCode(400)
+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));
+ elb("DescribeLoadBalancers", "LoadBalancerArns.member.1", arn).statusCode(200)
+ .body("DescribeLoadBalancersResponse.DescribeLoadBalancersResult.LoadBalancers.member"
+ + ".AvailabilityZones.member.SubnetId", hasItems(first, second));
+ ec2("DeleteSubnet", "SubnetId", unused).statusCode(200);
+ } finally {
+ elb("DeleteLoadBalancer", "LoadBalancerArn", arn).statusCode(200);
+ }
+ ec2("DeleteSubnet", "SubnetId", first).statusCode(200);
+ ec2("DeleteSubnet", "SubnetId", second).statusCode(200);
+ ec2("DeleteVpc", "VpcId", vpc).statusCode(200);
+ elb("DescribeTargetGroups", "TargetGroupArns.member.1", targetGroup).statusCode(200);
+ elb("DeleteTargetGroup", "TargetGroupArn", targetGroup).statusCode(200);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"application", "network"})
+ void changingSubnetsReleasesOnlyTheOldReferences(String type) {
+ String vpc = vpc();
+ String first = subnet(vpc, "10.92.1.0/24", "a");
+ String second = subnet(vpc, "10.92.2.0/24", "b");
+ String replacement = subnet(vpc, "10.92.3.0/24", "c");
+ String arn = loadBalancer(type, first, second);
+ try {
+ elb("SetSubnets", "LoadBalancerArn", arn, "Subnets.member.1", first,
+ "Subnets.member.2", replacement).statusCode(200);
+ ec2("DeleteSubnet", "SubnetId", second).statusCode(200);
+ ec2("DeleteSubnet", "SubnetId", replacement).statusCode(400)
+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));
+ } finally {
+ elb("DeleteLoadBalancer", "LoadBalancerArn", arn).statusCode(200);
+ }
+ ec2("DeleteSubnet", "SubnetId", first).statusCode(200);
+ ec2("DeleteSubnet", "SubnetId", replacement).statusCode(200);
+ ec2("DeleteVpc", "VpcId", vpc).statusCode(200);
+ }
+
+ /**
+ * The other account is fresh for each run, since deleting its default subnet is permanent for
+ * the rest of the suite's shared application.
+ */
+ @Test
+ void referencesInAnotherAccountDoNotPinTheSameDefaultSubnetId() {
+ String otherAccount = Long.toString(ThreadLocalRandom.current().nextLong(100_000_000_000L, 1_000_000_000_000L));
+ String first = Ec2Service.defaultSubnetId(REGION, "a");
+ String second = Ec2Service.defaultSubnetId(REGION, "b");
+ String arn = loadBalancer("application", first, second);
+ try {
+ request("ec2", otherAccount, REGION, "DeleteSubnet", "SubnetId", first).statusCode(200);
+ ec2("DeleteSubnet", "SubnetId", first).statusCode(400)
+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));
+ } finally {
+ elb("DeleteLoadBalancer", "LoadBalancerArn", arn).statusCode(200);
+ }
+ }
+
+ /**
+ * A load balancer's subnets keep its VPC from being deleted anyway, so the VPC check is driven
+ * directly: it stands for state such as a balancer stored before subnets were protected, whose
+ * subnets could already be gone.
+ */
+ @Test
+ void aVpcStillReferencedByALoadBalancerCannotBeDeletedEvenWithoutSubnets() {
+ String vpc = vpc();
+ doReturn(true).when(elbService).hasVpcDependency(REGION, vpc);
+ try {
+ ec2("DeleteVpc", "VpcId", vpc).statusCode(400)
+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));
+ } finally {
+ reset(elbService);
+ }
+ ec2("DeleteVpc", "VpcId", vpc).statusCode(200);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void deletionWaitsForCreateOrSetSubnetsToPublishItsReferences(boolean reassign) throws Exception {
+ String vpc = vpc();
+ String first = subnet(vpc, "10.92.1.0/24", "a");
+ String second = subnet(vpc, "10.92.2.0/24", "b");
+ String replacement = subnet(vpc, "10.92.3.0/24", "c");
+ String existing = reassign ? loadBalancer("application", first, second) : null;
+ CountDownLatch validated = new CountDownLatch(1);
+ CountDownLatch publish = new CountDownLatch(1);
+ CountDownLatch deleting = new CountDownLatch(1);
+ AtomicBoolean pauseOnce = new AtomicBoolean(true);
+ doAnswer(invocation -> {
+ List<Subnet> found = (List<Subnet>) invocation.callRealMethod();
+ if (((List<?>) invocation.getArgument(1)).contains(replacement) && pauseOnce.getAndSet(false)) {
+ validated.countDown();
+ assertTrue(publish.await(10, TimeUnit.SECONDS));
+ }
+ return found;
+ }).when(ec2Service).describeSubnets(eq(REGION), anyList(), anyMap());
+ doAnswer(invocation -> {
+ deleting.countDown();
+ return invocation.callRealMethod();
+ }).when(ec2Service).deleteSubnet(REGION, replacement);
+ String arn = existing;
+ try (ExecutorService executor = Executors.newFixedThreadPool(2)) {
+ Future<String> attach = executor.submit(() -> {
+ if (reassign) {
+ elb("SetSubnets", "LoadBalancerArn", existing, "Subnets.member.1", first,
+ "Subnets.member.2", replacement).statusCode(200);
+ return existing;
+ }
+ return loadBalancer("application", first, replacement);
+ });
+ try {
+ assertTrue(validated.await(10, TimeUnit.SECONDS));
+ Future<ValidatableResponse> deletion = executor.submit(
+ () -> ec2("DeleteSubnet", "SubnetId", replacement));
+ assertTrue(deleting.await(10, TimeUnit.SECONDS));
+ assertThrows(TimeoutException.class, () -> deletion.get(200, TimeUnit.MILLISECONDS));
+ publish.countDown();
+ arn = attach.get(10, TimeUnit.SECONDS);
+ deletion.get(10, TimeUnit.SECONDS).statusCode(400)
+ .body("Response.Errors.Error.Code", equalTo("DependencyViolation"));
+ } finally {
+ publish.countDown();
+ }
+ } finally {
+ reset(ec2Service);
+ if (arn != null) {
+ elb("DeleteLoadBalancer", "LoadBalancerArn", arn).statusCode(200);
+ }
+ }
+ ec2("DeleteSubnet", "SubnetId", first).statusCode(200);
+ ec2("DeleteSubnet", "SubnetId", second).statusCode(200);
+ ec2("DeleteSubnet", "SubnetId", replacement).statusCode(200);
+ ec2("DeleteVpc", "VpcId", vpc).statusCode(200);
+ }
+
+ @Test
+ void createCannotPublishAReferenceAfterSubnetDeletionWins() throws Exception {
+ String vpc = vpc();
+ String first = subnet(vpc, "10.92.1.0/24", "a");
+ String second = subnet(vpc, "10.92.2.0/24", "b");
+ CountDownLatch checked = new CountDownLatch(1);
+ CountDownLatch delete = new CountDownLatch(1);
+ CountDownLatch creating = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ boolean dependent = (boolean) invocation.callRealMethod();
+ checked.countDown();
+ assertTrue(delete.await(10, TimeUnit.SECONDS));
+ return dependent;
+ }).when(elbService).hasSubnetDependency(REGION, first);
+ doAnswer(invocation -> {
+ creating.countDown();
+ return invocation.callRealMethod();
+ }).when(elbService).createLoadBalancer(eq(REGION), anyString(),
+ any(), eq("application"), any(),
+ anyList(), any(), anyMap());
+ try (ExecutorService executor = Executors.newFixedThreadPool(2)) {
+ Future<ValidatableResponse> deletion = executor.submit(() -> ec2("DeleteSubnet", "SubnetId", first));
+ try {
+ assertTrue(checked.await(10, TimeUnit.SECONDS));
+ Future<ValidatableResponse> creation = executor.submit(() -> elb("CreateLoadBalancer", "Name", name(),
+ "Type", "application", "Subnets.member.1", first, "Subnets.member.2", second));
+ assertTrue(creating.await(10, TimeUnit.SECONDS));
+ assertThrows(TimeoutException.class, () -> creation.get(200, TimeUnit.MILLISECONDS));
+ delete.countDown();
+ deletion.get(10, TimeUnit.SECONDS).statusCode(200);
+ creation.get(10, TimeUnit.SECONDS).statusCode(400)
+ .body("ErrorResponse.Error.Code", equalTo("SubnetNotFound"));
+ } finally {
+ delete.countDown();
+ }
+ } finally {
+ reset(elbService);
+ }
+ ec2("DeleteSubnet", "SubnetId", second).statusCode(200);
+ ec2("DeleteVpc", "VpcId", vpc).statusCode(200);
+ }
+
+ private String vpc() {
+ return ec2("CreateVpc", "CidrBlock", "10.92.0.0/16").statusCode(200).extract()
+ .path("CreateVpcResponse.vpc.vpcId");
+ }
+
+ private String subnet(String vpc, String cidr, String zone) {
+ return ec2("CreateSubnet", "VpcId", vpc, "CidrBlock", cidr,
+ "AvailabilityZone", REGION + zone).statusCode(200).extract()
+ .path("CreateSubnetResponse.subnet.subnetId");
+ }
+
+ private String loadBalancer(String type, String first, String second) {
+ return elb("CreateLoadBalancer", "Name", name(), "Type", type,
+ "Subnets.member.1", first, "Subnets.member.2", second).statusCode(200).extract()
+ .path("CreateLoadBalancerResponse.CreateLoadBalancerResult.LoadBalancers.member.LoadBalancerArn");
+ }
+
+ private static String name() {
+ return "subnet-" + UUID.randomUUID().toString().substring(0, 20);
+ }
+
+ private ValidatableResponse ec2(String action, String... values) {
+ return request("ec2", ACCOUNT, REGION, action, values);
+ }
+
+ private ValidatableResponse elb(String action, String... values) {
+ return request("elasticloadbalancing", ACCOUNT, REGION, action, values);
+ }
+
+ private ValidatableResponse request(String service, String account, String region,
+ String action, String... values) {
+ Map<String, String> parameters = new LinkedHashMap<>();
+ parameters.put("Action", action);
+ for (int i = 0; i < values.length; i += 2) {
+ parameters.put(values[i], values[i + 1]);
+ }
+ return given().header("Authorization", "AWS4-HMAC-SHA256 Credential=" + account
+ + "/20261011/" + region + "/" + service + "/aws4_request")
+ .formParams(parameters).when().post("/").then();
+ }
+}