Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): delete dependents first on update rollback (#5943)
Jackson Priced273d4f4e0aa
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.javaindex fbaac39423..8f69538f17 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java@@ -3,6 +3,8 @@ package io.github.hectorvent.floci.services.cloudformation;import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.JsonNodeFactory;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;@@ -2792,7 +2794,11 @@ public class CloudFormationService implements ResourceProvider {Map<String, StackResource> previousResources,Set<String> attemptedResourceIds,String region) {- List<StackResource> resources = new ArrayList<>(stack.resourcesSnapshot().values());+ // Dependents first, in the template's order rather than the map's, as the update cleanup+ // walks it: a resource the update added sits after the resources that depend on it, so+ // reversing the map would delete a subnet the update added before an instance or NAT+ // gateway the update put on it, and EC2 refuses that delete.+ List<StackResource> resources = resourcesInCreationOrder(stack, region);Collections.reverse(resources);List<String> failures = new ArrayList<>();List<String> removedResources = new ArrayList<>();@@ -3108,13 +3114,22 @@ public class CloudFormationService implements ResourceProvider {}/**- * The stack's resources in the order the current template creates them, for a delete that walks- * them backwards. The resource map keeps insertion order, and a resource added by a later- * update sits after the resources that depend on it, so reversing the map would delete it- * first: a certificate still used by a user pool domain, for one. Resources the template no- * longer names, left behind by a failed cleanup, sort first here so they are deleted last,- * after anything that might still use them. Insertion order is the fallback when the template- * cannot be read.+ * The stack's resources in the order the current template creates them, for a delete, an update+ * cleanup or an update rollback that walks them backwards. The resource map keeps insertion+ * order, and a resource added by a later update sits after the resources that depend on it, so+ * reversing the map would delete it first: a certificate still used by a user pool domain, for+ * one. Resources the template no longer names, left behind by a failed cleanup or skipped by+ * ContinueUpdateRollback, are deleted as late as possible, after anything that might still use+ * them, but before any held resource whose physical id one of their attributes names, such as a+ * subnet's {@code VpcId}: with their template entry gone, that is the only record of what they+ * depend on, so a leftover whose attributes name nothing it uses is still deleted last.+ * Conditions are ignored, both on resources and in {@code Fn::If}: a resource depends on what+ * every branch names. The conditions read now need not be the ones a held resource was+ * provisioned under, since a failed rollback restores the parameters but can leave a resource+ * on the branch the failed update chose. When counting every branch closes a cycle, the walk+ * starts from the branches chosen now, evaluated with the SSM-resolved parameter values when+ * there are any, and adds each other-branch dependency that closes no cycle. Insertion order+ * is the fallback when the template cannot be read.*/private List<StackResource> resourcesInCreationOrder(Stack stack, String region) {List<StackResource> ordered = new ArrayList<>(stack.resourcesSnapshot().values());@@ -3124,23 +3139,191 @@ public class CloudFormationService implements ResourceProvider {if (!resources.isObject()) {return ordered;}- Map<String, Boolean> conditions = resolveConditions(- template, stack.parametersSnapshot(), stack, region, regionResolver.getAccountId());+ // Ordered without conditions: only the resources the stack holds are walked, and each+ // was provisioned, so its dependencies count whatever its condition reads now. A+ // ContinueUpdateRollback retry reads the failed update's template with the parameters+ // the first pass already restored, and a held resource may depend on a condition-false+ // one an earlier cleanup deleted, which would otherwise fail the whole ordering.+ ObjectNode unconditional = resources.deepCopy();+ for (Map.Entry<String, JsonNode> entry : unconditional.properties()) {+ if (entry.getValue() instanceof ObjectNode definition) {+ definition.remove("Condition");+ }+ }+ ObjectNode everyBranch = unconditional.deepCopy();+ for (Map.Entry<String, JsonNode> entry : everyBranch.properties()) {+ if (entry.getValue() instanceof ObjectNode definition && definition.has("Properties")) {+ definition.set("Properties", everyIfBranch(definition.get("Properties")));+ }+ }// A stack saved before circular templates were rejected may still have one; order it// by its dependencies anyway rather than giving up.- List<String> creationOrder = creationOrder(resources, conditions).sorted();+ CreationOrder order = creationOrder(everyBranch, Map.of());+ if (!order.circular().isEmpty()) {+ order = withoutBranchCycles(order, stack, template, unconditional, everyBranch, region);+ }+ List<String> creationOrder = order.sorted();Map<String, Integer> rank = new HashMap<>();for (int i = 0; i < creationOrder.size(); i++) {rank.put(creationOrder.get(i), i);}- ordered.sort(Comparator.comparingInt(r -> rank.getOrDefault(r.getLogicalId(), -1)));+ Map<String, List<StackResource>> holdersByPhysicalId = new HashMap<>();+ int leftovers = 0;+ for (StackResource resource : ordered) {+ if (!rank.containsKey(resource.getLogicalId())) {+ leftovers++;+ }+ if (resource.getPhysicalId() != null) {+ holdersByPhysicalId.computeIfAbsent(resource.getPhysicalId(), id -> new ArrayList<>())+ .add(resource);+ }+ }+ double step = 1.0 / (leftovers + 1);+ Map<String, Double> position = new HashMap<>();+ for (StackResource resource : ordered) {+ position.put(resource.getLogicalId(),+ leftoverPosition(resource, rank, holdersByPhysicalId, step, position, new HashSet<>()));+ }+ ordered.sort(Comparator.comparingDouble(r -> position.get(r.getLogicalId())));} catch (Exception e) {- LOG.debugv("Deleting stack {0} in insertion order, its template could not be ordered: {1}",+ LOG.debugv("Walking stack {0} in insertion order, its template could not be ordered: {1}",stack.getStackName(), e.getMessage());}return ordered;}+ /**+ * The order the branches chosen now give, with every dependency another {@code Fn::If} branch+ * names added to it unless that dependency would close a cycle. Counting every branch can close+ * a cycle no choice of branches has, such as two resources that each use the other under+ * opposite conditions; dropping only the edges that close it keeps every other branch's+ * dependencies. {@code everyBranch}, the order counting every branch, is kept when the+ * conditions cannot be read.+ */+ private CreationOrder withoutBranchCycles(CreationOrder everyBranch, Stack stack, JsonNode template,+ ObjectNode unconditional, ObjectNode everyBranchResources,+ String region) {+ try {+ Map<String, String> resolvedParameters = stack.resolvedParametersSnapshot();+ Map<String, Boolean> conditions = resolveConditions(template,+ resolvedParameters.isEmpty() ? stack.parametersSnapshot() : resolvedParameters,+ stack, region, regionResolver.getAccountId());+ Set<String> ids = new LinkedHashSet<>();+ unconditional.fieldNames().forEachRemaining(ids::add);+ Map<String, Set<String>> dependencies = new HashMap<>();+ for (String id : ids) {+ Set<String> chosen = collectResourceDependencies(unconditional.get(id), ids, conditions);+ chosen.retainAll(ids);+ dependencies.put(id, chosen);+ }+ ObjectNode merged = unconditional.deepCopy();+ for (String id : ids) {+ for (String dependency : collectResourceDependencies(everyBranchResources.get(id), ids, Map.of())) {+ if (ids.contains(dependency) && !dependency.equals(id)+ && !dependencies.get(id).contains(dependency)+ && !dependsOn(dependencies, dependency, id)) {+ dependencies.get(id).add(dependency);+ addDependsOn(merged.get(id), dependency);+ }+ }+ }+ return creationOrder(merged, conditions);+ } catch (RuntimeException e) {+ LOG.debugv("Ordering stack {0} by every Fn::If branch, its conditions could not be read: {1}",+ stack.getStackName(), e.getMessage());+ return everyBranch;+ }+ }++ /** Whether {@code from} depends on {@code to}, directly or through other resources. */+ private static boolean dependsOn(Map<String, Set<String>> dependencies, String from, String to) {+ Deque<String> pending = new ArrayDeque<>(List.of(from));+ Set<String> seen = new HashSet<>();+ while (!pending.isEmpty()) {+ String current = pending.pop();+ if (current.equals(to)) {+ return true;+ }+ if (seen.add(current)) {+ pending.addAll(dependencies.getOrDefault(current, Set.of()));+ }+ }+ return false;+ }++ /** Adds a resource to a definition's {@code DependsOn}, keeping what it already names. */+ private static void addDependsOn(JsonNode definition, String dependency) {+ if (!(definition instanceof ObjectNode object)) {+ return;+ }+ JsonNode existing = object.path("DependsOn");+ ArrayNode dependsOn = JsonNodeFactory.instance.arrayNode();+ if (existing.isTextual()) {+ dependsOn.add(existing.asText());+ } else if (existing.isArray()) {+ dependsOn.addAll((ArrayNode) existing);+ }+ dependsOn.add(dependency);+ object.set("DependsOn", dependsOn);+ }++ /**+ * A copy of a template value with each {@code Fn::If} replaced by a list of both its branches,+ * so the dependencies collected from it are what either branch names.+ */+ private static JsonNode everyIfBranch(JsonNode node) {+ if (node.isObject()) {+ JsonNode fnIf = node.path("Fn::If");+ if (fnIf.isArray() && fnIf.size() == 3) {+ ArrayNode branches = JsonNodeFactory.instance.arrayNode();+ branches.add(everyIfBranch(fnIf.get(1)));+ branches.add(everyIfBranch(fnIf.get(2)));+ return branches;+ }+ ObjectNode copy = JsonNodeFactory.instance.objectNode();+ node.properties().forEach(e -> copy.set(e.getKey(), everyIfBranch(e.getValue())));+ return copy;+ }+ if (node.isArray()) {+ ArrayNode copy = JsonNodeFactory.instance.arrayNode();+ node.forEach(item -> copy.add(everyIfBranch(item)));+ return copy;+ }+ return node;+ }++ /**+ * A resource's place in {@link #resourcesInCreationOrder}: its template rank, or for a leftover+ * one step after the latest held resource its attributes name, leftovers included, or else -1.+ * A chain of leftovers stays below the next template rank, and a cycle among them is cut where+ * it closes.+ */+ private static double leftoverPosition(StackResource resource, Map<String, Integer> rank,+ Map<String, List<StackResource>> holdersByPhysicalId,+ double step, Map<String, Double> position, Set<String> visiting) {+ Integer resourceRank = rank.get(resource.getLogicalId());+ if (resourceRank != null) {+ return resourceRank;+ }+ Double known = position.get(resource.getLogicalId());+ if (known != null) {+ return known;+ }+ visiting.add(resource.getLogicalId());+ double placed = -1;+ for (String value : resource.getAttributes().values()) {+ for (StackResource holder : holdersByPhysicalId.getOrDefault(value, List.of())) {+ if (!visiting.contains(holder.getLogicalId())) {+ placed = Math.max(placed,+ leftoverPosition(holder, rank, holdersByPhysicalId, step, position, visiting) + step);+ }+ }+ }+ visiting.remove(resource.getLogicalId());+ position.put(resource.getLogicalId(), placed);+ return placed;+ }+void deleteStackResources(Stack stack, String region, String accountId) {try {List<StackResource> resources = resourcesInCreationOrder(stack, region);diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.javaindex 1171e41212..b8fe80811c 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java@@ -205,6 +205,14 @@ public final class CfnRollback {*/public static final String REPLACEMENT_CLEANUP_ATTR = "__FlociReplacementCleanup";+ /**+ * Prefixes attributes that name, by physical id, a resource this one uses where none of its+ * public attributes does, such as a NAT gateway's subnet and Elastic IP. {@code+ * CloudFormationService} reads every attribute value of a resource the template no longer+ * names to delete it before what it uses; these keep {@code Fn::GetAtt} to what AWS resolves.+ */+ public static final String USES_ATTR_PREFIX = "__FlociUses";+/*** Holds the tags, stream setting and time-to-live setting a DynamoDB table carried before an* in-place update changed them, with the account and region that address it, so a faileddiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisioner.javaindex ec9efd8a8d..4998f4c482 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisioner.java@@ -583,6 +583,17 @@ public class Ec2NetworkCfnProvisioner implements CfnResourceProvisioner {}r.setPhysicalId(nat.getNatGatewayId());r.getAttributes().put("NatGatewayId", nat.getNatGatewayId());+ // The gateway holds its subnet and Elastic IP until it is deleted; an EIP's physical id is+ // its public IP.+ if (nat.getSubnetId() != null) {+ r.getAttributes().put(CfnRollback.USES_ATTR_PREFIX + "SubnetId", nat.getSubnetId());+ }+ Address eip = allocationId == null || allocationId.isBlank() ? null : findAddress(allocationId, null, ctx.region());+ if (eip != null && eip.getPublicIp() != null) {+ r.getAttributes().put(CfnRollback.USES_ATTR_PREFIX + "PublicIp", eip.getPublicIp());+ } else {+ r.getAttributes().remove(CfnRollback.USES_ATTR_PREFIX + "PublicIp");+ }}private void provisionEip(StackResource r, JsonNode props, ProvisionContext ctx) {diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanup.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanup.javaindex afeef16f20..ab3d26caa8 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanup.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanup.java@@ -93,7 +93,7 @@ public final class ReplacementCleanup {cleanup.put("priorPhysicalId", prior);ObjectNode priorAttributes = cleanup.putObject("priorAttributes");attributesBeforeProvision.forEach((key, value) -> {- if (!key.startsWith("__Floci") && value != null) {+ if (restoredOnRevert(key) && value != null) {priorAttributes.put(key, value);}});@@ -153,7 +153,7 @@ public final class ReplacementCleanup {r.setPhysicalId(prior);Iterator<String> keys = r.getAttributes().keySet().iterator();while (keys.hasNext()) {- if (!keys.next().startsWith("__Floci")) {+ if (restoredOnRevert(keys.next())) {keys.remove();}}@@ -174,6 +174,15 @@ public final class ReplacementCleanup {return deleteReplacement(r, cleanup, deleter, r.getResourceType(), replacement);}+ /**+ * Whether a revert puts back the prior value of an attribute: the public ones, and the hidden+ * {@link CfnRollback#USES_ATTR_PREFIX} ones, which describe the entity as the public ones do.+ * Other {@code __Floci} keys are rollback bookkeeping that outlives the revert.+ */+ private static boolean restoredOnRevert(String key) {+ return !key.startsWith("__Floci") || key.startsWith(CfnRollback.USES_ATTR_PREFIX);+ }+/** Deletes a rolled-back replacement; one whose delete throws stays owed and named for the retry. */private static boolean deleteReplacement(StackResource r, ObjectNode cleanup, Deleter deleter,String resourceType, String replacement) {diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationNatGatewayLeftoverIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationNatGatewayLeftoverIntegrationTest.javanew file mode 100644index 0000000000..6b70dc305e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationNatGatewayLeftoverIntegrationTest.java@@ -0,0 +1,138 @@+package io.github.hectorvent.floci.services.cloudformation;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.XmlParser;+import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.mockito.InjectSpy;+import io.restassured.response.ValidatableResponse;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.concurrent.atomic.AtomicBoolean;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.reset;++/**+ * A NAT gateway the template no longer names, left by a rollback that skipped it, still holds its+ * Elastic IP and its subnet, so DeleteStack has to delete it before either of them, though only+ * its hidden attributes say what it uses.+ */+@QuarkusTest+class CloudFormationNatGatewayLeftoverIntegrationTest {++ private static final String CFN_AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20261015/us-east-1/cloudformation/aws4_request";+ private static final String EC2_AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20261015/us-east-1/ec2/aws4_request";++ @InjectSpy+ Ec2Service ec2Service;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void deleteStackDeletesASkippedNatGatewayBeforeItsElasticIpSubnetAndVpc() {+ String stack = "nat-leftover-" + Long.toString(System.nanoTime(), 36);+ AtomicBoolean failNatDeletes = new AtomicBoolean(true);+ doAnswer(invocation -> {+ if (failNatDeletes.get()) {+ throw new AwsException("InternalError", "simulated NAT gateway delete failure", 500);+ }+ return invocation.callRealMethod();+ }).when(ec2Service).deleteNatGateway(anyString(), anyString());+ try {+ Map<String, String> created = createStack(stack, template(stack, ""));+ String nat = """+ ,+ "Nat": {"Type": "AWS::EC2::NatGateway",+ "Properties": {"SubnetId": {"Ref": "Subnet"},+ "AllocationId": {"Fn::GetAtt": ["Eip", "AllocationId"]}}},+ "BadSecret": {"Type": "AWS::SecretsManager::Secret", "DependsOn": "Nat",+ "Properties": {"Name": "%1$s-secret", "SecretString": "explicit",+ "GenerateSecretString": {"PasswordLength": 32}}}""".formatted(stack);+ request("UpdateStack", "StackName", stack, "TemplateBody", template(stack, nat)).statusCode(200);+ String reason = CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_FAILED").reason();+ assertTrue(reason.contains("Nat"), reason);+ failNatDeletes.set(false);++ request("ContinueUpdateRollback", "StackName", stack, "ResourcesToSkip.member.1", "Nat").statusCode(200);+ CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_COMPLETE");+ assertEquals(1, XmlParser.extractAll(ec2("DescribeNatGateways", "Filter.1.Name", "subnet-id",+ "Filter.1.Value.1", created.get("Subnet")).statusCode(200).extract().asString(), "natGatewayId")+ .size(), "the skipped gateway is left in place");++ request("DeleteStack", "StackName", stack).statusCode(200);+ CfnStackWaits.awaitStackDeleted(stack, CFN_AUTH);+ ec2("DescribeVpcs", "VpcId.1", created.get("Vpc")).statusCode(400)+ .body(containsString("InvalidVpcID.NotFound"));+ assertEquals(List.of(), XmlParser.extractAll(ec2("DescribeAddresses", "PublicIp.1", created.get("Eip"))+ .extract().asString(), "allocationId"));+ } finally {+ failNatDeletes.set(false);+ reset(ec2Service);+ // Best effort: a failed run may leave the stack DELETE_FAILED or mid-rollback.+ if (request("DescribeStacks", "StackName", stack).extract().statusCode() == 200) {+ request("DeleteStack", "StackName", stack);+ }+ }+ }++ /** A VPC with one subnet and an Elastic IP, plus whatever an update adds. */+ private static String template(String stack, String extraResources) {+ return """+ {+ "Resources": {+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.96.0.0/16",+ "Tags": [{"Key": "floci-it", "Value": "%1$s"}]}},+ "Subnet": {"Type": "AWS::EC2::Subnet",+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "10.96.1.0/24",+ "AvailabilityZone": "us-east-1a"}},+ "Eip": {"Type": "AWS::EC2::EIP", "Properties": {"Domain": "vpc"}}%2$s+ },+ "Outputs": {+ "Vpc": {"Value": {"Ref": "Vpc"}},+ "Subnet": {"Value": {"Ref": "Subnet"}},+ "Eip": {"Value": {"Ref": "Eip"}}+ }+ }""".formatted(stack, extraResources);+ }++ private static Map<String, String> createStack(String stack, String templateBody) {+ request("CreateStack", "StackName", stack, "TemplateBody", templateBody).statusCode(200);+ CfnStackWaits.awaitStatus(stack, "CREATE_COMPLETE");+ String xml = request("DescribeStacks", "StackName", stack).statusCode(200).extract().asString();+ return XmlParser.extractPairs(xml, "member", "OutputKey", "OutputValue");+ }++ private static ValidatableResponse ec2(String action, String... values) {+ return send(EC2_AUTH, action, values);+ }++ private static ValidatableResponse request(String action, String... values) {+ return send(CFN_AUTH, action, values);+ }++ private static ValidatableResponse send(String auth, String action, String... values) {+ Map<String, String> parameters = new LinkedHashMap<>();+ parameters.put("Action", action);+ for (int i = 0; i < values.length; i += 2) {+ parameters.put(values[i], values[i + 1]);+ }+ return given().contentType("application/x-www-form-urlencoded").header("Authorization", auth)+ .formParams(parameters).when().post("/").then().log().ifValidationFails();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.javaindex 5dcf83ee49..878e9dd4cf 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java@@ -783,6 +783,275 @@ class CloudFormationServiceRollbackTest {verify(provisioner, times(2)).rollbackUpdate(eq(dashboard), any());}+ /**+ * A retry walks the failed update's template with the parameters the first pass already+ * restored, so a condition the update turned on can read false again. A resource the stack+ * holds is still ordered by its dependencies, so the load balancer the update created on the+ * subnet it added is deleted before that subnet, not after it. The load balancer is inserted+ * first, so falling back to the map's order would delete the subnet first too.+ */+ @Test+ void resumeUpdateRollback_ordersAHeldResourceByItsDependenciesWhateverItsConditionReadsNow() {+ Stack stack = new Stack();+ stack.setStackName("retry-flipped-condition");+ stack.setStackId("stack-id");+ stack.setRegion(REGION);+ stack.setTemplateBody("""+ {+ "Parameters": {"Env": {"Type": "String"}},+ "Conditions": {"IsProd": {"Fn::Equals": [{"Ref": "Env"}, "prod"]}},+ "Resources": {+ "SubnetC": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": "vpc-1", "CidrBlock": "10.0.3.0/24"}},+ "ProdAlb": {+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",+ "Condition": "IsProd",+ "Properties": {"Subnets": [{"Ref": "SubnetC"}]}+ }+ }+ }+ """);+ stack.replaceParameters(Map.of("Env", "dev"));+ stack.replaceResolvedParameters(Map.of("Env", "dev"));+ StackResource subnet = resource("SubnetC", "subnet-c", "AWS::EC2::Subnet", "CREATE_COMPLETE");+ StackResource loadBalancer = resource("ProdAlb", "arn:aws:elasticloadbalancing:us-east-1:000000000000:"+ + "loadbalancer/app/prod/1", "AWS::ElasticLoadBalancingV2::LoadBalancer", "CREATE_COMPLETE");+ stack.getResources().put(loadBalancer.getLogicalId(), loadBalancer);+ stack.getResources().put(subnet.getLogicalId(), subnet);+ stack.setUpdateRollbackSnapshot(new StackUpdateSnapshot("previous-template", "previous-original",+ Map.of("Env", "dev"), Map.of("Env", "dev"), Map.of(), Map.of(), Map.of(), Map.of()));+ stack.setUpdateRollbackResourceIds(List.of("SubnetC", "ProdAlb"));+ stack.setStatus("UPDATE_ROLLBACK_IN_PROGRESS");++ service.resumeUpdateRollback(stack, List.of(), false, REGION);++ assertEquals("UPDATE_ROLLBACK_COMPLETE", stack.getStatus());+ InOrder deletes = inOrder(provisioner);+ deletes.verify(provisioner).delete(eq(loadBalancer), anyString());+ deletes.verify(provisioner).delete(eq(subnet), anyString());+ }++ /**+ * A held resource can depend on a condition-false one the stack no longer holds, such as a role+ * an earlier cleanup deleted while the load balancer on it failed to delete. Ordering must not+ * then give up on the whole template: the load balancer still goes before the subnet it uses.+ */+ @Test+ void deleteStack_withAHeldResourceOnAGoneConditionFalseOne_stillDeletesDependentsFirst() {+ Stack stack = new Stack();+ stack.setStackName("delete-held-on-gone-conditional");+ stack.setStackId("stack-id");+ stack.setRegion(REGION);+ stack.setStatus("UPDATE_COMPLETE");+ stack.setTemplateBody("""+ {+ "Parameters": {"Env": {"Type": "String"}},+ "Conditions": {"IsProd": {"Fn::Equals": [{"Ref": "Env"}, "prod"]}},+ "Resources": {+ "ProdRole": {"Type": "AWS::IAM::Role", "Condition": "IsProd"},+ "SubnetC": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": "vpc-1", "CidrBlock": "10.0.3.0/24"}},+ "ProdAlb": {+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",+ "Condition": "IsProd",+ "DependsOn": "ProdRole",+ "Properties": {"Subnets": [{"Ref": "SubnetC"}]}+ }+ }+ }+ """);+ stack.replaceParameters(Map.of("Env", "dev"));+ stack.replaceResolvedParameters(Map.of("Env", "dev"));+ StackResource loadBalancer = resource("ProdAlb", "arn:aws:elasticloadbalancing:us-east-1:000000000000:"+ + "loadbalancer/app/prod/1", "AWS::ElasticLoadBalancingV2::LoadBalancer", "DELETE_FAILED");+ StackResource subnet = resource("SubnetC", "subnet-c", "AWS::EC2::Subnet", "CREATE_COMPLETE");+ stack.getResources().put(loadBalancer.getLogicalId(), loadBalancer);+ stack.getResources().put(subnet.getLogicalId(), subnet);+ when(provisioner.completeDeleteCleanup(any())).thenReturn(UpdateCleanupResult.notApplicable());++ service.deleteStackResources(stack, REGION, ACCOUNT);++ InOrder deletes = inOrder(provisioner);+ deletes.verify(provisioner).delete(loadBalancer, REGION);+ deletes.verify(provisioner).delete(subnet, REGION);+ }++ /**+ * Counting every Fn::If branch can close a cycle no choice of branches has: here A uses B when+ * Flip is true, and B uses A when it is false. The walk then orders by the branches the+ * conditions choose now, so A, which uses B, still goes first.+ */+ @Test+ void deleteStack_whenEveryBranchWouldCloseACycle_ordersByTheBranchesChosenNow() {+ Stack stack = new Stack();+ stack.setStackName("delete-branch-cycle");+ stack.setStackId("stack-id");+ stack.setRegion(REGION);+ stack.setStatus("CREATE_COMPLETE");+ stack.setTemplateBody("""+ {+ "Parameters": {"Flip": {"Type": "String"}},+ "Conditions": {+ "Flipped": {"Fn::Equals": [{"Ref": "Flip"}, "yes"]},+ "NotFlipped": {"Fn::Not": [{"Condition": "Flipped"}]}+ },+ "Resources": {+ "B": {"Type": "AWS::SQS::Queue",+ "Properties": {"Tags": {"Fn::If": ["NotFlipped", [{"Key": "a", "Value": {"Ref": "A"}}],+ {"Ref": "AWS::NoValue"}]}}},+ "A": {"Type": "AWS::SQS::Queue",+ "Properties": {"Tags": {"Fn::If": ["Flipped", [{"Key": "b", "Value": {"Ref": "B"}}],+ {"Ref": "AWS::NoValue"}]}}}+ }+ }+ """);+ stack.replaceParameters(Map.of("Flip", "yes"));+ StackResource b = resource("B", "queue-b", "AWS::SQS::Queue", "CREATE_COMPLETE");+ StackResource a = resource("A", "queue-a", "AWS::SQS::Queue", "CREATE_COMPLETE");+ stack.getResources().put(b.getLogicalId(), b);+ stack.getResources().put(a.getLogicalId(), a);+ when(provisioner.completeDeleteCleanup(any())).thenReturn(UpdateCleanupResult.notApplicable());++ service.deleteStackResources(stack, REGION, ACCOUNT);++ InOrder deletes = inOrder(provisioner);+ deletes.verify(provisioner).delete(a, REGION);+ deletes.verify(provisioner).delete(b, REGION);+ }++ /**+ * A cycle that only counting every branch closes, between X and Y, must not cost the rest of+ * the stack its other-branch dependencies: the load balancer still on the subnet the failed+ * update chose goes before that subnet, though the restored parameter now picks the other.+ * SubnetC is created two steps after SubnetA, so only that dependency puts the load balancer+ * after it.+ */+ @Test+ void deleteStack_withAnUnrelatedBranchCycle_stillDeletesAResourceBeforeItsOtherBranchDependency() {+ Stack stack = new Stack();+ stack.setStackName("delete-branch-cycle-and-held-branch");+ stack.setStackId("stack-id");+ stack.setRegion(REGION);+ stack.setStatus("UPDATE_ROLLBACK_FAILED");+ stack.setTemplateBody("""+ {+ "Parameters": {"Placement": {"Type": "String"}},+ "Conditions": {+ "Second": {"Fn::Equals": [{"Ref": "Placement"}, "second"]},+ "First": {"Fn::Not": [{"Condition": "Second"}]}+ },+ "Resources": {+ "Alb": {"Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",+ "Properties": {"Subnets": {"Fn::If": ["Second", [{"Ref": "SubnetC"}], [{"Ref": "SubnetA"}]]}}},+ "SubnetA": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": "vpc-1", "CidrBlock": "10.0.1.0/24"}},+ "SubnetB": {"Type": "AWS::EC2::Subnet", "DependsOn": "SubnetA",+ "Properties": {"VpcId": "vpc-1", "CidrBlock": "10.0.2.0/24"}},+ "SubnetC": {"Type": "AWS::EC2::Subnet", "DependsOn": "SubnetB",+ "Properties": {"VpcId": "vpc-1", "CidrBlock": "10.0.3.0/24"}},+ "Y": {"Type": "AWS::SQS::Queue",+ "Properties": {"Tags": {"Fn::If": ["First", [{"Key": "x", "Value": {"Ref": "X"}}],+ {"Ref": "AWS::NoValue"}]}}},+ "X": {"Type": "AWS::SQS::Queue",+ "Properties": {"Tags": {"Fn::If": ["Second", [{"Key": "y", "Value": {"Ref": "Y"}}],+ {"Ref": "AWS::NoValue"}]}}}+ }+ }+ """);+ stack.replaceParameters(Map.of("Placement", "first"));+ StackResource loadBalancer = resource("Alb", "arn:aws:elasticloadbalancing:us-east-1:000000000000:"+ + "loadbalancer/app/web/1", "AWS::ElasticLoadBalancingV2::LoadBalancer", "UPDATE_FAILED");+ StackResource subnetA = resource("SubnetA", "subnet-a", "AWS::EC2::Subnet", "CREATE_COMPLETE");+ StackResource subnetB = resource("SubnetB", "subnet-b", "AWS::EC2::Subnet", "CREATE_COMPLETE");+ StackResource subnetC = resource("SubnetC", "subnet-c", "AWS::EC2::Subnet", "CREATE_COMPLETE");+ StackResource y = resource("Y", "queue-y", "AWS::SQS::Queue", "CREATE_COMPLETE");+ StackResource x = resource("X", "queue-x", "AWS::SQS::Queue", "CREATE_COMPLETE");+ for (StackResource resource : new StackResource[] {loadBalancer, subnetA, subnetB, subnetC, y, x}) {+ stack.getResources().put(resource.getLogicalId(), resource);+ }+ when(provisioner.completeDeleteCleanup(any())).thenReturn(UpdateCleanupResult.notApplicable());++ service.deleteStackResources(stack, REGION, ACCOUNT);++ InOrder subnets = inOrder(provisioner);+ subnets.verify(provisioner).delete(loadBalancer, REGION);+ subnets.verify(provisioner).delete(subnetC, REGION);+ InOrder queues = inOrder(provisioner);+ queues.verify(provisioner).delete(y, REGION);+ queues.verify(provisioner).delete(x, REGION);+ }++ /**+ * A resource the template no longer names, such as a subnet ContinueUpdateRollback skipped, has+ * no template entry left to order it by. It still goes as late as possible, after the load+ * balancer that may still use it, but before the VPC its attributes name, or that VPC delete+ * fails while the subnet exists.+ */+ @Test+ void deleteStack_deletesALeftoverResourceBeforeTheResourceItsAttributesName() {+ Stack stack = new Stack();+ stack.setStackName("delete-leftover-subnet");+ stack.setStackId("stack-id");+ stack.setRegion(REGION);+ stack.setStatus("UPDATE_ROLLBACK_COMPLETE");+ stack.setTemplateBody("""+ {"Resources": {+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.0.0.0/16"}},+ "SubnetA": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "10.0.1.0/24"}},+ "Alb": {"Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",+ "Properties": {"Subnets": [{"Ref": "SubnetA"}]}}+ }}""");+ StackResource leftover = resource("SubnetC", "subnet-c", "AWS::EC2::Subnet", "UPDATE_COMPLETE");+ leftover.getAttributes().put("VpcId", "vpc-1");+ StackResource vpc = resource("Vpc", "vpc-1", "AWS::EC2::VPC", "CREATE_COMPLETE");+ StackResource subnet = resource("SubnetA", "subnet-a", "AWS::EC2::Subnet", "CREATE_COMPLETE");+ subnet.getAttributes().put("VpcId", "vpc-1");+ StackResource loadBalancer = resource("Alb", "arn:aws:elasticloadbalancing:us-east-1:000000000000:"+ + "loadbalancer/app/web/1", "AWS::ElasticLoadBalancingV2::LoadBalancer", "UPDATE_COMPLETE");+ for (StackResource resource : new StackResource[] {vpc, subnet, loadBalancer, leftover}) {+ stack.getResources().put(resource.getLogicalId(), resource);+ }+ when(provisioner.completeDeleteCleanup(any())).thenReturn(UpdateCleanupResult.notApplicable());++ service.deleteStackResources(stack, REGION, ACCOUNT);++ InOrder deletes = inOrder(provisioner);+ deletes.verify(provisioner).delete(loadBalancer, REGION);+ deletes.verify(provisioner).delete(leftover, REGION);+ deletes.verify(provisioner).delete(vpc, REGION);+ }++ /**+ * A leftover may name another leftover: a VPC an update added and a subnet in it, both left by+ * a failed rollback. The subnet goes before its VPC, which names only itself.+ */+ @Test+ void deleteStack_deletesALeftoverBeforeTheLeftoverItsAttributesName() {+ Stack stack = new Stack();+ stack.setStackName("delete-leftover-chain");+ stack.setStackId("stack-id");+ stack.setRegion(REGION);+ stack.setStatus("UPDATE_ROLLBACK_FAILED");+ stack.setTemplateBody("""+ {"Resources": {+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.0.0.0/16"}}+ }}""");+ StackResource vpc = resource("Vpc", "vpc-1", "AWS::EC2::VPC", "CREATE_COMPLETE");+ vpc.getAttributes().put("VpcId", "vpc-1");+ StackResource leftoverSubnet = resource("SubnetX", "subnet-x", "AWS::EC2::Subnet", "UPDATE_COMPLETE");+ leftoverSubnet.getAttributes().put("SubnetId", "subnet-x");+ leftoverSubnet.getAttributes().put("VpcId", "vpc-2");+ StackResource leftoverVpc = resource("Vpc2", "vpc-2", "AWS::EC2::VPC", "UPDATE_COMPLETE");+ leftoverVpc.getAttributes().put("VpcId", "vpc-2");+ for (StackResource resource : new StackResource[] {vpc, leftoverSubnet, leftoverVpc}) {+ stack.getResources().put(resource.getLogicalId(), resource);+ }+ when(provisioner.completeDeleteCleanup(any())).thenReturn(UpdateCleanupResult.notApplicable());++ service.deleteStackResources(stack, REGION, ACCOUNT);++ InOrder deletes = inOrder(provisioner);+ deletes.verify(provisioner).delete(leftoverSubnet, REGION);+ deletes.verify(provisioner).delete(leftoverVpc, REGION);+ }+/*** The resume runs on the operation executor, whose future nobody reads. An unexpected failure* must still settle the stack in UPDATE_ROLLBACK_FAILED, keeping what a retry needs, insteaddiff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisionerTest.javaindex 9f8783127d..2978acd4ee 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisionerTest.java@@ -23,6 +23,7 @@ import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Set;+import java.util.stream.Collectors;import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -192,7 +193,30 @@ class Ec2NetworkCfnProvisionerTest {provisioner.provision(r, mapper.createObjectNode().put("SubnetId", SUBNET_ID).put("AllocationId", ALLOC_ID), ctx());assertEquals(NAT_ID, r.getPhysicalId());- assertEquals(Set.of("NatGatewayId"), r.getAttributes().keySet());+ assertEquals(Set.of("NatGatewayId"), r.getAttributes().keySet().stream()+ .filter(key -> !key.startsWith("__Floci")).collect(Collectors.toSet()));+ }++ /**+ * A gateway records the subnet and the Elastic IP it holds under hidden keys, by their physical+ * ids, so a stack that no longer names it still deletes it before them.+ */+ @Test+ void natGatewayRecordsTheSubnetAndElasticIpItUsesUnderHiddenKeys() {+ NatGateway nat = new NatGateway();+ nat.setNatGatewayId(NAT_ID);+ nat.setSubnetId(SUBNET_ID);+ when(ec2.createNatGateway(REGION, SUBNET_ID, ALLOC_ID, "public", List.of())).thenReturn(nat);+ Address addr = new Address();+ addr.setAllocationId(ALLOC_ID);+ addr.setPublicIp(PUBLIC_IP);+ when(ec2.describeAddresses(REGION, List.of(ALLOC_ID), Map.of())).thenReturn(List.of(addr));+ StackResource r = resource("AWS::EC2::NatGateway", "Nat");++ provisioner.provision(r, mapper.createObjectNode().put("SubnetId", SUBNET_ID).put("AllocationId", ALLOC_ID), ctx());++ assertEquals(SUBNET_ID, r.getAttributes().get(CfnRollback.USES_ATTR_PREFIX + "SubnetId"));+ assertEquals(PUBLIC_IP, r.getAttributes().get(CfnRollback.USES_ATTR_PREFIX + "PublicIp"));}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanupTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanupTest.javaindex 0db5388a26..511df481fd 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanupTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanupTest.java@@ -285,6 +285,29 @@ class ReplacementCleanupTest {assertEquals(List.of("subnet-2"), deleted);}+ /**+ * A revert puts back what the prior entity uses, as it does its public attributes, while other+ * hidden keys, rollback bookkeeping, stay as they are.+ */+ @Test+ void rollback_restoresThePriorUsesAttributesAndKeepsOtherHiddenKeys() {+ StackResource r = resource("subnet-1");+ r.getAttributes().put("VpcId", "vpc-new");+ r.getAttributes().put(CfnRollback.USES_ATTR_PREFIX + "PublicIp", "203.0.113.2");+ r.getAttributes().put(CfnRollback.ROLLBACK_OWNED_ATTR, "true");+ Map<String, String> before = Map.of("VpcId", "vpc-old",+ CfnRollback.USES_ATTR_PREFIX + "PublicIp", "203.0.113.1", CfnRollback.ROLLBACK_OWNED_ATTR, "false");+ r.setPhysicalId("subnet-2");+ ReplacementCleanup.record(r, update("subnet-1"), before);++ assertTrue(ReplacementCleanup.rollback(r, (type, id, region) -> { }));++ assertEquals("subnet-1", r.getPhysicalId());+ assertEquals("vpc-old", r.getAttributes().get("VpcId"));+ assertEquals("203.0.113.1", r.getAttributes().get(CfnRollback.USES_ATTR_PREFIX + "PublicIp"));+ assertEquals("true", r.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));+ }+/** A new provision supersedes the failed rollback; the replacement stays owed to the next cleanup instead. */@Testvoid record_afterAFailedRollbackDelete_doesNotRetryThatDeleteOnTheNextRollback() {