Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudformation): delete dependents first on update rollback (#5943)

Jackson Priced273d4f4e0aa
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
index fbaac39423..8f69538f17 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
@@ -3,6 +3,8 @@ package io.github.hectorvent.floci.services.cloudformation;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
@@ -2792,7 +2794,11 @@ public class CloudFormationService implements ResourceProvider {
Map<String, StackResource> previousResources,
Set<String> attemptedResourceIds,
String region) {
- List<StackResource> resources = new ArrayList<>(stack.resourcesSnapshot().values());
+ // Dependents first, in the template's order rather than the map's, as the update cleanup
+ // walks it: a resource the update added sits after the resources that depend on it, so
+ // reversing the map would delete a subnet the update added before an instance or NAT
+ // gateway the update put on it, and EC2 refuses that delete.
+ List<StackResource> resources = resourcesInCreationOrder(stack, region);
Collections.reverse(resources);
List<String> failures = new ArrayList<>();
List<String> removedResources = new ArrayList<>();
@@ -3108,13 +3114,22 @@ public class CloudFormationService implements ResourceProvider {
}
/**
- * The stack's resources in the order the current template creates them, for a delete that walks
- * them backwards. The resource map keeps insertion order, and a resource added by a later
- * update sits after the resources that depend on it, so reversing the map would delete it
- * first: a certificate still used by a user pool domain, for one. Resources the template no
- * longer names, left behind by a failed cleanup, sort first here so they are deleted last,
- * after anything that might still use them. Insertion order is the fallback when the template
- * cannot be read.
+ * The stack's resources in the order the current template creates them, for a delete, an update
+ * cleanup or an update rollback that walks them backwards. The resource map keeps insertion
+ * order, and a resource added by a later update sits after the resources that depend on it, so
+ * reversing the map would delete it first: a certificate still used by a user pool domain, for
+ * one. Resources the template no longer names, left behind by a failed cleanup or skipped by
+ * ContinueUpdateRollback, are deleted as late as possible, after anything that might still use
+ * them, but before any held resource whose physical id one of their attributes names, such as a
+ * subnet's {@code VpcId}: with their template entry gone, that is the only record of what they
+ * depend on, so a leftover whose attributes name nothing it uses is still deleted last.
+ * Conditions are ignored, both on resources and in {@code Fn::If}: a resource depends on what
+ * every branch names. The conditions read now need not be the ones a held resource was
+ * provisioned under, since a failed rollback restores the parameters but can leave a resource
+ * on the branch the failed update chose. When counting every branch closes a cycle, the walk
+ * starts from the branches chosen now, evaluated with the SSM-resolved parameter values when
+ * there are any, and adds each other-branch dependency that closes no cycle. Insertion order
+ * is the fallback when the template cannot be read.
*/
private List<StackResource> resourcesInCreationOrder(Stack stack, String region) {
List<StackResource> ordered = new ArrayList<>(stack.resourcesSnapshot().values());
@@ -3124,23 +3139,191 @@ public class CloudFormationService implements ResourceProvider {
if (!resources.isObject()) {
return ordered;
}
- Map<String, Boolean> conditions = resolveConditions(
- template, stack.parametersSnapshot(), stack, region, regionResolver.getAccountId());
+ // Ordered without conditions: only the resources the stack holds are walked, and each
+ // was provisioned, so its dependencies count whatever its condition reads now. A
+ // ContinueUpdateRollback retry reads the failed update's template with the parameters
+ // the first pass already restored, and a held resource may depend on a condition-false
+ // one an earlier cleanup deleted, which would otherwise fail the whole ordering.
+ ObjectNode unconditional = resources.deepCopy();
+ for (Map.Entry<String, JsonNode> entry : unconditional.properties()) {
+ if (entry.getValue() instanceof ObjectNode definition) {
+ definition.remove("Condition");
+ }
+ }
+ ObjectNode everyBranch = unconditional.deepCopy();
+ for (Map.Entry<String, JsonNode> entry : everyBranch.properties()) {
+ if (entry.getValue() instanceof ObjectNode definition && definition.has("Properties")) {
+ definition.set("Properties", everyIfBranch(definition.get("Properties")));
+ }
+ }
// A stack saved before circular templates were rejected may still have one; order it
// by its dependencies anyway rather than giving up.
- List<String> creationOrder = creationOrder(resources, conditions).sorted();
+ CreationOrder order = creationOrder(everyBranch, Map.of());
+ if (!order.circular().isEmpty()) {
+ order = withoutBranchCycles(order, stack, template, unconditional, everyBranch, region);
+ }
+ List<String> creationOrder = order.sorted();
Map<String, Integer> rank = new HashMap<>();
for (int i = 0; i < creationOrder.size(); i++) {
rank.put(creationOrder.get(i), i);
}
- ordered.sort(Comparator.comparingInt(r -> rank.getOrDefault(r.getLogicalId(), -1)));
+ Map<String, List<StackResource>> holdersByPhysicalId = new HashMap<>();
+ int leftovers = 0;
+ for (StackResource resource : ordered) {
+ if (!rank.containsKey(resource.getLogicalId())) {
+ leftovers++;
+ }
+ if (resource.getPhysicalId() != null) {
+ holdersByPhysicalId.computeIfAbsent(resource.getPhysicalId(), id -> new ArrayList<>())
+ .add(resource);
+ }
+ }
+ double step = 1.0 / (leftovers + 1);
+ Map<String, Double> position = new HashMap<>();
+ for (StackResource resource : ordered) {
+ position.put(resource.getLogicalId(),
+ leftoverPosition(resource, rank, holdersByPhysicalId, step, position, new HashSet<>()));
+ }
+ ordered.sort(Comparator.comparingDouble(r -> position.get(r.getLogicalId())));
} catch (Exception e) {
- LOG.debugv("Deleting stack {0} in insertion order, its template could not be ordered: {1}",
+ LOG.debugv("Walking stack {0} in insertion order, its template could not be ordered: {1}",
stack.getStackName(), e.getMessage());
}
return ordered;
}
+ /**
+ * The order the branches chosen now give, with every dependency another {@code Fn::If} branch
+ * names added to it unless that dependency would close a cycle. Counting every branch can close
+ * a cycle no choice of branches has, such as two resources that each use the other under
+ * opposite conditions; dropping only the edges that close it keeps every other branch's
+ * dependencies. {@code everyBranch}, the order counting every branch, is kept when the
+ * conditions cannot be read.
+ */
+ private CreationOrder withoutBranchCycles(CreationOrder everyBranch, Stack stack, JsonNode template,
+ ObjectNode unconditional, ObjectNode everyBranchResources,
+ String region) {
+ try {
+ Map<String, String> resolvedParameters = stack.resolvedParametersSnapshot();
+ Map<String, Boolean> conditions = resolveConditions(template,
+ resolvedParameters.isEmpty() ? stack.parametersSnapshot() : resolvedParameters,
+ stack, region, regionResolver.getAccountId());
+ Set<String> ids = new LinkedHashSet<>();
+ unconditional.fieldNames().forEachRemaining(ids::add);
+ Map<String, Set<String>> dependencies = new HashMap<>();
+ for (String id : ids) {
+ Set<String> chosen = collectResourceDependencies(unconditional.get(id), ids, conditions);
+ chosen.retainAll(ids);
+ dependencies.put(id, chosen);
+ }
+ ObjectNode merged = unconditional.deepCopy();
+ for (String id : ids) {
+ for (String dependency : collectResourceDependencies(everyBranchResources.get(id), ids, Map.of())) {
+ if (ids.contains(dependency) && !dependency.equals(id)
+ && !dependencies.get(id).contains(dependency)
+ && !dependsOn(dependencies, dependency, id)) {
+ dependencies.get(id).add(dependency);
+ addDependsOn(merged.get(id), dependency);
+ }
+ }
+ }
+ return creationOrder(merged, conditions);
+ } catch (RuntimeException e) {
+ LOG.debugv("Ordering stack {0} by every Fn::If branch, its conditions could not be read: {1}",
+ stack.getStackName(), e.getMessage());
+ return everyBranch;
+ }
+ }
+
+ /** Whether {@code from} depends on {@code to}, directly or through other resources. */
+ private static boolean dependsOn(Map<String, Set<String>> dependencies, String from, String to) {
+ Deque<String> pending = new ArrayDeque<>(List.of(from));
+ Set<String> seen = new HashSet<>();
+ while (!pending.isEmpty()) {
+ String current = pending.pop();
+ if (current.equals(to)) {
+ return true;
+ }
+ if (seen.add(current)) {
+ pending.addAll(dependencies.getOrDefault(current, Set.of()));
+ }
+ }
+ return false;
+ }
+
+ /** Adds a resource to a definition's {@code DependsOn}, keeping what it already names. */
+ private static void addDependsOn(JsonNode definition, String dependency) {
+ if (!(definition instanceof ObjectNode object)) {
+ return;
+ }
+ JsonNode existing = object.path("DependsOn");
+ ArrayNode dependsOn = JsonNodeFactory.instance.arrayNode();
+ if (existing.isTextual()) {
+ dependsOn.add(existing.asText());
+ } else if (existing.isArray()) {
+ dependsOn.addAll((ArrayNode) existing);
+ }
+ dependsOn.add(dependency);
+ object.set("DependsOn", dependsOn);
+ }
+
+ /**
+ * A copy of a template value with each {@code Fn::If} replaced by a list of both its branches,
+ * so the dependencies collected from it are what either branch names.
+ */
+ private static JsonNode everyIfBranch(JsonNode node) {
+ if (node.isObject()) {
+ JsonNode fnIf = node.path("Fn::If");
+ if (fnIf.isArray() && fnIf.size() == 3) {
+ ArrayNode branches = JsonNodeFactory.instance.arrayNode();
+ branches.add(everyIfBranch(fnIf.get(1)));
+ branches.add(everyIfBranch(fnIf.get(2)));
+ return branches;
+ }
+ ObjectNode copy = JsonNodeFactory.instance.objectNode();
+ node.properties().forEach(e -> copy.set(e.getKey(), everyIfBranch(e.getValue())));
+ return copy;
+ }
+ if (node.isArray()) {
+ ArrayNode copy = JsonNodeFactory.instance.arrayNode();
+ node.forEach(item -> copy.add(everyIfBranch(item)));
+ return copy;
+ }
+ return node;
+ }
+
+ /**
+ * A resource's place in {@link #resourcesInCreationOrder}: its template rank, or for a leftover
+ * one step after the latest held resource its attributes name, leftovers included, or else -1.
+ * A chain of leftovers stays below the next template rank, and a cycle among them is cut where
+ * it closes.
+ */
+ private static double leftoverPosition(StackResource resource, Map<String, Integer> rank,
+ Map<String, List<StackResource>> holdersByPhysicalId,
+ double step, Map<String, Double> position, Set<String> visiting) {
+ Integer resourceRank = rank.get(resource.getLogicalId());
+ if (resourceRank != null) {
+ return resourceRank;
+ }
+ Double known = position.get(resource.getLogicalId());
+ if (known != null) {
+ return known;
+ }
+ visiting.add(resource.getLogicalId());
+ double placed = -1;
+ for (String value : resource.getAttributes().values()) {
+ for (StackResource holder : holdersByPhysicalId.getOrDefault(value, List.of())) {
+ if (!visiting.contains(holder.getLogicalId())) {
+ placed = Math.max(placed,
+ leftoverPosition(holder, rank, holdersByPhysicalId, step, position, visiting) + step);
+ }
+ }
+ }
+ visiting.remove(resource.getLogicalId());
+ position.put(resource.getLogicalId(), placed);
+ return placed;
+ }
+
void deleteStackResources(Stack stack, String region, String accountId) {
try {
List<StackResource> resources = resourcesInCreationOrder(stack, region);
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java
index 1171e41212..b8fe80811c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java
@@ -205,6 +205,14 @@ public final class CfnRollback {
*/
public static final String REPLACEMENT_CLEANUP_ATTR = "__FlociReplacementCleanup";
+ /**
+ * Prefixes attributes that name, by physical id, a resource this one uses where none of its
+ * public attributes does, such as a NAT gateway's subnet and Elastic IP. {@code
+ * CloudFormationService} reads every attribute value of a resource the template no longer
+ * names to delete it before what it uses; these keep {@code Fn::GetAtt} to what AWS resolves.
+ */
+ public static final String USES_ATTR_PREFIX = "__FlociUses";
+
/**
* Holds the tags, stream setting and time-to-live setting a DynamoDB table carried before an
* in-place update changed them, with the account and region that address it, so a failed
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisioner.java
index ec9efd8a8d..4998f4c482 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisioner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisioner.java
@@ -583,6 +583,17 @@ public class Ec2NetworkCfnProvisioner implements CfnResourceProvisioner {
}
r.setPhysicalId(nat.getNatGatewayId());
r.getAttributes().put("NatGatewayId", nat.getNatGatewayId());
+ // The gateway holds its subnet and Elastic IP until it is deleted; an EIP's physical id is
+ // its public IP.
+ if (nat.getSubnetId() != null) {
+ r.getAttributes().put(CfnRollback.USES_ATTR_PREFIX + "SubnetId", nat.getSubnetId());
+ }
+ Address eip = allocationId == null || allocationId.isBlank() ? null : findAddress(allocationId, null, ctx.region());
+ if (eip != null && eip.getPublicIp() != null) {
+ r.getAttributes().put(CfnRollback.USES_ATTR_PREFIX + "PublicIp", eip.getPublicIp());
+ } else {
+ r.getAttributes().remove(CfnRollback.USES_ATTR_PREFIX + "PublicIp");
+ }
}
private void provisionEip(StackResource r, JsonNode props, ProvisionContext ctx) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanup.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanup.java
index afeef16f20..ab3d26caa8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanup.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanup.java
@@ -93,7 +93,7 @@ public final class ReplacementCleanup {
cleanup.put("priorPhysicalId", prior);
ObjectNode priorAttributes = cleanup.putObject("priorAttributes");
attributesBeforeProvision.forEach((key, value) -> {
- if (!key.startsWith("__Floci") && value != null) {
+ if (restoredOnRevert(key) && value != null) {
priorAttributes.put(key, value);
}
});
@@ -153,7 +153,7 @@ public final class ReplacementCleanup {
r.setPhysicalId(prior);
Iterator<String> keys = r.getAttributes().keySet().iterator();
while (keys.hasNext()) {
- if (!keys.next().startsWith("__Floci")) {
+ if (restoredOnRevert(keys.next())) {
keys.remove();
}
}
@@ -174,6 +174,15 @@ public final class ReplacementCleanup {
return deleteReplacement(r, cleanup, deleter, r.getResourceType(), replacement);
}
+ /**
+ * Whether a revert puts back the prior value of an attribute: the public ones, and the hidden
+ * {@link CfnRollback#USES_ATTR_PREFIX} ones, which describe the entity as the public ones do.
+ * Other {@code __Floci} keys are rollback bookkeeping that outlives the revert.
+ */
+ private static boolean restoredOnRevert(String key) {
+ return !key.startsWith("__Floci") || key.startsWith(CfnRollback.USES_ATTR_PREFIX);
+ }
+
/** Deletes a rolled-back replacement; one whose delete throws stays owed and named for the retry. */
private static boolean deleteReplacement(StackResource r, ObjectNode cleanup, Deleter deleter,
String resourceType, String replacement) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationNatGatewayLeftoverIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationNatGatewayLeftoverIntegrationTest.java
new file mode 100644
index 0000000000..6b70dc305e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationNatGatewayLeftoverIntegrationTest.java
@@ -0,0 +1,138 @@
+package io.github.hectorvent.floci.services.cloudformation;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.XmlParser;
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.mockito.InjectSpy;
+import io.restassured.response.ValidatableResponse;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.reset;
+
+/**
+ * A NAT gateway the template no longer names, left by a rollback that skipped it, still holds its
+ * Elastic IP and its subnet, so DeleteStack has to delete it before either of them, though only
+ * its hidden attributes say what it uses.
+ */
+@QuarkusTest
+class CloudFormationNatGatewayLeftoverIntegrationTest {
+
+ private static final String CFN_AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20261015/us-east-1/cloudformation/aws4_request";
+ private static final String EC2_AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20261015/us-east-1/ec2/aws4_request";
+
+ @InjectSpy
+ Ec2Service ec2Service;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void deleteStackDeletesASkippedNatGatewayBeforeItsElasticIpSubnetAndVpc() {
+ String stack = "nat-leftover-" + Long.toString(System.nanoTime(), 36);
+ AtomicBoolean failNatDeletes = new AtomicBoolean(true);
+ doAnswer(invocation -> {
+ if (failNatDeletes.get()) {
+ throw new AwsException("InternalError", "simulated NAT gateway delete failure", 500);
+ }
+ return invocation.callRealMethod();
+ }).when(ec2Service).deleteNatGateway(anyString(), anyString());
+ try {
+ Map<String, String> created = createStack(stack, template(stack, ""));
+ String nat = """
+ ,
+ "Nat": {"Type": "AWS::EC2::NatGateway",
+ "Properties": {"SubnetId": {"Ref": "Subnet"},
+ "AllocationId": {"Fn::GetAtt": ["Eip", "AllocationId"]}}},
+ "BadSecret": {"Type": "AWS::SecretsManager::Secret", "DependsOn": "Nat",
+ "Properties": {"Name": "%1$s-secret", "SecretString": "explicit",
+ "GenerateSecretString": {"PasswordLength": 32}}}""".formatted(stack);
+ request("UpdateStack", "StackName", stack, "TemplateBody", template(stack, nat)).statusCode(200);
+ String reason = CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_FAILED").reason();
+ assertTrue(reason.contains("Nat"), reason);
+ failNatDeletes.set(false);
+
+ request("ContinueUpdateRollback", "StackName", stack, "ResourcesToSkip.member.1", "Nat").statusCode(200);
+ CfnStackWaits.awaitStatus(stack, "UPDATE_ROLLBACK_COMPLETE");
+ assertEquals(1, XmlParser.extractAll(ec2("DescribeNatGateways", "Filter.1.Name", "subnet-id",
+ "Filter.1.Value.1", created.get("Subnet")).statusCode(200).extract().asString(), "natGatewayId")
+ .size(), "the skipped gateway is left in place");
+
+ request("DeleteStack", "StackName", stack).statusCode(200);
+ CfnStackWaits.awaitStackDeleted(stack, CFN_AUTH);
+ ec2("DescribeVpcs", "VpcId.1", created.get("Vpc")).statusCode(400)
+ .body(containsString("InvalidVpcID.NotFound"));
+ assertEquals(List.of(), XmlParser.extractAll(ec2("DescribeAddresses", "PublicIp.1", created.get("Eip"))
+ .extract().asString(), "allocationId"));
+ } finally {
+ failNatDeletes.set(false);
+ reset(ec2Service);
+ // Best effort: a failed run may leave the stack DELETE_FAILED or mid-rollback.
+ if (request("DescribeStacks", "StackName", stack).extract().statusCode() == 200) {
+ request("DeleteStack", "StackName", stack);
+ }
+ }
+ }
+
+ /** A VPC with one subnet and an Elastic IP, plus whatever an update adds. */
+ private static String template(String stack, String extraResources) {
+ return """
+ {
+ "Resources": {
+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.96.0.0/16",
+ "Tags": [{"Key": "floci-it", "Value": "%1$s"}]}},
+ "Subnet": {"Type": "AWS::EC2::Subnet",
+ "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "10.96.1.0/24",
+ "AvailabilityZone": "us-east-1a"}},
+ "Eip": {"Type": "AWS::EC2::EIP", "Properties": {"Domain": "vpc"}}%2$s
+ },
+ "Outputs": {
+ "Vpc": {"Value": {"Ref": "Vpc"}},
+ "Subnet": {"Value": {"Ref": "Subnet"}},
+ "Eip": {"Value": {"Ref": "Eip"}}
+ }
+ }""".formatted(stack, extraResources);
+ }
+
+ private static Map<String, String> createStack(String stack, String templateBody) {
+ request("CreateStack", "StackName", stack, "TemplateBody", templateBody).statusCode(200);
+ CfnStackWaits.awaitStatus(stack, "CREATE_COMPLETE");
+ String xml = request("DescribeStacks", "StackName", stack).statusCode(200).extract().asString();
+ return XmlParser.extractPairs(xml, "member", "OutputKey", "OutputValue");
+ }
+
+ private static ValidatableResponse ec2(String action, String... values) {
+ return send(EC2_AUTH, action, values);
+ }
+
+ private static ValidatableResponse request(String action, String... values) {
+ return send(CFN_AUTH, action, values);
+ }
+
+ private static ValidatableResponse send(String auth, String action, String... values) {
+ Map<String, String> parameters = new LinkedHashMap<>();
+ parameters.put("Action", action);
+ for (int i = 0; i < values.length; i += 2) {
+ parameters.put(values[i], values[i + 1]);
+ }
+ return given().contentType("application/x-www-form-urlencoded").header("Authorization", auth)
+ .formParams(parameters).when().post("/").then().log().ifValidationFails();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java
index 5dcf83ee49..878e9dd4cf 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java
@@ -783,6 +783,275 @@ class CloudFormationServiceRollbackTest {
verify(provisioner, times(2)).rollbackUpdate(eq(dashboard), any());
}
+ /**
+ * A retry walks the failed update's template with the parameters the first pass already
+ * restored, so a condition the update turned on can read false again. A resource the stack
+ * holds is still ordered by its dependencies, so the load balancer the update created on the
+ * subnet it added is deleted before that subnet, not after it. The load balancer is inserted
+ * first, so falling back to the map's order would delete the subnet first too.
+ */
+ @Test
+ void resumeUpdateRollback_ordersAHeldResourceByItsDependenciesWhateverItsConditionReadsNow() {
+ Stack stack = new Stack();
+ stack.setStackName("retry-flipped-condition");
+ stack.setStackId("stack-id");
+ stack.setRegion(REGION);
+ stack.setTemplateBody("""
+ {
+ "Parameters": {"Env": {"Type": "String"}},
+ "Conditions": {"IsProd": {"Fn::Equals": [{"Ref": "Env"}, "prod"]}},
+ "Resources": {
+ "SubnetC": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": "vpc-1", "CidrBlock": "10.0.3.0/24"}},
+ "ProdAlb": {
+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
+ "Condition": "IsProd",
+ "Properties": {"Subnets": [{"Ref": "SubnetC"}]}
+ }
+ }
+ }
+ """);
+ stack.replaceParameters(Map.of("Env", "dev"));
+ stack.replaceResolvedParameters(Map.of("Env", "dev"));
+ StackResource subnet = resource("SubnetC", "subnet-c", "AWS::EC2::Subnet", "CREATE_COMPLETE");
+ StackResource loadBalancer = resource("ProdAlb", "arn:aws:elasticloadbalancing:us-east-1:000000000000:"
+ + "loadbalancer/app/prod/1", "AWS::ElasticLoadBalancingV2::LoadBalancer", "CREATE_COMPLETE");
+ stack.getResources().put(loadBalancer.getLogicalId(), loadBalancer);
+ stack.getResources().put(subnet.getLogicalId(), subnet);
+ stack.setUpdateRollbackSnapshot(new StackUpdateSnapshot("previous-template", "previous-original",
+ Map.of("Env", "dev"), Map.of("Env", "dev"), Map.of(), Map.of(), Map.of(), Map.of()));
+ stack.setUpdateRollbackResourceIds(List.of("SubnetC", "ProdAlb"));
+ stack.setStatus("UPDATE_ROLLBACK_IN_PROGRESS");
+
+ service.resumeUpdateRollback(stack, List.of(), false, REGION);
+
+ assertEquals("UPDATE_ROLLBACK_COMPLETE", stack.getStatus());
+ InOrder deletes = inOrder(provisioner);
+ deletes.verify(provisioner).delete(eq(loadBalancer), anyString());
+ deletes.verify(provisioner).delete(eq(subnet), anyString());
+ }
+
+ /**
+ * A held resource can depend on a condition-false one the stack no longer holds, such as a role
+ * an earlier cleanup deleted while the load balancer on it failed to delete. Ordering must not
+ * then give up on the whole template: the load balancer still goes before the subnet it uses.
+ */
+ @Test
+ void deleteStack_withAHeldResourceOnAGoneConditionFalseOne_stillDeletesDependentsFirst() {
+ Stack stack = new Stack();
+ stack.setStackName("delete-held-on-gone-conditional");
+ stack.setStackId("stack-id");
+ stack.setRegion(REGION);
+ stack.setStatus("UPDATE_COMPLETE");
+ stack.setTemplateBody("""
+ {
+ "Parameters": {"Env": {"Type": "String"}},
+ "Conditions": {"IsProd": {"Fn::Equals": [{"Ref": "Env"}, "prod"]}},
+ "Resources": {
+ "ProdRole": {"Type": "AWS::IAM::Role", "Condition": "IsProd"},
+ "SubnetC": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": "vpc-1", "CidrBlock": "10.0.3.0/24"}},
+ "ProdAlb": {
+ "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
+ "Condition": "IsProd",
+ "DependsOn": "ProdRole",
+ "Properties": {"Subnets": [{"Ref": "SubnetC"}]}
+ }
+ }
+ }
+ """);
+ stack.replaceParameters(Map.of("Env", "dev"));
+ stack.replaceResolvedParameters(Map.of("Env", "dev"));
+ StackResource loadBalancer = resource("ProdAlb", "arn:aws:elasticloadbalancing:us-east-1:000000000000:"
+ + "loadbalancer/app/prod/1", "AWS::ElasticLoadBalancingV2::LoadBalancer", "DELETE_FAILED");
+ StackResource subnet = resource("SubnetC", "subnet-c", "AWS::EC2::Subnet", "CREATE_COMPLETE");
+ stack.getResources().put(loadBalancer.getLogicalId(), loadBalancer);
+ stack.getResources().put(subnet.getLogicalId(), subnet);
+ when(provisioner.completeDeleteCleanup(any())).thenReturn(UpdateCleanupResult.notApplicable());
+
+ service.deleteStackResources(stack, REGION, ACCOUNT);
+
+ InOrder deletes = inOrder(provisioner);
+ deletes.verify(provisioner).delete(loadBalancer, REGION);
+ deletes.verify(provisioner).delete(subnet, REGION);
+ }
+
+ /**
+ * Counting every Fn::If branch can close a cycle no choice of branches has: here A uses B when
+ * Flip is true, and B uses A when it is false. The walk then orders by the branches the
+ * conditions choose now, so A, which uses B, still goes first.
+ */
+ @Test
+ void deleteStack_whenEveryBranchWouldCloseACycle_ordersByTheBranchesChosenNow() {
+ Stack stack = new Stack();
+ stack.setStackName("delete-branch-cycle");
+ stack.setStackId("stack-id");
+ stack.setRegion(REGION);
+ stack.setStatus("CREATE_COMPLETE");
+ stack.setTemplateBody("""
+ {
+ "Parameters": {"Flip": {"Type": "String"}},
+ "Conditions": {
+ "Flipped": {"Fn::Equals": [{"Ref": "Flip"}, "yes"]},
+ "NotFlipped": {"Fn::Not": [{"Condition": "Flipped"}]}
+ },
+ "Resources": {
+ "B": {"Type": "AWS::SQS::Queue",
+ "Properties": {"Tags": {"Fn::If": ["NotFlipped", [{"Key": "a", "Value": {"Ref": "A"}}],
+ {"Ref": "AWS::NoValue"}]}}},
+ "A": {"Type": "AWS::SQS::Queue",
+ "Properties": {"Tags": {"Fn::If": ["Flipped", [{"Key": "b", "Value": {"Ref": "B"}}],
+ {"Ref": "AWS::NoValue"}]}}}
+ }
+ }
+ """);
+ stack.replaceParameters(Map.of("Flip", "yes"));
+ StackResource b = resource("B", "queue-b", "AWS::SQS::Queue", "CREATE_COMPLETE");
+ StackResource a = resource("A", "queue-a", "AWS::SQS::Queue", "CREATE_COMPLETE");
+ stack.getResources().put(b.getLogicalId(), b);
+ stack.getResources().put(a.getLogicalId(), a);
+ when(provisioner.completeDeleteCleanup(any())).thenReturn(UpdateCleanupResult.notApplicable());
+
+ service.deleteStackResources(stack, REGION, ACCOUNT);
+
+ InOrder deletes = inOrder(provisioner);
+ deletes.verify(provisioner).delete(a, REGION);
+ deletes.verify(provisioner).delete(b, REGION);
+ }
+
+ /**
+ * A cycle that only counting every branch closes, between X and Y, must not cost the rest of
+ * the stack its other-branch dependencies: the load balancer still on the subnet the failed
+ * update chose goes before that subnet, though the restored parameter now picks the other.
+ * SubnetC is created two steps after SubnetA, so only that dependency puts the load balancer
+ * after it.
+ */
+ @Test
+ void deleteStack_withAnUnrelatedBranchCycle_stillDeletesAResourceBeforeItsOtherBranchDependency() {
+ Stack stack = new Stack();
+ stack.setStackName("delete-branch-cycle-and-held-branch");
+ stack.setStackId("stack-id");
+ stack.setRegion(REGION);
+ stack.setStatus("UPDATE_ROLLBACK_FAILED");
+ stack.setTemplateBody("""
+ {
+ "Parameters": {"Placement": {"Type": "String"}},
+ "Conditions": {
+ "Second": {"Fn::Equals": [{"Ref": "Placement"}, "second"]},
+ "First": {"Fn::Not": [{"Condition": "Second"}]}
+ },
+ "Resources": {
+ "Alb": {"Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
+ "Properties": {"Subnets": {"Fn::If": ["Second", [{"Ref": "SubnetC"}], [{"Ref": "SubnetA"}]]}}},
+ "SubnetA": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": "vpc-1", "CidrBlock": "10.0.1.0/24"}},
+ "SubnetB": {"Type": "AWS::EC2::Subnet", "DependsOn": "SubnetA",
+ "Properties": {"VpcId": "vpc-1", "CidrBlock": "10.0.2.0/24"}},
+ "SubnetC": {"Type": "AWS::EC2::Subnet", "DependsOn": "SubnetB",
+ "Properties": {"VpcId": "vpc-1", "CidrBlock": "10.0.3.0/24"}},
+ "Y": {"Type": "AWS::SQS::Queue",
+ "Properties": {"Tags": {"Fn::If": ["First", [{"Key": "x", "Value": {"Ref": "X"}}],
+ {"Ref": "AWS::NoValue"}]}}},
+ "X": {"Type": "AWS::SQS::Queue",
+ "Properties": {"Tags": {"Fn::If": ["Second", [{"Key": "y", "Value": {"Ref": "Y"}}],
+ {"Ref": "AWS::NoValue"}]}}}
+ }
+ }
+ """);
+ stack.replaceParameters(Map.of("Placement", "first"));
+ StackResource loadBalancer = resource("Alb", "arn:aws:elasticloadbalancing:us-east-1:000000000000:"
+ + "loadbalancer/app/web/1", "AWS::ElasticLoadBalancingV2::LoadBalancer", "UPDATE_FAILED");
+ StackResource subnetA = resource("SubnetA", "subnet-a", "AWS::EC2::Subnet", "CREATE_COMPLETE");
+ StackResource subnetB = resource("SubnetB", "subnet-b", "AWS::EC2::Subnet", "CREATE_COMPLETE");
+ StackResource subnetC = resource("SubnetC", "subnet-c", "AWS::EC2::Subnet", "CREATE_COMPLETE");
+ StackResource y = resource("Y", "queue-y", "AWS::SQS::Queue", "CREATE_COMPLETE");
+ StackResource x = resource("X", "queue-x", "AWS::SQS::Queue", "CREATE_COMPLETE");
+ for (StackResource resource : new StackResource[] {loadBalancer, subnetA, subnetB, subnetC, y, x}) {
+ stack.getResources().put(resource.getLogicalId(), resource);
+ }
+ when(provisioner.completeDeleteCleanup(any())).thenReturn(UpdateCleanupResult.notApplicable());
+
+ service.deleteStackResources(stack, REGION, ACCOUNT);
+
+ InOrder subnets = inOrder(provisioner);
+ subnets.verify(provisioner).delete(loadBalancer, REGION);
+ subnets.verify(provisioner).delete(subnetC, REGION);
+ InOrder queues = inOrder(provisioner);
+ queues.verify(provisioner).delete(y, REGION);
+ queues.verify(provisioner).delete(x, REGION);
+ }
+
+ /**
+ * A resource the template no longer names, such as a subnet ContinueUpdateRollback skipped, has
+ * no template entry left to order it by. It still goes as late as possible, after the load
+ * balancer that may still use it, but before the VPC its attributes name, or that VPC delete
+ * fails while the subnet exists.
+ */
+ @Test
+ void deleteStack_deletesALeftoverResourceBeforeTheResourceItsAttributesName() {
+ Stack stack = new Stack();
+ stack.setStackName("delete-leftover-subnet");
+ stack.setStackId("stack-id");
+ stack.setRegion(REGION);
+ stack.setStatus("UPDATE_ROLLBACK_COMPLETE");
+ stack.setTemplateBody("""
+ {"Resources": {
+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.0.0.0/16"}},
+ "SubnetA": {"Type": "AWS::EC2::Subnet", "Properties": {"VpcId": {"Ref": "Vpc"}, "CidrBlock": "10.0.1.0/24"}},
+ "Alb": {"Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
+ "Properties": {"Subnets": [{"Ref": "SubnetA"}]}}
+ }}""");
+ StackResource leftover = resource("SubnetC", "subnet-c", "AWS::EC2::Subnet", "UPDATE_COMPLETE");
+ leftover.getAttributes().put("VpcId", "vpc-1");
+ StackResource vpc = resource("Vpc", "vpc-1", "AWS::EC2::VPC", "CREATE_COMPLETE");
+ StackResource subnet = resource("SubnetA", "subnet-a", "AWS::EC2::Subnet", "CREATE_COMPLETE");
+ subnet.getAttributes().put("VpcId", "vpc-1");
+ StackResource loadBalancer = resource("Alb", "arn:aws:elasticloadbalancing:us-east-1:000000000000:"
+ + "loadbalancer/app/web/1", "AWS::ElasticLoadBalancingV2::LoadBalancer", "UPDATE_COMPLETE");
+ for (StackResource resource : new StackResource[] {vpc, subnet, loadBalancer, leftover}) {
+ stack.getResources().put(resource.getLogicalId(), resource);
+ }
+ when(provisioner.completeDeleteCleanup(any())).thenReturn(UpdateCleanupResult.notApplicable());
+
+ service.deleteStackResources(stack, REGION, ACCOUNT);
+
+ InOrder deletes = inOrder(provisioner);
+ deletes.verify(provisioner).delete(loadBalancer, REGION);
+ deletes.verify(provisioner).delete(leftover, REGION);
+ deletes.verify(provisioner).delete(vpc, REGION);
+ }
+
+ /**
+ * A leftover may name another leftover: a VPC an update added and a subnet in it, both left by
+ * a failed rollback. The subnet goes before its VPC, which names only itself.
+ */
+ @Test
+ void deleteStack_deletesALeftoverBeforeTheLeftoverItsAttributesName() {
+ Stack stack = new Stack();
+ stack.setStackName("delete-leftover-chain");
+ stack.setStackId("stack-id");
+ stack.setRegion(REGION);
+ stack.setStatus("UPDATE_ROLLBACK_FAILED");
+ stack.setTemplateBody("""
+ {"Resources": {
+ "Vpc": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.0.0.0/16"}}
+ }}""");
+ StackResource vpc = resource("Vpc", "vpc-1", "AWS::EC2::VPC", "CREATE_COMPLETE");
+ vpc.getAttributes().put("VpcId", "vpc-1");
+ StackResource leftoverSubnet = resource("SubnetX", "subnet-x", "AWS::EC2::Subnet", "UPDATE_COMPLETE");
+ leftoverSubnet.getAttributes().put("SubnetId", "subnet-x");
+ leftoverSubnet.getAttributes().put("VpcId", "vpc-2");
+ StackResource leftoverVpc = resource("Vpc2", "vpc-2", "AWS::EC2::VPC", "UPDATE_COMPLETE");
+ leftoverVpc.getAttributes().put("VpcId", "vpc-2");
+ for (StackResource resource : new StackResource[] {vpc, leftoverSubnet, leftoverVpc}) {
+ stack.getResources().put(resource.getLogicalId(), resource);
+ }
+ when(provisioner.completeDeleteCleanup(any())).thenReturn(UpdateCleanupResult.notApplicable());
+
+ service.deleteStackResources(stack, REGION, ACCOUNT);
+
+ InOrder deletes = inOrder(provisioner);
+ deletes.verify(provisioner).delete(leftoverSubnet, REGION);
+ deletes.verify(provisioner).delete(leftoverVpc, REGION);
+ }
+
/**
* The resume runs on the operation executor, whose future nobody reads. An unexpected failure
* must still settle the stack in UPDATE_ROLLBACK_FAILED, keeping what a retry needs, instead
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisionerTest.java
index 9f8783127d..2978acd4ee 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisionerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2NetworkCfnProvisionerTest.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -192,7 +193,30 @@ class Ec2NetworkCfnProvisionerTest {
provisioner.provision(r, mapper.createObjectNode().put("SubnetId", SUBNET_ID).put("AllocationId", ALLOC_ID), ctx());
assertEquals(NAT_ID, r.getPhysicalId());
- assertEquals(Set.of("NatGatewayId"), r.getAttributes().keySet());
+ assertEquals(Set.of("NatGatewayId"), r.getAttributes().keySet().stream()
+ .filter(key -> !key.startsWith("__Floci")).collect(Collectors.toSet()));
+ }
+
+ /**
+ * A gateway records the subnet and the Elastic IP it holds under hidden keys, by their physical
+ * ids, so a stack that no longer names it still deletes it before them.
+ */
+ @Test
+ void natGatewayRecordsTheSubnetAndElasticIpItUsesUnderHiddenKeys() {
+ NatGateway nat = new NatGateway();
+ nat.setNatGatewayId(NAT_ID);
+ nat.setSubnetId(SUBNET_ID);
+ when(ec2.createNatGateway(REGION, SUBNET_ID, ALLOC_ID, "public", List.of())).thenReturn(nat);
+ Address addr = new Address();
+ addr.setAllocationId(ALLOC_ID);
+ addr.setPublicIp(PUBLIC_IP);
+ when(ec2.describeAddresses(REGION, List.of(ALLOC_ID), Map.of())).thenReturn(List.of(addr));
+ StackResource r = resource("AWS::EC2::NatGateway", "Nat");
+
+ provisioner.provision(r, mapper.createObjectNode().put("SubnetId", SUBNET_ID).put("AllocationId", ALLOC_ID), ctx());
+
+ assertEquals(SUBNET_ID, r.getAttributes().get(CfnRollback.USES_ATTR_PREFIX + "SubnetId"));
+ assertEquals(PUBLIC_IP, r.getAttributes().get(CfnRollback.USES_ATTR_PREFIX + "PublicIp"));
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanupTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanupTest.java
index 0db5388a26..511df481fd 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanupTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ReplacementCleanupTest.java
@@ -285,6 +285,29 @@ class ReplacementCleanupTest {
assertEquals(List.of("subnet-2"), deleted);
}
+ /**
+ * A revert puts back what the prior entity uses, as it does its public attributes, while other
+ * hidden keys, rollback bookkeeping, stay as they are.
+ */
+ @Test
+ void rollback_restoresThePriorUsesAttributesAndKeepsOtherHiddenKeys() {
+ StackResource r = resource("subnet-1");
+ r.getAttributes().put("VpcId", "vpc-new");
+ r.getAttributes().put(CfnRollback.USES_ATTR_PREFIX + "PublicIp", "203.0.113.2");
+ r.getAttributes().put(CfnRollback.ROLLBACK_OWNED_ATTR, "true");
+ Map<String, String> before = Map.of("VpcId", "vpc-old",
+ CfnRollback.USES_ATTR_PREFIX + "PublicIp", "203.0.113.1", CfnRollback.ROLLBACK_OWNED_ATTR, "false");
+ r.setPhysicalId("subnet-2");
+ ReplacementCleanup.record(r, update("subnet-1"), before);
+
+ assertTrue(ReplacementCleanup.rollback(r, (type, id, region) -> { }));
+
+ assertEquals("subnet-1", r.getPhysicalId());
+ assertEquals("vpc-old", r.getAttributes().get("VpcId"));
+ assertEquals("203.0.113.1", r.getAttributes().get(CfnRollback.USES_ATTR_PREFIX + "PublicIp"));
+ assertEquals("true", r.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));
+ }
+
/** A new provision supersedes the failed rollback; the replacement stays owed to the next cleanup instead. */
@Test
void record_afterAFailedRollbackDelete_doesNotRetryThatDeleteOnTheNextRollback() {