Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudformation): physically delete nested stacks and removed resources on update (#2345)

z9rtm291a97473eee
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
index dba8a3e914..b8d970965b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
@@ -33,6 +33,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -727,19 +728,19 @@ public class CloudFormationService {
if (!isCreate) {
updateCommitted = true;
- if (hasReplacementUpdates(stack)) {
+ if (hasReplacementUpdates(stack) || hasRemovedOrConditionFalseResources(stack, resources, conditions)) {
stack.setStatus("UPDATE_COMPLETE_CLEANUP_IN_PROGRESS");
stack.setLastUpdatedTime(now());
addEvent(stack, stack.getStackName(), stack.getStackId(),
"AWS::CloudFormation::Stack",
"UPDATE_COMPLETE_CLEANUP_IN_PROGRESS", null);
// The committed template and new physical IDs must be durable before old
- // resources are deleted during replacement cleanup.
+ // resources are deleted during post-update cleanup.
persistStack(stack);
}
// Both cleanup paths feed the single final status/reason writer.
List<UpdateCleanupFailure> cleanupFailures =
- new ArrayList<>(deleteInactiveConditionResources(
+ new ArrayList<>(deleteRemovedOrConditionFalseResources(
stack, resources, conditions, region));
cleanupFailures.addAll(finishCommittedResourceCleanup(stack));
finishCommittedStackUpdate(stack, cleanupFailures);
@@ -912,6 +913,52 @@ public class CloudFormationService {
.anyMatch(provisioner::hasReplacementUpdate);
}
+ private boolean hasRemovedOrConditionFalseResources(Stack stack, JsonNode resources, Map<String, Boolean> conditions) {
+ if (!resources.isObject()) {
+ return false;
+ }
+ for (StackResource resource : stack.getResources().values()) {
+ JsonNode resDef = resources.get(resource.getLogicalId());
+ if (resDef == null) {
+ return true;
+ }
+ String condition = resDef.path("Condition").asText(null);
+ if (condition != null && !conditions.getOrDefault(condition, false)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void deleteResourcePhysically(StackResource resource, String region) throws Exception {
+ if ("AWS::CloudFormation::Stack".equals(resource.getResourceType())) {
+ Future<?> future = deleteStack(resource.getPhysicalId(), region, regionResolver.getAccountId());
+ if (future != null) {
+ try {
+ future.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw e;
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof Exception ex) {
+ throw ex;
+ }
+ throw e;
+ }
+ }
+ Stack child = resolveStack(resource.getPhysicalId(), region);
+ if (child != null && "DELETE_FAILED".equals(child.getStatus())) {
+ String reason = child.getStatusReason() != null
+ ? child.getStatusReason()
+ : "Nested stack deletion failed";
+ throw new IllegalStateException(reason);
+ }
+ } else {
+ provisioner.delete(resource, region);
+ }
+ }
+
private void rollbackFailedUpdate(
Stack stack,
String region,
@@ -977,7 +1024,7 @@ public class CloudFormationService {
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_IN_PROGRESS",
"Resource creation cancelled during update rollback");
- provisioner.delete(resource, region);
+ deleteResourcePhysically(resource, region);
}
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_COMPLETE",
@@ -1120,7 +1167,7 @@ public class CloudFormationService {
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_IN_PROGRESS", null);
try {
- provisioner.delete(resource, region);
+ deleteResourcePhysically(resource, region);
resource.setStatus("DELETE_COMPLETE");
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_COMPLETE", null);
@@ -1140,15 +1187,15 @@ public class CloudFormationService {
/**
* Removes resources that were provisioned by an earlier execution but whose resource-level
- * {@code Condition} is false in the current template. Physical deletion honors the resource's
- * deletion policy. A failed physical deletion leaves the resource in the underlying service and
- * keeps it under stack management as {@code DELETE_FAILED}, so a later {@code DeleteStack} or
- * {@code UpdateStack} can retry the cleanup instead of orphaning the backing resource.
- * Resources removed from the template entirely are outside this condition-specific cleanup.
+ * {@code Condition} is false in the current template, or that were removed from the template entirely.
+ * Physical deletion honors the resource's retention policy ({@code Retain}, {@code RetainExceptOnCreate}).
+ * A failed physical deletion leaves the resource in the underlying service and keeps it under stack management
+ * as {@code DELETE_FAILED}, so a later {@code DeleteStack} or {@code UpdateStack} can retry the cleanup
+ * instead of orphaning the backing resource.
*
* @return one {@link UpdateCleanupFailure} per resource whose physical deletion failed
*/
- private List<UpdateCleanupFailure> deleteInactiveConditionResources(Stack stack, JsonNode resources,
+ private List<UpdateCleanupFailure> deleteRemovedOrConditionFalseResources(Stack stack, JsonNode resources,
Map<String, Boolean> conditions, String region) {
if (!resources.isObject()) {
return List.of();
@@ -1159,12 +1206,11 @@ public class CloudFormationService {
Collections.reverse(ordered);
for (StackResource resource : ordered) {
JsonNode resDef = resources.get(resource.getLogicalId());
- if (resDef == null) {
- continue;
- }
- String condition = resDef.path("Condition").asText(null);
- if (condition == null || conditions.getOrDefault(condition, false)) {
- continue;
+ if (resDef != null) {
+ String condition = resDef.path("Condition").asText(null);
+ if (condition == null || conditions.getOrDefault(condition, false)) {
+ continue;
+ }
}
if (resource.getPhysicalId() == null || skipRetainedResource(stack, resource, false)) {
@@ -1175,7 +1221,7 @@ public class CloudFormationService {
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_IN_PROGRESS", null);
try {
- provisioner.delete(resource, region);
+ deleteResourcePhysically(resource, region);
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_COMPLETE", null);
stack.getResources().remove(resource.getLogicalId());
@@ -1192,7 +1238,7 @@ public class CloudFormationService {
resource.setStatusReason(reason);
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_FAILED", reason);
- LOG.warnv("Failed to delete condition-disabled {0} ({1}) in stack {2}: {3}",
+ LOG.warnv("Failed to delete removed or condition-disabled {0} ({1}) in stack {2}: {3}",
resource.getResourceType(), resource.getPhysicalId(),
stack.getStackName(), reason);
}
@@ -1223,7 +1269,7 @@ public class CloudFormationService {
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_IN_PROGRESS", null);
try {
- provisioner.delete(resource, region);
+ deleteResourcePhysically(resource, region);
resource.setStatus("DELETE_COMPLETE");
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_COMPLETE", null);
@@ -1250,8 +1296,9 @@ public class CloudFormationService {
stack.setStatusReason(reason);
addEvent(stack, stack.getStackName(), stack.getStackId(),
"AWS::CloudFormation::Stack", "DELETE_FAILED", reason);
+ persistStack(stack);
LOG.errorv("Stack {0} delete failed: {1}", stack.getStackName(), reason);
- return;
+ throw new IllegalStateException(reason);
}
stack.setStatus("DELETE_COMPLETE");
@@ -1269,6 +1316,8 @@ public class CloudFormationService {
LOG.errorv("Stack {0} delete failed: {1}", stack.getStackName(), e.getMessage());
stack.setStatus("DELETE_FAILED");
stack.setStatusReason(e.getMessage());
+ persistStack(stack);
+ throw (e instanceof RuntimeException re ? re : new RuntimeException(e));
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java
index 25a619afb0..017369a886 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java
@@ -7,6 +7,7 @@ import org.junit.jupiter.api.Test;
import static io.restassured.RestAssured.given;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
import static org.junit.jupiter.api.Assertions.fail;
/**
@@ -20,6 +21,9 @@ import static org.junit.jupiter.api.Assertions.fail;
@QuarkusTest
class CloudFormationDeletionPolicyIntegrationTest {
+ private static final String CUSTOM_AUTH =
+ "AWS4-HMAC-SHA256 Credential=111122223333/20260205/eu-west-1/cloudformation/aws4_request";
+
@Test
void retainKeepsANonEmptyBucketAndTheStackStillCompletesTheDelete() throws InterruptedException {
String suffix = Long.toString(System.nanoTime(), 36);
@@ -171,9 +175,704 @@ class CloudFormationDeletionPolicyIntegrationTest {
assertBucketDeleted(unrecognized);
}
+ @Test
+ void retainKeepsResourceWhenRemovedFromTemplateOnUpdate() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucket1 = "cfn-update-remove-bucket1-" + suffix;
+ String bucket2 = "cfn-update-remove-bucket2-" + suffix;
+ String stackName = "cfn-update-remove-stack-" + suffix;
+
+ String template1 = """
+ {
+ "Resources": {
+ "KeptBucket": {
+ "Type": "AWS::S3::Bucket",
+ "DeletionPolicy": "Retain",
+ "Properties": { "BucketName": "%s" }
+ },
+ "DeletedBucket": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ }
+ }
+ }
+ """.formatted(bucket1, bucket2);
+
+ String stackId = createStack(stackName, template1);
+ awaitStackStatus(stackId, "CREATE_COMPLETE");
+
+ String template2 = """
+ {
+ "Resources": {
+ }
+ }
+ """;
+
+ updateStack(stackName, template2);
+ awaitStackStatus(stackId, "UPDATE_COMPLETE");
+
+ assertBucketExists(bucket1);
+ assertBucketDeleted(bucket2);
+
+ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();
+ assertThat(resXml, not(containsString("<LogicalResourceId>KeptBucket</LogicalResourceId>")));
+ assertThat(resXml, not(containsString("<LogicalResourceId>DeletedBucket</LogicalResourceId>")));
+
+ String events = describeStackEvents(stackId);
+ assertThat(events, containsString("<ResourceStatus>DELETE_SKIPPED</ResourceStatus>"));
+ }
+
+ @Test
+ void retainExceptOnCreateKeepsResourceWhenRemovedFromTemplateOnUpdate() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucket1 = "cfn-reoc-update-bucket1-" + suffix;
+ String bucket2 = "cfn-reoc-update-bucket2-" + suffix;
+ String stackName = "cfn-reoc-update-stack-" + suffix;
+
+ String template1 = """
+ {
+ "Resources": {
+ "KeptBucket": {
+ "Type": "AWS::S3::Bucket",
+ "DeletionPolicy": "RetainExceptOnCreate",
+ "Properties": { "BucketName": "%s" }
+ },
+ "DeletedBucket": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ }
+ }
+ }
+ """.formatted(bucket1, bucket2);
+
+ String stackId = createStack(stackName, template1);
+ try {
+ awaitStackStatus(stackId, "CREATE_COMPLETE");
+
+ String template2 = """
+ {
+ "Resources": {
+ }
+ }
+ """;
+
+ updateStack(stackName, template2);
+ awaitStackStatus(stackId, "UPDATE_COMPLETE");
+
+ assertBucketExists(bucket1);
+ assertBucketDeleted(bucket2);
+
+ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();
+ assertThat(resXml, not(containsString("<LogicalResourceId>KeptBucket</LogicalResourceId>")));
+ assertThat(resXml, not(containsString("<LogicalResourceId>DeletedBucket</LogicalResourceId>")));
+
+ String events = describeStackEvents(stackId);
+ assertThat(events, containsString("<ResourceStatus>DELETE_SKIPPED</ResourceStatus>"));
+ } finally {
+ deleteStack(stackName);
+ given().header("Host", bucket1 + ".localhost").when().delete("/");
+ }
+ }
+
+ @Test
+ void nestedStackIsPhysicallyDeletedWhenRemovedFromTemplateOnUpdate() throws InterruptedException {
+ given().header("Authorization", CUSTOM_AUTH).when().put("/nested-stack-templates").then().statusCode(200);
+ String childTemplate = "{\"Resources\": {\"Queue\": {\"Type\": \"AWS::SQS::Queue\"}}}";
+ given()
+ .header("Authorization", CUSTOM_AUTH)
+ .contentType("application/json")
+ .body(childTemplate)
+ .when()
+ .put("/nested-stack-templates/child-update.json")
+ .then()
+ .statusCode(200);
+
+ String stackName = "parent-stack-" + System.nanoTime();
+ String template1 = """
+ {
+ "Resources": {
+ "ChildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-update.json" }
+ }
+ }
+ }
+ """;
+
+ String parentStackId = createStack(stackName, template1, CUSTOM_AUTH);
+ try {
+ awaitStackStatus(parentStackId, "CREATE_COMPLETE", CUSTOM_AUTH);
+
+ String nestedStackId = getNestedStackId(parentStackId, "ChildStack", CUSTOM_AUTH);
+ assertThat(nestedStackId, containsString("111122223333"));
+ assertThat(nestedStackId, containsString("eu-west-1"));
+
+ String template2 = "{\"Resources\": {}}";
+ updateStack(stackName, template2, CUSTOM_AUTH);
+ awaitStackStatus(parentStackId, "UPDATE_COMPLETE", CUSTOM_AUTH);
+
+ awaitStackStatus(nestedStackId, "DELETE_COMPLETE", CUSTOM_AUTH);
+ } finally {
+ deleteStack(stackName);
+ }
+ }
+
+ @Test
+ void nestedStackIsRetainedWhenRemovedFromTemplateOnUpdateWithRetainPolicy() throws InterruptedException {
+ given().when().put("/nested-stack-templates").then().statusCode(200);
+ String childTemplate = "{\"Resources\": {\"Queue\": {\"Type\": \"AWS::SQS::Queue\"}}}";
+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-retain.json");
+
+ String stackName = "parent-retain-" + System.nanoTime();
+ String template1 = """
+ {
+ "Resources": {
+ "ChildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "DeletionPolicy": "Retain",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-retain.json" }
+ }
+ }
+ }
+ """;
+
+ String parentStackId = createStack(stackName, template1, null);
+ String nestedStackId = null;
+ try {
+ awaitStackStatus(parentStackId, "CREATE_COMPLETE", null);
+ nestedStackId = getNestedStackId(parentStackId, "ChildStack", null);
+
+ String template2 = "{\"Resources\": {}}";
+ updateStack(stackName, template2, null);
+ awaitStackStatus(parentStackId, "UPDATE_COMPLETE", null);
+
+ awaitStackStatus(nestedStackId, "CREATE_COMPLETE", null);
+
+ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();
+ assertThat(resXml, not(containsString("<LogicalResourceId>ChildStack</LogicalResourceId>")));
+ } finally {
+ deleteStack(stackName);
+ if (nestedStackId != null) {
+ deleteStack(nestedStackId);
+ }
+ }
+ }
+
+ @Test
+ void nestedStackWithNonEmptyBucketFailsDeletionOnUpdate_leavesChildAsDeleteFailedAndTracksInParent() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucketName = "cfn-nested-orphan-" + suffix;
+ given().when().put("/nested-stack-templates").then().statusCode(200);
+ String childTemplate = """
+ {
+ "Resources": {
+ "ChildBucket": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ }
+ }
+ }
+ """.formatted(bucketName);
+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-orphan-" + suffix + ".json");
+
+ String stackName = "parent-orphan-" + suffix;
+ String template1 = """
+ {
+ "Resources": {
+ "ChildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-orphan-%s.json" }
+ }
+ }
+ }
+ """.formatted(suffix);
+
+ String parentStackId = createStack(stackName, template1);
+ try {
+ awaitStackStatus(parentStackId, "CREATE_COMPLETE");
+ String nestedStackId = getNestedStackId(parentStackId, "ChildStack", null);
+
+ // Put an object in the nested bucket so nested stack deletion will fail
+ given().contentType("text/plain").body("keep").when().put("/" + bucketName + "/object.txt").then().statusCode(200);
+
+ String template2 = "{\"Resources\": {}}";
+ updateStack(stackName, template2);
+ awaitStackStatus(parentStackId, "UPDATE_COMPLETE");
+ assertThat(describeStacks(parentStackId), containsString("could not be deleted during update cleanup"));
+
+ // Nested stack resource stays in parent as DELETE_FAILED
+ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();
+ assertThat(resXml, containsString("<LogicalResourceId>ChildStack</LogicalResourceId>"));
+ assertThat(resXml, containsString("<ResourceStatus>DELETE_FAILED</ResourceStatus>"));
+
+ // Child stack itself is DELETE_FAILED
+ awaitStackStatus(nestedStackId, "DELETE_FAILED");
+
+ // Clean up bucket and delete parent stack to verify clean recovery
+ given().when().delete("/" + bucketName + "/object.txt").then().statusCode(204);
+ deleteStack(stackName);
+ awaitStackStatus(parentStackId, "DELETE_COMPLETE");
+ awaitStackStatus(nestedStackId, "DELETE_COMPLETE");
+ assertBucketDeleted(bucketName);
+ } finally {
+ given().when().delete("/" + bucketName + "/object.txt");
+ deleteStack(stackName);
+ }
+ }
+
+ @Test
+ void deleteStack_withNestedStack_deletesChildStackAndPhysicalResources() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucketName = "cfn-nested-del-" + suffix;
+ given().when().put("/nested-stack-templates").then().statusCode(200);
+ String childTemplate = """
+ {
+ "Resources": {
+ "ChildBucket": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ }
+ }
+ }
+ """.formatted(bucketName);
+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-del-" + suffix + ".json");
+
+ String stackName = "parent-del-" + suffix;
+ String template = """
+ {
+ "Resources": {
+ "ChildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-del-%s.json" }
+ }
+ }
+ }
+ """.formatted(suffix);
+
+ String parentStackId = createStack(stackName, template);
+ try {
+ awaitStackStatus(parentStackId, "CREATE_COMPLETE");
+ String nestedStackId = getNestedStackId(parentStackId, "ChildStack", null);
+ assertBucketExists(bucketName);
+
+ deleteStack(stackName);
+ awaitStackStatus(parentStackId, "DELETE_COMPLETE");
+ awaitStackStatus(nestedStackId, "DELETE_COMPLETE");
+ assertBucketDeleted(bucketName);
+ } finally {
+ deleteStack(stackName);
+ }
+ }
+
+ @Test
+ void deleteStack_withNestedStackContainingNonEmptyBucket_failsAndLeavesBothStacksInDeleteFailed() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucketName = "cfn-nested-delfail-" + suffix;
+ given().when().put("/nested-stack-templates").then().statusCode(200);
+ String childTemplate = """
+ {
+ "Resources": {
+ "ChildBucket": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ }
+ }
+ }
+ """.formatted(bucketName);
+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-delfail-" + suffix + ".json");
+
+ String stackName = "parent-delfail-" + suffix;
+ String template = """
+ {
+ "Resources": {
+ "ChildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-delfail-%s.json" }
+ }
+ }
+ }
+ """.formatted(suffix);
+
+ String parentStackId = createStack(stackName, template);
+ try {
+ awaitStackStatus(parentStackId, "CREATE_COMPLETE");
+ String nestedStackId = getNestedStackId(parentStackId, "ChildStack", null);
+
+ // Put object in child bucket
+ given().contentType("text/plain").body("blocker").when().put("/" + bucketName + "/object.txt").then().statusCode(200);
+
+ deleteStack(stackName);
+ awaitStackStatus(parentStackId, "DELETE_FAILED");
+ awaitStackStatus(nestedStackId, "DELETE_FAILED");
+ assertBucketExists(bucketName);
+
+ // Clean up object and retry DeleteStack
+ given().when().delete("/" + bucketName + "/object.txt").then().statusCode(204);
+ deleteStack(stackName);
+ awaitStackStatus(parentStackId, "DELETE_COMPLETE");
+ awaitStackStatus(nestedStackId, "DELETE_COMPLETE");
+ assertBucketDeleted(bucketName);
+ } finally {
+ given().when().delete("/" + bucketName + "/object.txt");
+ deleteStack(stackName);
+ }
+ }
+
+ @Test
+ void createStack_rollbackWithNestedStack_deletesChildStackAndResources() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucketName = "cfn-nested-createrb-" + suffix;
+ given().when().put("/nested-stack-templates").then().statusCode(200);
+ String childTemplate = """
+ {
+ "Resources": {
+ "ChildBucket": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ }
+ }
+ }
+ """.formatted(bucketName);
+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-createrb-" + suffix + ".json");
+
+ String stackName = "parent-createrb-" + suffix;
+ String template = """
+ {
+ "Resources": {
+ "ChildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-createrb-%s.json" }
+ },
+ "BadSecret": {
+ "Type": "AWS::SecretsManager::Secret",
+ "DependsOn": "ChildStack",
+ "Properties": {
+ "Name": "bad-secret-%s",
+ "SecretString": "explicit",
+ "GenerateSecretString": { "PasswordLength": 32 }
+ }
+ }
+ }
+ }
+ """.formatted(suffix, suffix);
+
+ String parentStackId = createStack(stackName, template);
+ try {
+ assertThat(describeStacks(parentStackId), containsString("<StackStatus>ROLLBACK_COMPLETE</StackStatus>"));
+ assertBucketDeleted(bucketName);
+ } finally {
+ deleteStack(stackName);
+ }
+ }
+
+ @Test
+ void updateStack_rollbackWithNestedStack_deletesNewlyAddedChildStack() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucketName = "cfn-nested-updaterb-" + suffix;
+ given().when().put("/nested-stack-templates").then().statusCode(200);
+ String childTemplate = """
+ {
+ "Resources": {
+ "ChildBucket": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ }
+ }
+ }
+ """.formatted(bucketName);
+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-updaterb-" + suffix + ".json");
+
+ String stackName = "parent-updaterb-" + suffix;
+ String sfnName = "initial-sfn-" + suffix;
+ String initialTemplate = """
+ {
+ "Resources": {
+ "InitialStateMachine": {
+ "Type": "AWS::StepFunctions::StateMachine",
+ "Properties": {
+ "StateMachineName": "%s",
+ "RoleArn": "arn:aws:iam::000000000000:role/cfn-sfn-rollback-role",
+ "DefinitionString": "{\\"StartAt\\":\\"Done\\",\\"States\\":{\\"Done\\":{\\"Type\\":\\"Pass\\",\\"Result\\":\\"marker-v1\\",\\"End\\":true}}}"
+ }
+ }
+ }
+ }
+ """.formatted(sfnName);
+
+ String parentStackId = createStack(stackName, initialTemplate);
+ try {
+ awaitStackStatus(parentStackId, "CREATE_COMPLETE");
+
+ String failingUpdateTemplate = """
+ {
+ "Resources": {
+ "InitialStateMachine": {
+ "Type": "AWS::StepFunctions::StateMachine",
+ "Properties": {
+ "StateMachineName": "%s",
+ "RoleArn": "arn:aws:iam::000000000000:role/cfn-sfn-rollback-role",
+ "DefinitionString": "{\\"StartAt\\":\\"Done\\",\\"States\\":{\\"Done\\":{\\"Type\\":\\"Pass\\",\\"Result\\":\\"marker-v2\\",\\"End\\":true}}}"
+ }
+ },
+ "ChildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-updaterb-%s.json" }
+ },
+ "BadSecret": {
+ "Type": "AWS::SecretsManager::Secret",
+ "DependsOn": "ChildStack",
+ "Properties": {
+ "Name": "bad-secret-update-%s",
+ "SecretString": "explicit",
+ "GenerateSecretString": { "PasswordLength": 32 }
+ }
+ }
+ }
+ }
+ """.formatted(sfnName, suffix, suffix);
+
+ updateStack(stackName, failingUpdateTemplate);
+ awaitStackStatus(parentStackId, "UPDATE_ROLLBACK_COMPLETE");
+ assertBucketDeleted(bucketName);
+
+ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();
+ assertThat(resXml, containsString("<LogicalResourceId>InitialStateMachine</LogicalResourceId>"));
+ assertThat(resXml, not(containsString("<LogicalResourceId>ChildStack</LogicalResourceId>")));
+ assertThat(resXml, not(containsString("<LogicalResourceId>BadSecret</LogicalResourceId>")));
+
+ deleteStack(stackName);
+ awaitStackStatus(parentStackId, "DELETE_COMPLETE");
+ } finally {
+ deleteStack(stackName);
+ }
+ }
+
+ @Test
+ void nestedStackWithMultiLevelHierarchy_deletesRecursivelyOnUpdate() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucketName = "cfn-nested3-del-" + suffix;
+ given().when().put("/nested-stack-templates").then().statusCode(200);
+
+ String grandchildTemplate = """
+ {
+ "Resources": {
+ "GrandchildBucket": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ }
+ }
+ }
+ """.formatted(bucketName);
+ given().contentType("application/json").body(grandchildTemplate).when().put("/nested-stack-templates/grandchild-" + suffix + ".json");
+
+ String childTemplate = """
+ {
+ "Resources": {
+ "GrandchildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/grandchild-%s.json" }
+ }
+ }
+ }
+ """.formatted(suffix);
+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child3-" + suffix + ".json");
+
+ String rootStackName = "root3-del-" + suffix;
+ String rootTemplate = """
+ {
+ "Resources": {
+ "ChildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child3-%s.json" }
+ }
+ }
+ }
+ """.formatted(suffix);
+
+ String rootStackId = createStack(rootStackName, rootTemplate);
+ try {
+ awaitStackStatus(rootStackId, "CREATE_COMPLETE");
+ String childStackId = getNestedStackId(rootStackId, "ChildStack", null);
+ String grandchildStackId = getNestedStackId(childStackId, "GrandchildStack", null);
+ assertBucketExists(bucketName);
+
+ String updatedRootTemplate = "{\"Resources\": {}}";
+ updateStack(rootStackName, updatedRootTemplate);
+ awaitStackStatus(rootStackId, "UPDATE_COMPLETE");
+
+ awaitStackStatus(childStackId, "DELETE_COMPLETE");
+ awaitStackStatus(grandchildStackId, "DELETE_COMPLETE");
+ assertBucketDeleted(bucketName);
+ } finally {
+ deleteStack(rootStackName);
+ }
+ }
+
+ @Test
+ void nestedStackWithMultiLevelHierarchy_propagatesDeleteFailedOnUpdate() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucketName = "cfn-nested3-fail-" + suffix;
+ given().when().put("/nested-stack-templates").then().statusCode(200);
+
+ String grandchildTemplate = """
+ {
+ "Resources": {
+ "GrandchildBucket": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ }
+ }
+ }
+ """.formatted(bucketName);
+ given().contentType("application/json").body(grandchildTemplate).when().put("/nested-stack-templates/grandchild-fail-" + suffix + ".json");
+
+ String childTemplate = """
+ {
+ "Resources": {
+ "GrandchildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/grandchild-fail-%s.json" }
+ }
+ }
+ }
+ """.formatted(suffix);
+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child3-fail-" + suffix + ".json");
+
+ String rootStackName = "root3-fail-" + suffix;
+ String rootTemplate = """
+ {
+ "Resources": {
+ "ChildStack": {
+ "Type": "AWS::CloudFormation::Stack",
+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child3-fail-%s.json" }
+ }
+ }
+ }
+ """.formatted(suffix);
+
+ String rootStackId = createStack(rootStackName, rootTemplate);
+ try {
+ awaitStackStatus(rootStackId, "CREATE_COMPLETE");
+ String childStackId = getNestedStackId(rootStackId, "ChildStack", null);
+ String grandchildStackId = getNestedStackId(childStackId, "GrandchildStack", null);
+
+ // Put blocker object into grandchild's bucket
+ given().contentType("text/plain").body("keep").when().put("/" + bucketName + "/blocker.txt").then().statusCode(200);
+
+ String updatedRootTemplate = "{\"Resources\": {}}";
+ updateStack(rootStackName, updatedRootTemplate);
+ awaitStackStatus(rootStackId, "UPDATE_COMPLETE");
+ assertThat(describeStacks(rootStackId), containsString("could not be deleted during update cleanup"));
+
+ awaitStackStatus(grandchildStackId, "DELETE_FAILED");
+ awaitStackStatus(childStackId, "DELETE_FAILED");
+
+ // Clean up blocker and delete root stack
+ given().when().delete("/" + bucketName + "/blocker.txt").then().statusCode(204);
+ deleteStack(rootStackName);
+ awaitStackStatus(rootStackId, "DELETE_COMPLETE");
+ awaitStackStatus(childStackId, "DELETE_COMPLETE");
+ awaitStackStatus(grandchildStackId, "DELETE_COMPLETE");
+ assertBucketDeleted(bucketName);
+ } finally {
+ given().when().delete("/" + bucketName + "/blocker.txt");
+ deleteStack(rootStackName);
+ }
+ }
+
+ @Test
+ void updateStack_rollbackPreservesRemovedResourcesWhenNewResourceFails() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucketA = "cfn-rb-keep-a-" + suffix;
+ String sfnName = "cfn-rb-sfn-" + suffix;
+ String stackName = "cfn-rb-preserve-" + suffix;
+
+ String initialTemplate = """
+ {
+ "Resources": {
+ "BucketA": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ },
+ "InitialStateMachine": {
+ "Type": "AWS::StepFunctions::StateMachine",
+ "Properties": {
+ "StateMachineName": "%s",
+ "RoleArn": "arn:aws:iam::000000000000:role/cfn-sfn-rollback-role",
+ "DefinitionString": "{\\"StartAt\\":\\"Done\\",\\"States\\":{\\"Done\\":{\\"Type\\":\\"Pass\\",\\"Result\\":\\"marker-v1\\",\\"End\\":true}}}"
+ }
+ }
+ }
+ }
+ """.formatted(bucketA, sfnName);
+
+ String stackId = createStack(stackName, initialTemplate);
+ try {
+ awaitStackStatus(stackId, "CREATE_COMPLETE");
+ assertBucketExists(bucketA);
+
+ // Update template: omits BucketA, updates InitialStateMachine, and adds a resource that fails creation
+ String failingUpdateTemplate = """
+ {
+ "Resources": {
+ "InitialStateMachine": {
+ "Type": "AWS::StepFunctions::StateMachine",
+ "Properties": {
+ "StateMachineName": "%s",
+ "RoleArn": "arn:aws:iam::000000000000:role/cfn-sfn-rollback-role",
+ "DefinitionString": "{\\"StartAt\\":\\"Done\\",\\"States\\":{\\"Done\\":{\\"Type\\":\\"Pass\\",\\"Result\\":\\"marker-v2\\",\\"End\\":true}}}"
+ }
+ },
+ "BadSecret": {
+ "Type": "AWS::SecretsManager::Secret",
+ "Properties": {
+ "Name": "bad-secret-%s",
+ "SecretString": "explicit",
+ "GenerateSecretString": { "PasswordLength": 32 }
+ }
+ }
+ }
+ }
+ """.formatted(sfnName, suffix);
+
+ updateStack(stackName, failingUpdateTemplate);
+ awaitStackStatus(stackId, "UPDATE_ROLLBACK_COMPLETE");
+
+ // BucketA was removed in the failing update template, but rollback must preserve it!
+ assertBucketExists(bucketA);
+
+ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();
+ assertThat(resXml, containsString("<LogicalResourceId>BucketA</LogicalResourceId>"));
+ assertThat(resXml, containsString("<LogicalResourceId>InitialStateMachine</LogicalResourceId>"));
+ assertThat(resXml, not(containsString("<LogicalResourceId>BadSecret</LogicalResourceId>")));
+ } finally {
+ deleteStack(stackName);
+ awaitStackStatus(stackId, "DELETE_COMPLETE");
+ assertBucketDeleted(bucketA);
+ }
+ }
+
+ private static String getNestedStackId(String stackName, String logicalId, String auth) {
+ String xml = cfnQuery("DescribeStackResources", stackName, auth).then().statusCode(200).extract().asString();
+ int logIdx = xml.indexOf("<LogicalResourceId>" + logicalId + "</LogicalResourceId>");
+ if (logIdx == -1) {
+ fail("Logical resource " + logicalId + " not found");
+ }
+ int physStart = xml.indexOf("<PhysicalResourceId>", logIdx) + "<PhysicalResourceId>".length();
+ int physEnd = xml.indexOf("</PhysicalResourceId>", physStart);
+ return xml.substring(physStart, physEnd);
+ }
+
private static String createStack(String stackName, String template) {
- String xml = given()
- .contentType("application/x-www-form-urlencoded")
+ return createStack(stackName, template, null);
+ }
+
+ private static String createStack(String stackName, String template, String auth) {
+ var req = given().contentType("application/x-www-form-urlencoded");
+ if (auth != null) {
+ req.header("Authorization", auth);
+ }
+ String xml = req
.formParam("Action", "CreateStack")
.formParam("StackName", stackName)
.formParam("TemplateBody", template)
@@ -198,12 +897,34 @@ class CloudFormationDeletionPolicyIntegrationTest {
.statusCode(200);
}
+ private static void updateStack(String stackName, String template) {
+ updateStack(stackName, template, null);
+ }
+
+ private static void updateStack(String stackName, String template, String auth) {
+ var req = given().contentType("application/x-www-form-urlencoded");
+ if (auth != null) {
+ req.header("Authorization", auth);
+ }
+ req.formParam("Action", "UpdateStack")
+ .formParam("StackName", stackName)
+ .formParam("TemplateBody", template)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
/** DeleteStack runs asynchronously, and a deleted stack stays describable by its stack ID. */
private static void awaitStackStatus(String stackId, String status) throws InterruptedException {
+ awaitStackStatus(stackId, status, null);
+ }
+
+ private static void awaitStackStatus(String stackId, String status, String auth) throws InterruptedException {
String xml = "";
long deadline = System.currentTimeMillis() + 10_000;
while (System.currentTimeMillis() < deadline) {
- xml = describeStacks(stackId);
+ xml = describeStacks(stackId, auth);
if (xml.contains("<StackStatus>" + status + "</StackStatus>")) {
return;
}
@@ -213,20 +934,27 @@ class CloudFormationDeletionPolicyIntegrationTest {
}
private static String describeStacks(String stackId) {
- return cfnQuery("DescribeStacks", stackId).then().statusCode(200).extract().asString();
+ return describeStacks(stackId, null);
+ }
+
+ private static String describeStacks(String stackId, String auth) {
+ return cfnQuery("DescribeStacks", stackId, auth).then().statusCode(200).extract().asString();
}
private static String describeStackEvents(String stackId) {
- return cfnQuery("DescribeStackEvents", stackId).then().statusCode(200).extract().asString();
+ return cfnQuery("DescribeStackEvents", stackId, null).then().statusCode(200).extract().asString();
}
private static Response cfnQuery(String action, String stackId) {
- return given()
- .contentType("application/x-www-form-urlencoded")
- .formParam("Action", action)
- .formParam("StackName", stackId)
- .when()
- .post("/");
+ return cfnQuery(action, stackId, null);
+ }
+
+ private static Response cfnQuery(String action, String stackId, String auth) {
+ var req = given().contentType("application/x-www-form-urlencoded");
+ if (auth != null) {
+ req.header("Authorization", auth);
+ }
+ return req.formParam("Action", action).formParam("StackName", stackId).when().post("/");
}
private static void assertBucketExists(String bucket) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceConditionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceConditionIntegrationTest.java
index d54ffe54ac..182034caae 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceConditionIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceConditionIntegrationTest.java
@@ -927,6 +927,141 @@ class CloudFormationResourceConditionIntegrationTest {
}
}
+ @Test
+ void updateStack_deletesResourceWhenRemovedFromTemplate() {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String stackName = "cfn-rem-update-" + suffix;
+ String queueName = "removed-update-" + suffix;
+
+ String initialTemplate = """
+ {
+ "Resources": {
+ "OptionalQueue": {
+ "Type": "AWS::SQS::Queue",
+ "Properties": { "QueueName": "%s" }
+ }
+ }
+ }
+ """.formatted(queueName);
+ String updatedTemplate = """
+ {
+ "Resources": {
+ "DummyQueue": {
+ "Type": "AWS::SQS::Queue",
+ "Properties": { "QueueName": "dummy-%s" }
+ }
+ }
+ }
+ """.formatted(suffix);
+
+ createStack(stackName, initialTemplate);
+ try {
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "GetQueueUrl")
+ .formParam("QueueName", queueName)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString(queueName));
+
+ updateStack(stackName, updatedTemplate);
+
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "GetQueueUrl")
+ .formParam("QueueName", queueName)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("ErrorResponse.Error.Code", equalTo("AWS.SimpleQueueService.NonExistentQueue"));
+
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeStackResources")
+ .formParam("StackName", stackName)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(not(containsString("<LogicalResourceId>OptionalQueue</LogicalResourceId>")));
+ } finally {
+ deleteStack(stackName);
+ }
+ }
+
+ @Test
+ void updateStack_whenResourceRemovedFromTemplateWithNonEmptyBucket_keepsBucketAsDeleteFailedAndReclaimsOnDelete() {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String stackName = "cfn-rem-orphan-" + suffix;
+ String bucketName = "removed-orphan-" + suffix;
+
+ String initialTemplate = """
+ {
+ "Resources": {
+ "OptionalBucket": {
+ "Type": "AWS::S3::Bucket",
+ "Properties": { "BucketName": "%s" }
+ }
+ }
+ }
+ """.formatted(bucketName);
+ String updatedTemplate = """
+ {
+ "Resources": {
+ }
+ }
+ """;
+
+ createStack(stackName, initialTemplate);
+ try {
+ given().header("Host", bucketName + ".localhost").when().get("/").then().statusCode(200);
+ given()
+ .contentType("text/plain")
+ .body("prevent removed bucket deletion")
+ .when()
+ .put("/" + bucketName + "/object.txt")
+ .then()
+ .statusCode(200);
+
+ // Cleanup delete fails (bucket non-empty), but update completes; bucket stays under stack management as DELETE_FAILED.
+ updateStack(stackName, updatedTemplate);
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeStacks")
+ .formParam("StackName", stackName)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("<StackStatus>UPDATE_COMPLETE</StackStatus>"))
+ .body(containsString("could not be deleted during update cleanup"));
+
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeStackResources")
+ .formParam("StackName", stackName)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("<LogicalResourceId>OptionalBucket</LogicalResourceId>"))
+ .body(containsString("<ResourceStatus>DELETE_FAILED</ResourceStatus>"));
+
+ // Clear the blocker, then DeleteStack retries the DELETE_FAILED resource
+ given().header("Host", bucketName + ".localhost").delete("/object.txt").then().statusCode(204);
+ deleteStack(stackName);
+ awaitStackGone(stackName);
+ given().header("Host", bucketName + ".localhost").when().get("/").then().statusCode(404);
+ } finally {
+ given().header("Host", bucketName + ".localhost").delete("/object.txt");
+ given().header("Host", bucketName + ".localhost").delete("/");
+ deleteStack(stackName);
+ }
+ }
+
private static void createStack(String stackName, String template) {
given()
.contentType("application/x-www-form-urlencoded")