Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): physically delete nested stacks and removed resources on update (#2345)
z9rtm291a97473eee
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.javaindex dba8a3e914..b8d970965b 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java@@ -33,6 +33,7 @@ import java.util.regex.Matcher;import java.util.regex.Pattern;import java.util.concurrent.CompletableFuture;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ExecutionException;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;@@ -727,19 +728,19 @@ public class CloudFormationService {if (!isCreate) {updateCommitted = true;- if (hasReplacementUpdates(stack)) {+ if (hasReplacementUpdates(stack) || hasRemovedOrConditionFalseResources(stack, resources, conditions)) {stack.setStatus("UPDATE_COMPLETE_CLEANUP_IN_PROGRESS");stack.setLastUpdatedTime(now());addEvent(stack, stack.getStackName(), stack.getStackId(),"AWS::CloudFormation::Stack","UPDATE_COMPLETE_CLEANUP_IN_PROGRESS", null);// The committed template and new physical IDs must be durable before old- // resources are deleted during replacement cleanup.+ // resources are deleted during post-update cleanup.persistStack(stack);}// Both cleanup paths feed the single final status/reason writer.List<UpdateCleanupFailure> cleanupFailures =- new ArrayList<>(deleteInactiveConditionResources(+ new ArrayList<>(deleteRemovedOrConditionFalseResources(stack, resources, conditions, region));cleanupFailures.addAll(finishCommittedResourceCleanup(stack));finishCommittedStackUpdate(stack, cleanupFailures);@@ -912,6 +913,52 @@ public class CloudFormationService {.anyMatch(provisioner::hasReplacementUpdate);}+ private boolean hasRemovedOrConditionFalseResources(Stack stack, JsonNode resources, Map<String, Boolean> conditions) {+ if (!resources.isObject()) {+ return false;+ }+ for (StackResource resource : stack.getResources().values()) {+ JsonNode resDef = resources.get(resource.getLogicalId());+ if (resDef == null) {+ return true;+ }+ String condition = resDef.path("Condition").asText(null);+ if (condition != null && !conditions.getOrDefault(condition, false)) {+ return true;+ }+ }+ return false;+ }++ private void deleteResourcePhysically(StackResource resource, String region) throws Exception {+ if ("AWS::CloudFormation::Stack".equals(resource.getResourceType())) {+ Future<?> future = deleteStack(resource.getPhysicalId(), region, regionResolver.getAccountId());+ if (future != null) {+ try {+ future.get();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw e;+ } catch (ExecutionException e) {+ Throwable cause = e.getCause();+ if (cause instanceof Exception ex) {+ throw ex;+ }+ throw e;+ }+ }+ Stack child = resolveStack(resource.getPhysicalId(), region);+ if (child != null && "DELETE_FAILED".equals(child.getStatus())) {+ String reason = child.getStatusReason() != null+ ? child.getStatusReason()+ : "Nested stack deletion failed";+ throw new IllegalStateException(reason);+ }+ } else {+ provisioner.delete(resource, region);+ }+ }+private void rollbackFailedUpdate(Stack stack,String region,@@ -977,7 +1024,7 @@ public class CloudFormationService {addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_IN_PROGRESS","Resource creation cancelled during update rollback");- provisioner.delete(resource, region);+ deleteResourcePhysically(resource, region);}addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_COMPLETE",@@ -1120,7 +1167,7 @@ public class CloudFormationService {addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_IN_PROGRESS", null);try {- provisioner.delete(resource, region);+ deleteResourcePhysically(resource, region);resource.setStatus("DELETE_COMPLETE");addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_COMPLETE", null);@@ -1140,15 +1187,15 @@ public class CloudFormationService {/*** Removes resources that were provisioned by an earlier execution but whose resource-level- * {@code Condition} is false in the current template. Physical deletion honors the resource's- * deletion policy. A failed physical deletion leaves the resource in the underlying service and- * keeps it under stack management as {@code DELETE_FAILED}, so a later {@code DeleteStack} or- * {@code UpdateStack} can retry the cleanup instead of orphaning the backing resource.- * Resources removed from the template entirely are outside this condition-specific cleanup.+ * {@code Condition} is false in the current template, or that were removed from the template entirely.+ * Physical deletion honors the resource's retention policy ({@code Retain}, {@code RetainExceptOnCreate}).+ * A failed physical deletion leaves the resource in the underlying service and keeps it under stack management+ * as {@code DELETE_FAILED}, so a later {@code DeleteStack} or {@code UpdateStack} can retry the cleanup+ * instead of orphaning the backing resource.** @return one {@link UpdateCleanupFailure} per resource whose physical deletion failed*/- private List<UpdateCleanupFailure> deleteInactiveConditionResources(Stack stack, JsonNode resources,+ private List<UpdateCleanupFailure> deleteRemovedOrConditionFalseResources(Stack stack, JsonNode resources,Map<String, Boolean> conditions, String region) {if (!resources.isObject()) {return List.of();@@ -1159,12 +1206,11 @@ public class CloudFormationService {Collections.reverse(ordered);for (StackResource resource : ordered) {JsonNode resDef = resources.get(resource.getLogicalId());- if (resDef == null) {- continue;- }- String condition = resDef.path("Condition").asText(null);- if (condition == null || conditions.getOrDefault(condition, false)) {- continue;+ if (resDef != null) {+ String condition = resDef.path("Condition").asText(null);+ if (condition == null || conditions.getOrDefault(condition, false)) {+ continue;+ }}if (resource.getPhysicalId() == null || skipRetainedResource(stack, resource, false)) {@@ -1175,7 +1221,7 @@ public class CloudFormationService {addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_IN_PROGRESS", null);try {- provisioner.delete(resource, region);+ deleteResourcePhysically(resource, region);addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_COMPLETE", null);stack.getResources().remove(resource.getLogicalId());@@ -1192,7 +1238,7 @@ public class CloudFormationService {resource.setStatusReason(reason);addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_FAILED", reason);- LOG.warnv("Failed to delete condition-disabled {0} ({1}) in stack {2}: {3}",+ LOG.warnv("Failed to delete removed or condition-disabled {0} ({1}) in stack {2}: {3}",resource.getResourceType(), resource.getPhysicalId(),stack.getStackName(), reason);}@@ -1223,7 +1269,7 @@ public class CloudFormationService {addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_IN_PROGRESS", null);try {- provisioner.delete(resource, region);+ deleteResourcePhysically(resource, region);resource.setStatus("DELETE_COMPLETE");addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_COMPLETE", null);@@ -1250,8 +1296,9 @@ public class CloudFormationService {stack.setStatusReason(reason);addEvent(stack, stack.getStackName(), stack.getStackId(),"AWS::CloudFormation::Stack", "DELETE_FAILED", reason);+ persistStack(stack);LOG.errorv("Stack {0} delete failed: {1}", stack.getStackName(), reason);- return;+ throw new IllegalStateException(reason);}stack.setStatus("DELETE_COMPLETE");@@ -1269,6 +1316,8 @@ public class CloudFormationService {LOG.errorv("Stack {0} delete failed: {1}", stack.getStackName(), e.getMessage());stack.setStatus("DELETE_FAILED");stack.setStatusReason(e.getMessage());+ persistStack(stack);+ throw (e instanceof RuntimeException re ? re : new RuntimeException(e));}}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.javaindex 25a619afb0..017369a886 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java@@ -7,6 +7,7 @@ import org.junit.jupiter.api.Test;import static io.restassured.RestAssured.given;import static org.hamcrest.MatcherAssert.assertThat;import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.not;import static org.junit.jupiter.api.Assertions.fail;/**@@ -20,6 +21,9 @@ import static org.junit.jupiter.api.Assertions.fail;@QuarkusTestclass CloudFormationDeletionPolicyIntegrationTest {+ private static final String CUSTOM_AUTH =+ "AWS4-HMAC-SHA256 Credential=111122223333/20260205/eu-west-1/cloudformation/aws4_request";+@Testvoid retainKeepsANonEmptyBucketAndTheStackStillCompletesTheDelete() throws InterruptedException {String suffix = Long.toString(System.nanoTime(), 36);@@ -171,9 +175,704 @@ class CloudFormationDeletionPolicyIntegrationTest {assertBucketDeleted(unrecognized);}+ @Test+ void retainKeepsResourceWhenRemovedFromTemplateOnUpdate() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucket1 = "cfn-update-remove-bucket1-" + suffix;+ String bucket2 = "cfn-update-remove-bucket2-" + suffix;+ String stackName = "cfn-update-remove-stack-" + suffix;++ String template1 = """+ {+ "Resources": {+ "KeptBucket": {+ "Type": "AWS::S3::Bucket",+ "DeletionPolicy": "Retain",+ "Properties": { "BucketName": "%s" }+ },+ "DeletedBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucket1, bucket2);++ String stackId = createStack(stackName, template1);+ awaitStackStatus(stackId, "CREATE_COMPLETE");++ String template2 = """+ {+ "Resources": {+ }+ }+ """;++ updateStack(stackName, template2);+ awaitStackStatus(stackId, "UPDATE_COMPLETE");++ assertBucketExists(bucket1);+ assertBucketDeleted(bucket2);++ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();+ assertThat(resXml, not(containsString("<LogicalResourceId>KeptBucket</LogicalResourceId>")));+ assertThat(resXml, not(containsString("<LogicalResourceId>DeletedBucket</LogicalResourceId>")));++ String events = describeStackEvents(stackId);+ assertThat(events, containsString("<ResourceStatus>DELETE_SKIPPED</ResourceStatus>"));+ }++ @Test+ void retainExceptOnCreateKeepsResourceWhenRemovedFromTemplateOnUpdate() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucket1 = "cfn-reoc-update-bucket1-" + suffix;+ String bucket2 = "cfn-reoc-update-bucket2-" + suffix;+ String stackName = "cfn-reoc-update-stack-" + suffix;++ String template1 = """+ {+ "Resources": {+ "KeptBucket": {+ "Type": "AWS::S3::Bucket",+ "DeletionPolicy": "RetainExceptOnCreate",+ "Properties": { "BucketName": "%s" }+ },+ "DeletedBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucket1, bucket2);++ String stackId = createStack(stackName, template1);+ try {+ awaitStackStatus(stackId, "CREATE_COMPLETE");++ String template2 = """+ {+ "Resources": {+ }+ }+ """;++ updateStack(stackName, template2);+ awaitStackStatus(stackId, "UPDATE_COMPLETE");++ assertBucketExists(bucket1);+ assertBucketDeleted(bucket2);++ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();+ assertThat(resXml, not(containsString("<LogicalResourceId>KeptBucket</LogicalResourceId>")));+ assertThat(resXml, not(containsString("<LogicalResourceId>DeletedBucket</LogicalResourceId>")));++ String events = describeStackEvents(stackId);+ assertThat(events, containsString("<ResourceStatus>DELETE_SKIPPED</ResourceStatus>"));+ } finally {+ deleteStack(stackName);+ given().header("Host", bucket1 + ".localhost").when().delete("/");+ }+ }++ @Test+ void nestedStackIsPhysicallyDeletedWhenRemovedFromTemplateOnUpdate() throws InterruptedException {+ given().header("Authorization", CUSTOM_AUTH).when().put("/nested-stack-templates").then().statusCode(200);+ String childTemplate = "{\"Resources\": {\"Queue\": {\"Type\": \"AWS::SQS::Queue\"}}}";+ given()+ .header("Authorization", CUSTOM_AUTH)+ .contentType("application/json")+ .body(childTemplate)+ .when()+ .put("/nested-stack-templates/child-update.json")+ .then()+ .statusCode(200);++ String stackName = "parent-stack-" + System.nanoTime();+ String template1 = """+ {+ "Resources": {+ "ChildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-update.json" }+ }+ }+ }+ """;++ String parentStackId = createStack(stackName, template1, CUSTOM_AUTH);+ try {+ awaitStackStatus(parentStackId, "CREATE_COMPLETE", CUSTOM_AUTH);++ String nestedStackId = getNestedStackId(parentStackId, "ChildStack", CUSTOM_AUTH);+ assertThat(nestedStackId, containsString("111122223333"));+ assertThat(nestedStackId, containsString("eu-west-1"));++ String template2 = "{\"Resources\": {}}";+ updateStack(stackName, template2, CUSTOM_AUTH);+ awaitStackStatus(parentStackId, "UPDATE_COMPLETE", CUSTOM_AUTH);++ awaitStackStatus(nestedStackId, "DELETE_COMPLETE", CUSTOM_AUTH);+ } finally {+ deleteStack(stackName);+ }+ }++ @Test+ void nestedStackIsRetainedWhenRemovedFromTemplateOnUpdateWithRetainPolicy() throws InterruptedException {+ given().when().put("/nested-stack-templates").then().statusCode(200);+ String childTemplate = "{\"Resources\": {\"Queue\": {\"Type\": \"AWS::SQS::Queue\"}}}";+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-retain.json");++ String stackName = "parent-retain-" + System.nanoTime();+ String template1 = """+ {+ "Resources": {+ "ChildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "DeletionPolicy": "Retain",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-retain.json" }+ }+ }+ }+ """;++ String parentStackId = createStack(stackName, template1, null);+ String nestedStackId = null;+ try {+ awaitStackStatus(parentStackId, "CREATE_COMPLETE", null);+ nestedStackId = getNestedStackId(parentStackId, "ChildStack", null);++ String template2 = "{\"Resources\": {}}";+ updateStack(stackName, template2, null);+ awaitStackStatus(parentStackId, "UPDATE_COMPLETE", null);++ awaitStackStatus(nestedStackId, "CREATE_COMPLETE", null);++ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();+ assertThat(resXml, not(containsString("<LogicalResourceId>ChildStack</LogicalResourceId>")));+ } finally {+ deleteStack(stackName);+ if (nestedStackId != null) {+ deleteStack(nestedStackId);+ }+ }+ }++ @Test+ void nestedStackWithNonEmptyBucketFailsDeletionOnUpdate_leavesChildAsDeleteFailedAndTracksInParent() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucketName = "cfn-nested-orphan-" + suffix;+ given().when().put("/nested-stack-templates").then().statusCode(200);+ String childTemplate = """+ {+ "Resources": {+ "ChildBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucketName);+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-orphan-" + suffix + ".json");++ String stackName = "parent-orphan-" + suffix;+ String template1 = """+ {+ "Resources": {+ "ChildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-orphan-%s.json" }+ }+ }+ }+ """.formatted(suffix);++ String parentStackId = createStack(stackName, template1);+ try {+ awaitStackStatus(parentStackId, "CREATE_COMPLETE");+ String nestedStackId = getNestedStackId(parentStackId, "ChildStack", null);++ // Put an object in the nested bucket so nested stack deletion will fail+ given().contentType("text/plain").body("keep").when().put("/" + bucketName + "/object.txt").then().statusCode(200);++ String template2 = "{\"Resources\": {}}";+ updateStack(stackName, template2);+ awaitStackStatus(parentStackId, "UPDATE_COMPLETE");+ assertThat(describeStacks(parentStackId), containsString("could not be deleted during update cleanup"));++ // Nested stack resource stays in parent as DELETE_FAILED+ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();+ assertThat(resXml, containsString("<LogicalResourceId>ChildStack</LogicalResourceId>"));+ assertThat(resXml, containsString("<ResourceStatus>DELETE_FAILED</ResourceStatus>"));++ // Child stack itself is DELETE_FAILED+ awaitStackStatus(nestedStackId, "DELETE_FAILED");++ // Clean up bucket and delete parent stack to verify clean recovery+ given().when().delete("/" + bucketName + "/object.txt").then().statusCode(204);+ deleteStack(stackName);+ awaitStackStatus(parentStackId, "DELETE_COMPLETE");+ awaitStackStatus(nestedStackId, "DELETE_COMPLETE");+ assertBucketDeleted(bucketName);+ } finally {+ given().when().delete("/" + bucketName + "/object.txt");+ deleteStack(stackName);+ }+ }++ @Test+ void deleteStack_withNestedStack_deletesChildStackAndPhysicalResources() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucketName = "cfn-nested-del-" + suffix;+ given().when().put("/nested-stack-templates").then().statusCode(200);+ String childTemplate = """+ {+ "Resources": {+ "ChildBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucketName);+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-del-" + suffix + ".json");++ String stackName = "parent-del-" + suffix;+ String template = """+ {+ "Resources": {+ "ChildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-del-%s.json" }+ }+ }+ }+ """.formatted(suffix);++ String parentStackId = createStack(stackName, template);+ try {+ awaitStackStatus(parentStackId, "CREATE_COMPLETE");+ String nestedStackId = getNestedStackId(parentStackId, "ChildStack", null);+ assertBucketExists(bucketName);++ deleteStack(stackName);+ awaitStackStatus(parentStackId, "DELETE_COMPLETE");+ awaitStackStatus(nestedStackId, "DELETE_COMPLETE");+ assertBucketDeleted(bucketName);+ } finally {+ deleteStack(stackName);+ }+ }++ @Test+ void deleteStack_withNestedStackContainingNonEmptyBucket_failsAndLeavesBothStacksInDeleteFailed() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucketName = "cfn-nested-delfail-" + suffix;+ given().when().put("/nested-stack-templates").then().statusCode(200);+ String childTemplate = """+ {+ "Resources": {+ "ChildBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucketName);+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-delfail-" + suffix + ".json");++ String stackName = "parent-delfail-" + suffix;+ String template = """+ {+ "Resources": {+ "ChildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-delfail-%s.json" }+ }+ }+ }+ """.formatted(suffix);++ String parentStackId = createStack(stackName, template);+ try {+ awaitStackStatus(parentStackId, "CREATE_COMPLETE");+ String nestedStackId = getNestedStackId(parentStackId, "ChildStack", null);++ // Put object in child bucket+ given().contentType("text/plain").body("blocker").when().put("/" + bucketName + "/object.txt").then().statusCode(200);++ deleteStack(stackName);+ awaitStackStatus(parentStackId, "DELETE_FAILED");+ awaitStackStatus(nestedStackId, "DELETE_FAILED");+ assertBucketExists(bucketName);++ // Clean up object and retry DeleteStack+ given().when().delete("/" + bucketName + "/object.txt").then().statusCode(204);+ deleteStack(stackName);+ awaitStackStatus(parentStackId, "DELETE_COMPLETE");+ awaitStackStatus(nestedStackId, "DELETE_COMPLETE");+ assertBucketDeleted(bucketName);+ } finally {+ given().when().delete("/" + bucketName + "/object.txt");+ deleteStack(stackName);+ }+ }++ @Test+ void createStack_rollbackWithNestedStack_deletesChildStackAndResources() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucketName = "cfn-nested-createrb-" + suffix;+ given().when().put("/nested-stack-templates").then().statusCode(200);+ String childTemplate = """+ {+ "Resources": {+ "ChildBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucketName);+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-createrb-" + suffix + ".json");++ String stackName = "parent-createrb-" + suffix;+ String template = """+ {+ "Resources": {+ "ChildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-createrb-%s.json" }+ },+ "BadSecret": {+ "Type": "AWS::SecretsManager::Secret",+ "DependsOn": "ChildStack",+ "Properties": {+ "Name": "bad-secret-%s",+ "SecretString": "explicit",+ "GenerateSecretString": { "PasswordLength": 32 }+ }+ }+ }+ }+ """.formatted(suffix, suffix);++ String parentStackId = createStack(stackName, template);+ try {+ assertThat(describeStacks(parentStackId), containsString("<StackStatus>ROLLBACK_COMPLETE</StackStatus>"));+ assertBucketDeleted(bucketName);+ } finally {+ deleteStack(stackName);+ }+ }++ @Test+ void updateStack_rollbackWithNestedStack_deletesNewlyAddedChildStack() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucketName = "cfn-nested-updaterb-" + suffix;+ given().when().put("/nested-stack-templates").then().statusCode(200);+ String childTemplate = """+ {+ "Resources": {+ "ChildBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucketName);+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child-updaterb-" + suffix + ".json");++ String stackName = "parent-updaterb-" + suffix;+ String sfnName = "initial-sfn-" + suffix;+ String initialTemplate = """+ {+ "Resources": {+ "InitialStateMachine": {+ "Type": "AWS::StepFunctions::StateMachine",+ "Properties": {+ "StateMachineName": "%s",+ "RoleArn": "arn:aws:iam::000000000000:role/cfn-sfn-rollback-role",+ "DefinitionString": "{\\"StartAt\\":\\"Done\\",\\"States\\":{\\"Done\\":{\\"Type\\":\\"Pass\\",\\"Result\\":\\"marker-v1\\",\\"End\\":true}}}"+ }+ }+ }+ }+ """.formatted(sfnName);++ String parentStackId = createStack(stackName, initialTemplate);+ try {+ awaitStackStatus(parentStackId, "CREATE_COMPLETE");++ String failingUpdateTemplate = """+ {+ "Resources": {+ "InitialStateMachine": {+ "Type": "AWS::StepFunctions::StateMachine",+ "Properties": {+ "StateMachineName": "%s",+ "RoleArn": "arn:aws:iam::000000000000:role/cfn-sfn-rollback-role",+ "DefinitionString": "{\\"StartAt\\":\\"Done\\",\\"States\\":{\\"Done\\":{\\"Type\\":\\"Pass\\",\\"Result\\":\\"marker-v2\\",\\"End\\":true}}}"+ }+ },+ "ChildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child-updaterb-%s.json" }+ },+ "BadSecret": {+ "Type": "AWS::SecretsManager::Secret",+ "DependsOn": "ChildStack",+ "Properties": {+ "Name": "bad-secret-update-%s",+ "SecretString": "explicit",+ "GenerateSecretString": { "PasswordLength": 32 }+ }+ }+ }+ }+ """.formatted(sfnName, suffix, suffix);++ updateStack(stackName, failingUpdateTemplate);+ awaitStackStatus(parentStackId, "UPDATE_ROLLBACK_COMPLETE");+ assertBucketDeleted(bucketName);++ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();+ assertThat(resXml, containsString("<LogicalResourceId>InitialStateMachine</LogicalResourceId>"));+ assertThat(resXml, not(containsString("<LogicalResourceId>ChildStack</LogicalResourceId>")));+ assertThat(resXml, not(containsString("<LogicalResourceId>BadSecret</LogicalResourceId>")));++ deleteStack(stackName);+ awaitStackStatus(parentStackId, "DELETE_COMPLETE");+ } finally {+ deleteStack(stackName);+ }+ }++ @Test+ void nestedStackWithMultiLevelHierarchy_deletesRecursivelyOnUpdate() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucketName = "cfn-nested3-del-" + suffix;+ given().when().put("/nested-stack-templates").then().statusCode(200);++ String grandchildTemplate = """+ {+ "Resources": {+ "GrandchildBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucketName);+ given().contentType("application/json").body(grandchildTemplate).when().put("/nested-stack-templates/grandchild-" + suffix + ".json");++ String childTemplate = """+ {+ "Resources": {+ "GrandchildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/grandchild-%s.json" }+ }+ }+ }+ """.formatted(suffix);+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child3-" + suffix + ".json");++ String rootStackName = "root3-del-" + suffix;+ String rootTemplate = """+ {+ "Resources": {+ "ChildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child3-%s.json" }+ }+ }+ }+ """.formatted(suffix);++ String rootStackId = createStack(rootStackName, rootTemplate);+ try {+ awaitStackStatus(rootStackId, "CREATE_COMPLETE");+ String childStackId = getNestedStackId(rootStackId, "ChildStack", null);+ String grandchildStackId = getNestedStackId(childStackId, "GrandchildStack", null);+ assertBucketExists(bucketName);++ String updatedRootTemplate = "{\"Resources\": {}}";+ updateStack(rootStackName, updatedRootTemplate);+ awaitStackStatus(rootStackId, "UPDATE_COMPLETE");++ awaitStackStatus(childStackId, "DELETE_COMPLETE");+ awaitStackStatus(grandchildStackId, "DELETE_COMPLETE");+ assertBucketDeleted(bucketName);+ } finally {+ deleteStack(rootStackName);+ }+ }++ @Test+ void nestedStackWithMultiLevelHierarchy_propagatesDeleteFailedOnUpdate() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucketName = "cfn-nested3-fail-" + suffix;+ given().when().put("/nested-stack-templates").then().statusCode(200);++ String grandchildTemplate = """+ {+ "Resources": {+ "GrandchildBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucketName);+ given().contentType("application/json").body(grandchildTemplate).when().put("/nested-stack-templates/grandchild-fail-" + suffix + ".json");++ String childTemplate = """+ {+ "Resources": {+ "GrandchildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/grandchild-fail-%s.json" }+ }+ }+ }+ """.formatted(suffix);+ given().contentType("application/json").body(childTemplate).when().put("/nested-stack-templates/child3-fail-" + suffix + ".json");++ String rootStackName = "root3-fail-" + suffix;+ String rootTemplate = """+ {+ "Resources": {+ "ChildStack": {+ "Type": "AWS::CloudFormation::Stack",+ "Properties": { "TemplateURL": "http://localhost/nested-stack-templates/child3-fail-%s.json" }+ }+ }+ }+ """.formatted(suffix);++ String rootStackId = createStack(rootStackName, rootTemplate);+ try {+ awaitStackStatus(rootStackId, "CREATE_COMPLETE");+ String childStackId = getNestedStackId(rootStackId, "ChildStack", null);+ String grandchildStackId = getNestedStackId(childStackId, "GrandchildStack", null);++ // Put blocker object into grandchild's bucket+ given().contentType("text/plain").body("keep").when().put("/" + bucketName + "/blocker.txt").then().statusCode(200);++ String updatedRootTemplate = "{\"Resources\": {}}";+ updateStack(rootStackName, updatedRootTemplate);+ awaitStackStatus(rootStackId, "UPDATE_COMPLETE");+ assertThat(describeStacks(rootStackId), containsString("could not be deleted during update cleanup"));++ awaitStackStatus(grandchildStackId, "DELETE_FAILED");+ awaitStackStatus(childStackId, "DELETE_FAILED");++ // Clean up blocker and delete root stack+ given().when().delete("/" + bucketName + "/blocker.txt").then().statusCode(204);+ deleteStack(rootStackName);+ awaitStackStatus(rootStackId, "DELETE_COMPLETE");+ awaitStackStatus(childStackId, "DELETE_COMPLETE");+ awaitStackStatus(grandchildStackId, "DELETE_COMPLETE");+ assertBucketDeleted(bucketName);+ } finally {+ given().when().delete("/" + bucketName + "/blocker.txt");+ deleteStack(rootStackName);+ }+ }++ @Test+ void updateStack_rollbackPreservesRemovedResourcesWhenNewResourceFails() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucketA = "cfn-rb-keep-a-" + suffix;+ String sfnName = "cfn-rb-sfn-" + suffix;+ String stackName = "cfn-rb-preserve-" + suffix;++ String initialTemplate = """+ {+ "Resources": {+ "BucketA": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ },+ "InitialStateMachine": {+ "Type": "AWS::StepFunctions::StateMachine",+ "Properties": {+ "StateMachineName": "%s",+ "RoleArn": "arn:aws:iam::000000000000:role/cfn-sfn-rollback-role",+ "DefinitionString": "{\\"StartAt\\":\\"Done\\",\\"States\\":{\\"Done\\":{\\"Type\\":\\"Pass\\",\\"Result\\":\\"marker-v1\\",\\"End\\":true}}}"+ }+ }+ }+ }+ """.formatted(bucketA, sfnName);++ String stackId = createStack(stackName, initialTemplate);+ try {+ awaitStackStatus(stackId, "CREATE_COMPLETE");+ assertBucketExists(bucketA);++ // Update template: omits BucketA, updates InitialStateMachine, and adds a resource that fails creation+ String failingUpdateTemplate = """+ {+ "Resources": {+ "InitialStateMachine": {+ "Type": "AWS::StepFunctions::StateMachine",+ "Properties": {+ "StateMachineName": "%s",+ "RoleArn": "arn:aws:iam::000000000000:role/cfn-sfn-rollback-role",+ "DefinitionString": "{\\"StartAt\\":\\"Done\\",\\"States\\":{\\"Done\\":{\\"Type\\":\\"Pass\\",\\"Result\\":\\"marker-v2\\",\\"End\\":true}}}"+ }+ },+ "BadSecret": {+ "Type": "AWS::SecretsManager::Secret",+ "Properties": {+ "Name": "bad-secret-%s",+ "SecretString": "explicit",+ "GenerateSecretString": { "PasswordLength": 32 }+ }+ }+ }+ }+ """.formatted(sfnName, suffix);++ updateStack(stackName, failingUpdateTemplate);+ awaitStackStatus(stackId, "UPDATE_ROLLBACK_COMPLETE");++ // BucketA was removed in the failing update template, but rollback must preserve it!+ assertBucketExists(bucketA);++ String resXml = cfnQuery("DescribeStackResources", stackName, null).then().statusCode(200).extract().asString();+ assertThat(resXml, containsString("<LogicalResourceId>BucketA</LogicalResourceId>"));+ assertThat(resXml, containsString("<LogicalResourceId>InitialStateMachine</LogicalResourceId>"));+ assertThat(resXml, not(containsString("<LogicalResourceId>BadSecret</LogicalResourceId>")));+ } finally {+ deleteStack(stackName);+ awaitStackStatus(stackId, "DELETE_COMPLETE");+ assertBucketDeleted(bucketA);+ }+ }++ private static String getNestedStackId(String stackName, String logicalId, String auth) {+ String xml = cfnQuery("DescribeStackResources", stackName, auth).then().statusCode(200).extract().asString();+ int logIdx = xml.indexOf("<LogicalResourceId>" + logicalId + "</LogicalResourceId>");+ if (logIdx == -1) {+ fail("Logical resource " + logicalId + " not found");+ }+ int physStart = xml.indexOf("<PhysicalResourceId>", logIdx) + "<PhysicalResourceId>".length();+ int physEnd = xml.indexOf("</PhysicalResourceId>", physStart);+ return xml.substring(physStart, physEnd);+ }+private static String createStack(String stackName, String template) {- String xml = given()- .contentType("application/x-www-form-urlencoded")+ return createStack(stackName, template, null);+ }++ private static String createStack(String stackName, String template, String auth) {+ var req = given().contentType("application/x-www-form-urlencoded");+ if (auth != null) {+ req.header("Authorization", auth);+ }+ String xml = req.formParam("Action", "CreateStack").formParam("StackName", stackName).formParam("TemplateBody", template)@@ -198,12 +897,34 @@ class CloudFormationDeletionPolicyIntegrationTest {.statusCode(200);}+ private static void updateStack(String stackName, String template) {+ updateStack(stackName, template, null);+ }++ private static void updateStack(String stackName, String template, String auth) {+ var req = given().contentType("application/x-www-form-urlencoded");+ if (auth != null) {+ req.header("Authorization", auth);+ }+ req.formParam("Action", "UpdateStack")+ .formParam("StackName", stackName)+ .formParam("TemplateBody", template)+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }+/** DeleteStack runs asynchronously, and a deleted stack stays describable by its stack ID. */private static void awaitStackStatus(String stackId, String status) throws InterruptedException {+ awaitStackStatus(stackId, status, null);+ }++ private static void awaitStackStatus(String stackId, String status, String auth) throws InterruptedException {String xml = "";long deadline = System.currentTimeMillis() + 10_000;while (System.currentTimeMillis() < deadline) {- xml = describeStacks(stackId);+ xml = describeStacks(stackId, auth);if (xml.contains("<StackStatus>" + status + "</StackStatus>")) {return;}@@ -213,20 +934,27 @@ class CloudFormationDeletionPolicyIntegrationTest {}private static String describeStacks(String stackId) {- return cfnQuery("DescribeStacks", stackId).then().statusCode(200).extract().asString();+ return describeStacks(stackId, null);+ }++ private static String describeStacks(String stackId, String auth) {+ return cfnQuery("DescribeStacks", stackId, auth).then().statusCode(200).extract().asString();}private static String describeStackEvents(String stackId) {- return cfnQuery("DescribeStackEvents", stackId).then().statusCode(200).extract().asString();+ return cfnQuery("DescribeStackEvents", stackId, null).then().statusCode(200).extract().asString();}private static Response cfnQuery(String action, String stackId) {- return given()- .contentType("application/x-www-form-urlencoded")- .formParam("Action", action)- .formParam("StackName", stackId)- .when()- .post("/");+ return cfnQuery(action, stackId, null);+ }++ private static Response cfnQuery(String action, String stackId, String auth) {+ var req = given().contentType("application/x-www-form-urlencoded");+ if (auth != null) {+ req.header("Authorization", auth);+ }+ return req.formParam("Action", action).formParam("StackName", stackId).when().post("/");}private static void assertBucketExists(String bucket) {diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceConditionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceConditionIntegrationTest.javaindex d54ffe54ac..182034caae 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceConditionIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceConditionIntegrationTest.java@@ -927,6 +927,141 @@ class CloudFormationResourceConditionIntegrationTest {}}+ @Test+ void updateStack_deletesResourceWhenRemovedFromTemplate() {+ String suffix = Long.toString(System.nanoTime(), 36);+ String stackName = "cfn-rem-update-" + suffix;+ String queueName = "removed-update-" + suffix;++ String initialTemplate = """+ {+ "Resources": {+ "OptionalQueue": {+ "Type": "AWS::SQS::Queue",+ "Properties": { "QueueName": "%s" }+ }+ }+ }+ """.formatted(queueName);+ String updatedTemplate = """+ {+ "Resources": {+ "DummyQueue": {+ "Type": "AWS::SQS::Queue",+ "Properties": { "QueueName": "dummy-%s" }+ }+ }+ }+ """.formatted(suffix);++ createStack(stackName, initialTemplate);+ try {+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "GetQueueUrl")+ .formParam("QueueName", queueName)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString(queueName));++ updateStack(stackName, updatedTemplate);++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "GetQueueUrl")+ .formParam("QueueName", queueName)+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("ErrorResponse.Error.Code", equalTo("AWS.SimpleQueueService.NonExistentQueue"));++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStackResources")+ .formParam("StackName", stackName)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(not(containsString("<LogicalResourceId>OptionalQueue</LogicalResourceId>")));+ } finally {+ deleteStack(stackName);+ }+ }++ @Test+ void updateStack_whenResourceRemovedFromTemplateWithNonEmptyBucket_keepsBucketAsDeleteFailedAndReclaimsOnDelete() {+ String suffix = Long.toString(System.nanoTime(), 36);+ String stackName = "cfn-rem-orphan-" + suffix;+ String bucketName = "removed-orphan-" + suffix;++ String initialTemplate = """+ {+ "Resources": {+ "OptionalBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucketName);+ String updatedTemplate = """+ {+ "Resources": {+ }+ }+ """;++ createStack(stackName, initialTemplate);+ try {+ given().header("Host", bucketName + ".localhost").when().get("/").then().statusCode(200);+ given()+ .contentType("text/plain")+ .body("prevent removed bucket deletion")+ .when()+ .put("/" + bucketName + "/object.txt")+ .then()+ .statusCode(200);++ // Cleanup delete fails (bucket non-empty), but update completes; bucket stays under stack management as DELETE_FAILED.+ updateStack(stackName, updatedTemplate);+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", stackName)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<StackStatus>UPDATE_COMPLETE</StackStatus>"))+ .body(containsString("could not be deleted during update cleanup"));++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStackResources")+ .formParam("StackName", stackName)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<LogicalResourceId>OptionalBucket</LogicalResourceId>"))+ .body(containsString("<ResourceStatus>DELETE_FAILED</ResourceStatus>"));++ // Clear the blocker, then DeleteStack retries the DELETE_FAILED resource+ given().header("Host", bucketName + ".localhost").delete("/object.txt").then().statusCode(204);+ deleteStack(stackName);+ awaitStackGone(stackName);+ given().header("Host", bucketName + ".localhost").when().get("/").then().statusCode(404);+ } finally {+ given().header("Host", bucketName + ".localhost").delete("/object.txt");+ given().header("Host", bucketName + ".localhost").delete("/");+ deleteStack(stackName);+ }+ }+private static void createStack(String stackName, String template) {given().contentType("application/x-www-form-urlencoded")