Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): fail stack delete when a managed resource cannot be deleted (#1554)
Dixit R Jain0530ca11ab2f
diff --git a/compatibility-tests/sdk-test-awscli/test/cloudformation.bats b/compatibility-tests/sdk-test-awscli/test/cloudformation.batsindex 89abfa320f..644e76422b 100644--- a/compatibility-tests/sdk-test-awscli/test/cloudformation.bats+++ b/compatibility-tests/sdk-test-awscli/test/cloudformation.bats@@ -200,3 +200,111 @@ EOF[ -n "$lb_arn" ][[ "$lb_arn" == *":loadbalancer/"* ]]}++# ── DeleteStack with managed S3 buckets (regression: issue #1539) ──────────────+#+# Real AWS CloudFormation does not report a stack as DELETE_COMPLETE while a+# managed resource still exists. Deleting a stack that owns a NON-EMPTY+# AWS::S3::Bucket must put the stack into DELETE_FAILED (S3 DeleteBucket refuses+# a non-empty bucket) and leave the bucket in place — it must NOT silently report+# success while the bucket and its objects remain.+# An EMPTY managed bucket must be deleted as part of the stack delete.+# https://github.com/floci-io/floci/issues/1539++# Poll describe-stacks until the stack reaches the expected status, or report the+# special value DOES_NOT_EXIST once the stack is gone.+cfn_wait_status() {+ local stack="$1" expected="$2" tries="${3:-40}"+ local i out status+ for ((i = 0; i < tries; i++)); do+ # `|| true`: describe-stacks exits non-zero once the stack is gone, which+ # would otherwise abort this command substitution under bats' errexit.+ out=$(aws_cmd cloudformation describe-stacks --stack-name "$stack" 2>&1 || true)+ if [[ "$out" == *"does not exist"* ]]; then+ status="DOES_NOT_EXIST"+ else+ status=$(echo "$out" | jq -r '.Stacks[0].StackStatus' 2>/dev/null)+ fi+ if [ "$status" = "$expected" ]; then+ return 0+ fi+ sleep 0.25+ done+ echo "timeout waiting for stack '$stack' to reach '$expected' (last status: '$status')" >&2+ return 1+}++@test "CloudFormation: delete-stack with non-empty S3 bucket fails and keeps the bucket (issue #1539)" {+ local bucket="bats-cfn-nonempty-$(unique_name bkt)"+ cat > "$TEMPLATE_FILE" << EOF+{+ "AWSTemplateFormatVersion": "2010-09-09",+ "Resources": {+ "Bucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "$bucket" }+ }+ }+}+EOF+ run aws_cmd cloudformation create-stack \+ --stack-name "$STACK_NAME" \+ --template-body "file://$TEMPLATE_FILE"+ assert_success+ cfn_wait_status "$STACK_NAME" "CREATE_COMPLETE"++ # Put an object so the managed bucket is non-empty.+ local body_file+ body_file=$(mktemp)+ echo -n "hello floci 1539" > "$body_file"+ run aws_cmd s3api put-object --bucket "$bucket" --key "object.txt" --body "$body_file"+ assert_success+ rm -f "$body_file"++ run aws_cmd cloudformation delete-stack --stack-name "$STACK_NAME"+ assert_success++ # AWS leaves the stack in DELETE_FAILED — it must NOT report DELETE_COMPLETE+ # (i.e. the stack must NOT silently disappear).+ cfn_wait_status "$STACK_NAME" "DELETE_FAILED"++ # The managed bucket and its object must still exist after the failed delete.+ run aws_cmd s3api head-bucket --bucket "$bucket"+ assert_success++ # Cleanup: empty + remove the leftover bucket, then drop the stack.+ aws_cmd s3 rm "s3://$bucket" --recursive >/dev/null 2>&1 || true+ aws_cmd s3api delete-bucket --bucket "$bucket" >/dev/null 2>&1 || true+}++@test "CloudFormation: delete-stack with empty S3 bucket removes the bucket" {+ local bucket="bats-cfn-empty-$(unique_name bkt)"+ cat > "$TEMPLATE_FILE" << EOF+{+ "AWSTemplateFormatVersion": "2010-09-09",+ "Resources": {+ "Bucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "$bucket" }+ }+ }+}+EOF+ run aws_cmd cloudformation create-stack \+ --stack-name "$STACK_NAME" \+ --template-body "file://$TEMPLATE_FILE"+ assert_success+ cfn_wait_status "$STACK_NAME" "CREATE_COMPLETE"++ run aws_cmd s3api head-bucket --bucket "$bucket"+ assert_success++ run aws_cmd cloudformation delete-stack --stack-name "$STACK_NAME"+ assert_success+ cfn_wait_status "$STACK_NAME" "DOES_NOT_EXIST"+ STACK_NAME="" # prevent teardown from trying again++ # The empty managed bucket must be gone.+ run aws_cmd s3api head-bucket --bucket "$bucket"+ assert_failure+}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.javaindex a822df2246..edfb888357 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java@@ -389,63 +389,67 @@ public class CloudFormationResourceProvisioner {delete(resourceType, resource.getPhysicalId(), region);}+ /**+ * Deletes a single resource by type + physical id. Failures propagate to the caller+ * (CloudFormationService#deleteStackResources) so the stack transitions to DELETE_FAILED,+ * matching AWS — e.g. deleting a non-empty S3 bucket raises BucketNotEmpty and must not be+ * silently reported as a successful stack deletion. Resource types that AWS itself treats+ * leniently keep their dedicated handling: the {@code *Safe} helpers below swallow expected+ * conflicts, and KMS keys are intentionally left for scheduled deletion.+ */public void delete(String resourceType, String physicalId, String region) {- try {- switch (resourceType) {- case "AWS::S3::Bucket" -> s3Service.deleteBucket(physicalId);- case "AWS::SQS::Queue" -> sqsService.deleteQueue(physicalId, region);- case "AWS::SNS::Topic" -> snsService.deleteTopic(physicalId, region);- case "AWS::SNS::Subscription" -> snsService.unsubscribe(physicalId, region);- case "AWS::DynamoDB::Table" -> dynamoDbService.deleteTable(physicalId, region);- case "AWS::Lambda::Function" -> lambdaService.deleteFunction(region, physicalId);- case "AWS::IAM::Role" -> deleteRoleSafe(physicalId);- case "AWS::IAM::Policy", "AWS::IAM::ManagedPolicy" -> deletePolicySafe(physicalId);- case "AWS::IAM::InstanceProfile" -> iamService.deleteInstanceProfile(physicalId);- case "AWS::SSM::Parameter" -> ssmService.deleteParameter(physicalId, region);- case "AWS::KMS::Key" -> {- } // KMS keys can't be immediately deleted; skip- case "AWS::KMS::Alias" -> kmsService.deleteAlias(physicalId, region);- case "AWS::SecretsManager::Secret" ->- secretsManagerService.deleteSecret(physicalId, null, true, region);- case "AWS::Events::Rule" -> deleteEventBridgeRuleSafe(physicalId, region);- case "AWS::ApiGateway::RestApi" -> apiGatewayService.deleteRestApi(region, physicalId);- case "AWS::ApiGatewayV2::Api" -> apiGatewayV2Service.deleteApi(region, physicalId);- case "AWS::ECR::Repository" ->- ecrService.deleteRepository(physicalId, null, true, region);- case "AWS::Pipes::Pipe" -> pipesService.deletePipe(physicalId, region);- case "AWS::StepFunctions::StateMachine" -> stepFunctionsService.deleteStateMachine(physicalId);- case "AWS::Lambda::EventSourceMapping" -> lambdaService.deleteEventSourceMapping(physicalId);- case "AWS::Lambda::LayerVersion" -> deleteLambdaLayerVersion(physicalId, region);- case "AWS::Cognito::UserPool" -> cognitoService.deleteUserPool(physicalId);- case "AWS::Cognito::UserPoolClient" -> cognitoService.deleteUserPoolClient(physicalId);- case "AWS::ECS::Cluster" -> ecsService.deleteCluster(physicalId, region);- case "AWS::ECS::TaskDefinition" -> ecsService.deregisterTaskDefinition(physicalId, region);- case "AWS::ECS::Service" -> deleteEcsServiceSafe(physicalId, region);- case "AWS::ElasticLoadBalancingV2::LoadBalancer" -> elbV2Service.deleteLoadBalancer(region, physicalId);- case "AWS::ElasticLoadBalancingV2::TargetGroup" -> elbV2Service.deleteTargetGroup(region, physicalId);- case "AWS::ElasticLoadBalancingV2::Listener" -> elbV2Service.deleteListener(region, physicalId);- case "AWS::ElasticLoadBalancingV2::ListenerRule" -> elbV2Service.deleteRule(region, physicalId);- case "AWS::KinesisFirehose::DeliveryStream" -> firehoseService.deleteDeliveryStream(physicalId);- case "AWS::EC2::SecurityGroup" -> ec2Service.deleteSecurityGroup(region, physicalId);- case "AWS::EC2::Instance" -> ec2Service.terminateInstances(region, List.of(physicalId));- case "AWS::RDS::DBInstance" -> rdsService.deleteDbInstance(physicalId);- case "AWS::RDS::DBCluster" -> rdsService.deleteDbCluster(physicalId);- case "AWS::RDS::DBSubnetGroup" -> rdsService.deleteDbSubnetGroup(physicalId);- case "AWS::RDS::DBParameterGroup" -> rdsService.deleteDbParameterGroup(physicalId);- case "AWS::RDS::DBClusterParameterGroup" -> rdsService.deleteDbClusterParameterGroup(physicalId);- case "AWS::EKS::Cluster" -> eksService.deleteCluster(physicalId);- case "AWS::Logs::LogGroup" -> logsService.deleteLogGroup(physicalId, region);- case "AWS::Kinesis::Stream" -> kinesisService.deleteStream(physicalId, region);- case "AWS::CloudWatch::Alarm" ->- cloudWatchMetricsService.deleteAlarms(List.of(physicalId), region);- case "AWS::AutoScaling::LaunchConfiguration" ->- autoScalingService.deleteLaunchConfiguration(region, physicalId);- case "AWS::AutoScaling::AutoScalingGroup" ->- autoScalingService.deleteAutoScalingGroup(region, physicalId, true);- default -> LOG.debugv("Skipping delete of unsupported resource type: {0}", resourceType);- }- } catch (Exception e) {- LOG.debugv("Error deleting {0} ({1}): {2}", resourceType, physicalId, e.getMessage());+ switch (resourceType) {+ case "AWS::S3::Bucket" -> s3Service.deleteBucket(physicalId);+ case "AWS::SQS::Queue" -> sqsService.deleteQueue(physicalId, region);+ case "AWS::SNS::Topic" -> snsService.deleteTopic(physicalId, region);+ case "AWS::SNS::Subscription" -> snsService.unsubscribe(physicalId, region);+ case "AWS::DynamoDB::Table" -> dynamoDbService.deleteTable(physicalId, region);+ case "AWS::Lambda::Function" -> lambdaService.deleteFunction(region, physicalId);+ case "AWS::IAM::Role" -> deleteRoleSafe(physicalId);+ case "AWS::IAM::Policy", "AWS::IAM::ManagedPolicy" -> deletePolicySafe(physicalId);+ case "AWS::IAM::InstanceProfile" -> iamService.deleteInstanceProfile(physicalId);+ case "AWS::SSM::Parameter" -> ssmService.deleteParameter(physicalId, region);+ case "AWS::KMS::Key" -> {+ } // KMS keys can't be immediately deleted; skip+ case "AWS::KMS::Alias" -> kmsService.deleteAlias(physicalId, region);+ case "AWS::SecretsManager::Secret" ->+ secretsManagerService.deleteSecret(physicalId, null, true, region);+ case "AWS::Events::Rule" -> deleteEventBridgeRuleSafe(physicalId, region);+ case "AWS::ApiGateway::RestApi" -> apiGatewayService.deleteRestApi(region, physicalId);+ case "AWS::ApiGatewayV2::Api" -> apiGatewayV2Service.deleteApi(region, physicalId);+ case "AWS::ECR::Repository" ->+ ecrService.deleteRepository(physicalId, null, true, region);+ case "AWS::Pipes::Pipe" -> pipesService.deletePipe(physicalId, region);+ case "AWS::StepFunctions::StateMachine" -> stepFunctionsService.deleteStateMachine(physicalId);+ case "AWS::Lambda::EventSourceMapping" -> lambdaService.deleteEventSourceMapping(physicalId);+ case "AWS::Lambda::LayerVersion" -> deleteLambdaLayerVersion(physicalId, region);+ case "AWS::Cognito::UserPool" -> cognitoService.deleteUserPool(physicalId);+ case "AWS::Cognito::UserPoolClient" -> cognitoService.deleteUserPoolClient(physicalId);+ case "AWS::ECS::Cluster" -> ecsService.deleteCluster(physicalId, region);+ case "AWS::ECS::TaskDefinition" -> ecsService.deregisterTaskDefinition(physicalId, region);+ case "AWS::ECS::Service" -> deleteEcsServiceSafe(physicalId, region);+ case "AWS::ElasticLoadBalancingV2::LoadBalancer" -> elbV2Service.deleteLoadBalancer(region, physicalId);+ case "AWS::ElasticLoadBalancingV2::TargetGroup" -> elbV2Service.deleteTargetGroup(region, physicalId);+ case "AWS::ElasticLoadBalancingV2::Listener" -> elbV2Service.deleteListener(region, physicalId);+ case "AWS::ElasticLoadBalancingV2::ListenerRule" -> elbV2Service.deleteRule(region, physicalId);+ case "AWS::KinesisFirehose::DeliveryStream" -> firehoseService.deleteDeliveryStream(physicalId);+ case "AWS::EC2::SecurityGroup" -> ec2Service.deleteSecurityGroup(region, physicalId);+ case "AWS::EC2::Instance" -> ec2Service.terminateInstances(region, List.of(physicalId));+ case "AWS::RDS::DBInstance" -> rdsService.deleteDbInstance(physicalId);+ case "AWS::RDS::DBCluster" -> rdsService.deleteDbCluster(physicalId);+ case "AWS::RDS::DBSubnetGroup" -> rdsService.deleteDbSubnetGroup(physicalId);+ case "AWS::RDS::DBParameterGroup" -> rdsService.deleteDbParameterGroup(physicalId);+ case "AWS::RDS::DBClusterParameterGroup" -> rdsService.deleteDbClusterParameterGroup(physicalId);+ case "AWS::EKS::Cluster" -> eksService.deleteCluster(physicalId);+ case "AWS::Logs::LogGroup" -> logsService.deleteLogGroup(physicalId, region);+ case "AWS::Kinesis::Stream" -> kinesisService.deleteStream(physicalId, region);+ case "AWS::CloudWatch::Alarm" ->+ cloudWatchMetricsService.deleteAlarms(List.of(physicalId), region);+ case "AWS::AutoScaling::LaunchConfiguration" ->+ autoScalingService.deleteLaunchConfiguration(region, physicalId);+ case "AWS::AutoScaling::AutoScalingGroup" ->+ autoScalingService.deleteAutoScalingGroup(region, physicalId, true);+ default -> LOG.debugv("Skipping delete of unsupported resource type: {0}", resourceType);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.javaindex 80b5f032f2..75b9b2242e 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java@@ -549,17 +549,49 @@ public class CloudFormationService {List<StackResource> resources = new ArrayList<>(stack.getResources().values());Collections.reverse(resources); // Delete in reverse order+ List<String> failedResources = new ArrayList<>();for (StackResource resource : resources) {- if (resource.getPhysicalId() != null && "CREATE_COMPLETE".equals(resource.getStatus())) {- addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),- resource.getResourceType(), "DELETE_IN_PROGRESS", null);+ // CREATE_COMPLETE: first delete attempt. DELETE_FAILED: a previous delete left the+ // resource behind (e.g. the bucket was non-empty); AWS re-attempts it on retry.+ boolean deletable = "CREATE_COMPLETE".equals(resource.getStatus())+ || "DELETE_FAILED".equals(resource.getStatus());+ if (resource.getPhysicalId() == null || !deletable) {+ continue;+ }+ addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),+ resource.getResourceType(), "DELETE_IN_PROGRESS", null);+ try {provisioner.delete(resource, region);resource.setStatus("DELETE_COMPLETE");addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_COMPLETE", null);+ } catch (Exception e) {+ // AWS leaves the stack in DELETE_FAILED when a managed resource cannot be+ // deleted (e.g. a non-empty S3 bucket raises BucketNotEmpty). The stack must+ // not be reported as a successful deletion while the resource still exists.+ // Remaining resources are still attempted, matching AWS.+ failedResources.add(resource.getLogicalId());+ resource.setStatus("DELETE_FAILED");+ resource.setStatusReason(e.getMessage());+ addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),+ resource.getResourceType(), "DELETE_FAILED", e.getMessage());+ LOG.warnv("Failed to delete {0} ({1}) in stack {2}: {3}",+ resource.getResourceType(), resource.getPhysicalId(),+ stack.getStackName(), e.getMessage());}}+ if (!failedResources.isEmpty()) {+ String reason = "The following resource(s) failed to delete: ["+ + String.join(", ", failedResources) + "].";+ stack.setStatus("DELETE_FAILED");+ stack.setStatusReason(reason);+ addEvent(stack, stack.getStackName(), stack.getStackId(),+ "AWS::CloudFormation::Stack", "DELETE_FAILED", reason);+ LOG.errorv("Stack {0} delete failed: {1}", stack.getStackName(), reason);+ return;+ }+stack.setStatus("DELETE_COMPLETE");addEvent(stack, stack.getStackName(), stack.getStackId(),"AWS::CloudFormation::Stack", "DELETE_COMPLETE", null);diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.javaindex 0fb1b2b34f..f022833979 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java@@ -1138,6 +1138,95 @@ class CloudFormationIntegrationTest {.body(containsString("does not exist"));}+ // Regression: issue #1539. Deleting a stack that owns a NON-EMPTY S3 bucket must leave the+ // stack in DELETE_FAILED (S3 refuses to delete a non-empty bucket) and keep the bucket — it+ // must not silently report DELETE_COMPLETE while the bucket and its objects still exist.+ @Test+ void deleteStack_withNonEmptyS3Bucket_failsAndKeepsBucket() throws Exception {+ String bucket = "cfn-nonempty-delete-test-bucket";+ String template = """+ {+ "Resources": {+ "MyBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": {+ "BucketName": "%s"+ }+ }+ }+ }+ """.formatted(bucket);++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateStack")+ .formParam("StackName", "cfn-nonempty-delete-stack")+ .formParam("TemplateBody", template)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<StackId>"));++ // The managed bucket exists.+ given()+ .header("Host", bucket + ".localhost")+ .when()+ .get("/")+ .then()+ .statusCode(200);++ // Put an object so the bucket is non-empty.+ given()+ .contentType("text/plain")+ .body("hello floci 1539")+ .when()+ .put("/" + bucket + "/object.txt")+ .then()+ .statusCode(200);++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteStack")+ .formParam("StackName", "cfn-nonempty-delete-stack")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ // The stack must settle into DELETE_FAILED — never DELETE_COMPLETE.+ String statusXml = null;+ long deadline = System.currentTimeMillis() + 10_000;+ while (System.currentTimeMillis() < deadline) {+ statusXml = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", "cfn-nonempty-delete-stack")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().asString();++ if (statusXml.contains("<StackStatus>DELETE_FAILED</StackStatus>")+ || statusXml.contains("<StackStatus>DELETE_COMPLETE</StackStatus>")) {+ break;+ }+ Thread.sleep(200);+ }++ assertThat(statusXml, containsString("<StackStatus>DELETE_FAILED</StackStatus>"));+ assertThat(statusXml, not(containsString("<StackStatus>DELETE_COMPLETE</StackStatus>")));++ // The managed bucket must still exist after the failed delete.+ given()+ .header("Host", bucket + ".localhost")+ .when()+ .get("/")+ .then()+ .statusCode(200);+ }+@Testvoid describeDeletedStack_byArn_expiresAfterRetentionWindow() throws Exception {String template = """