Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): tolerate missing stack resources during deletion (#2039)
Aniket Shukla12b8ddd6291b
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.javaindex 1585d604d1..8396fc2dc1 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java@@ -879,7 +879,7 @@ public class CloudFormationService {* <p>On a <b>create</b>, rolls back by deleting resources created by the failed execution. On* an <b>update</b>, restores the prior resource, template, output, and export state.*/- private void rollbackFailedExecution(+ void rollbackFailedExecution(Stack stack,String region,boolean isCreate,@@ -1267,10 +1267,14 @@ public class CloudFormationService {resource.getResourceType(), "DELETE_IN_PROGRESS", null);try {deleteResourcePhysically(resource, region);- resource.setStatus("DELETE_COMPLETE");- addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),- resource.getResourceType(), "DELETE_COMPLETE", null);+ completeResourceDeletion(stack, resource);} catch (Exception e) {+ if (isAlreadyDeleted(e)) {+ completeResourceDeletion(stack, resource);+ LOG.debugv("Resource {0} ({1}) was already deleted while rolling back stack {2}",+ resource.getResourceType(), resource.getPhysicalId(), stack.getStackName());+ continue;+ }failedResources.add(resource.getLogicalId());resource.setStatus("DELETE_FAILED");resource.setStatusReason(e.getMessage());@@ -1369,10 +1373,14 @@ public class CloudFormationService {resource.getResourceType(), "DELETE_IN_PROGRESS", null);try {deleteResourcePhysically(resource, region);- resource.setStatus("DELETE_COMPLETE");- addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),- resource.getResourceType(), "DELETE_COMPLETE", null);+ completeResourceDeletion(stack, resource);} catch (Exception e) {+ if (isAlreadyDeleted(e)) {+ completeResourceDeletion(stack, resource);+ LOG.debugv("Resource {0} ({1}) was already deleted while deleting stack {2}",+ resource.getResourceType(), resource.getPhysicalId(), stack.getStackName());+ continue;+ }// AWS leaves the stack in DELETE_FAILED when a managed resource cannot be// deleted (e.g. a non-empty S3 bucket raises BucketNotEmpty). The stack must// not be reported as a successful deletion while the resource still exists.@@ -1443,6 +1451,27 @@ public class CloudFormationService {return true;}+ private void completeResourceDeletion(Stack stack, StackResource resource) {+ resource.setStatus("DELETE_COMPLETE");+ resource.setStatusReason(null);+ addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),+ resource.getResourceType(), "DELETE_COMPLETE", null);+ }++ /** Returns whether a resource deletion failed solely because the resource is already gone. */+ private static boolean isAlreadyDeleted(Throwable failure) {+ Set<Throwable> seen = Collections.newSetFromMap(new IdentityHashMap<>());+ for (Throwable current = failure; current != null && seen.add(current); current = current.getCause()) {+ if (current instanceof AwsException awsException+ && (awsException.getHttpStatus() == 404+ || (awsException.getErrorCode() != null+ && awsException.getErrorCode().endsWith("NotFoundException")))) {+ return true;+ }+ }+ return false;+ }+private Map<String, Boolean> resolveConditions(JsonNode template, Map<String, String> params,Stack stack, String region, String accountId) {Map<String, Boolean> conditions = new HashMap<>();diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.javaindex f3804e23c0..7119dd11cb 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java@@ -1797,6 +1797,106 @@ class CloudFormationIntegrationTest {throw new AssertionError("Stack did not reach DELETE_COMPLETE within timeout");}+ // Regression: issue #1966. A resource removed outside CloudFormation must be treated as+ // already deleted, even when its provisioner does not have a resource-specific safe delete.+ @Test+ void deleteStack_withAlreadyDeletedApiGatewayRestApi_reachesDeleteComplete() throws Exception {+ String stackName = "cfn-1966-missing-rest-api";+ String template = """+ {+ "Resources": {+ "RestApi": {+ "Type": "AWS::ApiGateway::RestApi",+ "Properties": {+ "Name": "cfn-1966-missing-rest-api"+ }+ }+ }+ }+ """;++ String createResponse = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateStack")+ .formParam("StackName", stackName)+ .formParam("TemplateBody", template)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<StackId>"))+ .extract().asString();+ String stackArn = createResponse.substring(+ createResponse.indexOf("<StackId>") + "<StackId>".length(), createResponse.indexOf("</StackId>"));++ boolean created = false;+ long createDeadline = System.currentTimeMillis() + 10_000;+ while (System.currentTimeMillis() < createDeadline) {+ String statusXml = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", stackArn)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().asString();+ if (statusXml.contains("<StackStatus>CREATE_COMPLETE</StackStatus>")) {+ created = true;+ break;+ }+ Thread.sleep(200);+ }+ assertThat("Stack did not reach CREATE_COMPLETE within timeout", created, equalTo(true));++ String resourcesXml = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStackResources")+ .formParam("StackName", stackArn)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().asString();+ String apiId = physicalIdByLogicalId(resourcesXml, "RestApi");++ // Simulate state loss or an out-of-band delete. API Gateway reports NotFoundException (404)+ // when CloudFormation later tries to delete the tracked physical ID.+ given()+ .when()+ .delete("/restapis/" + apiId)+ .then()+ .statusCode(202);++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteStack")+ .formParam("StackName", stackName)+ .when()+ .post("/")+ .then()+ .statusCode(200);++ long deleteDeadline = System.currentTimeMillis() + 10_000;+ while (System.currentTimeMillis() < deleteDeadline) {+ String statusXml = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", stackArn)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().asString();+ if (statusXml.contains("<StackStatus>DELETE_COMPLETE</StackStatus>")) {+ return;+ }+ assertThat(statusXml, not(containsString("<StackStatus>DELETE_FAILED</StackStatus>")));+ Thread.sleep(200);+ }+ throw new AssertionError("Stack did not reach DELETE_COMPLETE within timeout");+ }+@Testvoid describeDeletedStack_byArn_expiresAfterRetentionWindow() throws Exception {String template = """diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.javanew file mode 100644index 0000000000..df3b1cc453--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java@@ -0,0 +1,126 @@+package io.github.hectorvent.floci.services.cloudformation;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.cloudformation.model.Stack;+import io.github.hectorvent.floci.services.cloudformation.model.StackResource;+import io.github.hectorvent.floci.services.s3.S3Service;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.time.Clock;+import java.util.Map;+import java.util.Set;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * Covers rollback cleanup when another actor removes a resource after its create succeeded.+ */+class CloudFormationServiceRollbackTest {++ private static final String ACCOUNT = "000000000000";+ private static final String REGION = "us-east-1";++ private CloudFormationResourceProvisioner provisioner;+ private CloudFormationService service;++ @BeforeEach+ void setUp() {+ provisioner = mock(CloudFormationResourceProvisioner.class);+ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.defaultAccountId()).thenReturn(ACCOUNT);++ service = new CloudFormationService(+ provisioner,+ mock(S3Service.class),+ new ObjectMapper(),+ config,+ mock(RegionResolver.class),+ Clock.systemUTC(),+ new InMemoryStorageFactory());+ }++ @Test+ void createRollback_withAlreadyDeletedResource_reachesRollbackComplete() {+ Stack stack = new Stack();+ stack.setStackName("rollback-missing-resource");+ stack.setStackId("stack-id");+ stack.setRegion(REGION);++ StackResource created = resource("RestApi", "api-id", "AWS::ApiGateway::RestApi", "CREATE_COMPLETE");+ StackResource failed = resource("FailingResource", null, "AWS::Test::Failure", "CREATE_FAILED");+ failed.setStatusReason("simulated create failure");+ stack.getResources().put(created.getLogicalId(), created);+ stack.getResources().put(failed.getLogicalId(), failed);++ doThrow(new AwsException("NotFoundException", "Invalid API id specified", 404))+ .when(provisioner).delete(eq(created), eq(REGION));++ service.rollbackFailedExecution(stack, REGION, true, failed, null, Set.of());++ assertEquals("ROLLBACK_COMPLETE", stack.getStatus());+ assertEquals("DELETE_COMPLETE", created.getStatus());+ assertNull(created.getStatusReason());+ verify(provisioner).delete(created, REGION);+ }++ @Test+ void createRollback_withDependencyNotFoundMessage_reachesRollbackFailed() {+ Stack stack = new Stack();+ stack.setStackName("rollback-delete-failure");+ stack.setStackId("stack-id");+ stack.setRegion(REGION);++ StackResource created = resource("ListenerRule", "rule-id", "AWS::ElasticLoadBalancingV2::ListenerRule",+ "CREATE_COMPLETE");+ StackResource failed = resource("FailingResource", null, "AWS::Test::Failure", "CREATE_FAILED");+ failed.setStatusReason("simulated create failure");+ stack.getResources().put(created.getLogicalId(), created);+ stack.getResources().put(failed.getLogicalId(), failed);++ doThrow(new IllegalStateException("Cannot delete listener rule: target group floci-tg-1 not found"))+ .when(provisioner).delete(eq(created), eq(REGION));++ service.rollbackFailedExecution(stack, REGION, true, failed, null, Set.of());++ assertEquals("ROLLBACK_FAILED", stack.getStatus());+ assertEquals("DELETE_FAILED", created.getStatus());+ assertEquals(+ "Cannot delete listener rule: target group floci-tg-1 not found",+ created.getStatusReason());+ verify(provisioner).delete(created, REGION);+ }++ private static StackResource resource(String logicalId, String physicalId, String resourceType, String status) {+ StackResource resource = new StackResource();+ resource.setLogicalId(logicalId);+ resource.setPhysicalId(physicalId);+ resource.setResourceType(resourceType);+ resource.setStatus(status);+ return resource;+ }++ private static final class InMemoryStorageFactory extends StorageFactory {+ private InMemoryStorageFactory() {+ super(null, null);+ }++ @Override+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return AccountAwareStorageBackend.inMemory(ACCOUNT);+ }+ }+}