Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudformation): tolerate missing stack resources during deletion (#2039)

Aniket Shukla12b8ddd6291b
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
index 1585d604d1..8396fc2dc1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
@@ -879,7 +879,7 @@ public class CloudFormationService {
* <p>On a <b>create</b>, rolls back by deleting resources created by the failed execution. On
* an <b>update</b>, restores the prior resource, template, output, and export state.
*/
- private void rollbackFailedExecution(
+ void rollbackFailedExecution(
Stack stack,
String region,
boolean isCreate,
@@ -1267,10 +1267,14 @@ public class CloudFormationService {
resource.getResourceType(), "DELETE_IN_PROGRESS", null);
try {
deleteResourcePhysically(resource, region);
- resource.setStatus("DELETE_COMPLETE");
- addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
- resource.getResourceType(), "DELETE_COMPLETE", null);
+ completeResourceDeletion(stack, resource);
} catch (Exception e) {
+ if (isAlreadyDeleted(e)) {
+ completeResourceDeletion(stack, resource);
+ LOG.debugv("Resource {0} ({1}) was already deleted while rolling back stack {2}",
+ resource.getResourceType(), resource.getPhysicalId(), stack.getStackName());
+ continue;
+ }
failedResources.add(resource.getLogicalId());
resource.setStatus("DELETE_FAILED");
resource.setStatusReason(e.getMessage());
@@ -1369,10 +1373,14 @@ public class CloudFormationService {
resource.getResourceType(), "DELETE_IN_PROGRESS", null);
try {
deleteResourcePhysically(resource, region);
- resource.setStatus("DELETE_COMPLETE");
- addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
- resource.getResourceType(), "DELETE_COMPLETE", null);
+ completeResourceDeletion(stack, resource);
} catch (Exception e) {
+ if (isAlreadyDeleted(e)) {
+ completeResourceDeletion(stack, resource);
+ LOG.debugv("Resource {0} ({1}) was already deleted while deleting stack {2}",
+ resource.getResourceType(), resource.getPhysicalId(), stack.getStackName());
+ continue;
+ }
// AWS leaves the stack in DELETE_FAILED when a managed resource cannot be
// deleted (e.g. a non-empty S3 bucket raises BucketNotEmpty). The stack must
// not be reported as a successful deletion while the resource still exists.
@@ -1443,6 +1451,27 @@ public class CloudFormationService {
return true;
}
+ private void completeResourceDeletion(Stack stack, StackResource resource) {
+ resource.setStatus("DELETE_COMPLETE");
+ resource.setStatusReason(null);
+ addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
+ resource.getResourceType(), "DELETE_COMPLETE", null);
+ }
+
+ /** Returns whether a resource deletion failed solely because the resource is already gone. */
+ private static boolean isAlreadyDeleted(Throwable failure) {
+ Set<Throwable> seen = Collections.newSetFromMap(new IdentityHashMap<>());
+ for (Throwable current = failure; current != null && seen.add(current); current = current.getCause()) {
+ if (current instanceof AwsException awsException
+ && (awsException.getHttpStatus() == 404
+ || (awsException.getErrorCode() != null
+ && awsException.getErrorCode().endsWith("NotFoundException")))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private Map<String, Boolean> resolveConditions(JsonNode template, Map<String, String> params,
Stack stack, String region, String accountId) {
Map<String, Boolean> conditions = new HashMap<>();
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java
index f3804e23c0..7119dd11cb 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java
@@ -1797,6 +1797,106 @@ class CloudFormationIntegrationTest {
throw new AssertionError("Stack did not reach DELETE_COMPLETE within timeout");
}
+ // Regression: issue #1966. A resource removed outside CloudFormation must be treated as
+ // already deleted, even when its provisioner does not have a resource-specific safe delete.
+ @Test
+ void deleteStack_withAlreadyDeletedApiGatewayRestApi_reachesDeleteComplete() throws Exception {
+ String stackName = "cfn-1966-missing-rest-api";
+ String template = """
+ {
+ "Resources": {
+ "RestApi": {
+ "Type": "AWS::ApiGateway::RestApi",
+ "Properties": {
+ "Name": "cfn-1966-missing-rest-api"
+ }
+ }
+ }
+ }
+ """;
+
+ String createResponse = given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "CreateStack")
+ .formParam("StackName", stackName)
+ .formParam("TemplateBody", template)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("<StackId>"))
+ .extract().asString();
+ String stackArn = createResponse.substring(
+ createResponse.indexOf("<StackId>") + "<StackId>".length(), createResponse.indexOf("</StackId>"));
+
+ boolean created = false;
+ long createDeadline = System.currentTimeMillis() + 10_000;
+ while (System.currentTimeMillis() < createDeadline) {
+ String statusXml = given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeStacks")
+ .formParam("StackName", stackArn)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().asString();
+ if (statusXml.contains("<StackStatus>CREATE_COMPLETE</StackStatus>")) {
+ created = true;
+ break;
+ }
+ Thread.sleep(200);
+ }
+ assertThat("Stack did not reach CREATE_COMPLETE within timeout", created, equalTo(true));
+
+ String resourcesXml = given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeStackResources")
+ .formParam("StackName", stackArn)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().asString();
+ String apiId = physicalIdByLogicalId(resourcesXml, "RestApi");
+
+ // Simulate state loss or an out-of-band delete. API Gateway reports NotFoundException (404)
+ // when CloudFormation later tries to delete the tracked physical ID.
+ given()
+ .when()
+ .delete("/restapis/" + apiId)
+ .then()
+ .statusCode(202);
+
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DeleteStack")
+ .formParam("StackName", stackName)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ long deleteDeadline = System.currentTimeMillis() + 10_000;
+ while (System.currentTimeMillis() < deleteDeadline) {
+ String statusXml = given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeStacks")
+ .formParam("StackName", stackArn)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().asString();
+ if (statusXml.contains("<StackStatus>DELETE_COMPLETE</StackStatus>")) {
+ return;
+ }
+ assertThat(statusXml, not(containsString("<StackStatus>DELETE_FAILED</StackStatus>")));
+ Thread.sleep(200);
+ }
+ throw new AssertionError("Stack did not reach DELETE_COMPLETE within timeout");
+ }
+
@Test
void describeDeletedStack_byArn_expiresAfterRetentionWindow() throws Exception {
String template = """
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java
new file mode 100644
index 0000000000..df3b1cc453
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceRollbackTest.java
@@ -0,0 +1,126 @@
+package io.github.hectorvent.floci.services.cloudformation;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.cloudformation.model.Stack;
+import io.github.hectorvent.floci.services.cloudformation.model.StackResource;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Clock;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Covers rollback cleanup when another actor removes a resource after its create succeeded.
+ */
+class CloudFormationServiceRollbackTest {
+
+ private static final String ACCOUNT = "000000000000";
+ private static final String REGION = "us-east-1";
+
+ private CloudFormationResourceProvisioner provisioner;
+ private CloudFormationService service;
+
+ @BeforeEach
+ void setUp() {
+ provisioner = mock(CloudFormationResourceProvisioner.class);
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ when(config.defaultAccountId()).thenReturn(ACCOUNT);
+
+ service = new CloudFormationService(
+ provisioner,
+ mock(S3Service.class),
+ new ObjectMapper(),
+ config,
+ mock(RegionResolver.class),
+ Clock.systemUTC(),
+ new InMemoryStorageFactory());
+ }
+
+ @Test
+ void createRollback_withAlreadyDeletedResource_reachesRollbackComplete() {
+ Stack stack = new Stack();
+ stack.setStackName("rollback-missing-resource");
+ stack.setStackId("stack-id");
+ stack.setRegion(REGION);
+
+ StackResource created = resource("RestApi", "api-id", "AWS::ApiGateway::RestApi", "CREATE_COMPLETE");
+ StackResource failed = resource("FailingResource", null, "AWS::Test::Failure", "CREATE_FAILED");
+ failed.setStatusReason("simulated create failure");
+ stack.getResources().put(created.getLogicalId(), created);
+ stack.getResources().put(failed.getLogicalId(), failed);
+
+ doThrow(new AwsException("NotFoundException", "Invalid API id specified", 404))
+ .when(provisioner).delete(eq(created), eq(REGION));
+
+ service.rollbackFailedExecution(stack, REGION, true, failed, null, Set.of());
+
+ assertEquals("ROLLBACK_COMPLETE", stack.getStatus());
+ assertEquals("DELETE_COMPLETE", created.getStatus());
+ assertNull(created.getStatusReason());
+ verify(provisioner).delete(created, REGION);
+ }
+
+ @Test
+ void createRollback_withDependencyNotFoundMessage_reachesRollbackFailed() {
+ Stack stack = new Stack();
+ stack.setStackName("rollback-delete-failure");
+ stack.setStackId("stack-id");
+ stack.setRegion(REGION);
+
+ StackResource created = resource("ListenerRule", "rule-id", "AWS::ElasticLoadBalancingV2::ListenerRule",
+ "CREATE_COMPLETE");
+ StackResource failed = resource("FailingResource", null, "AWS::Test::Failure", "CREATE_FAILED");
+ failed.setStatusReason("simulated create failure");
+ stack.getResources().put(created.getLogicalId(), created);
+ stack.getResources().put(failed.getLogicalId(), failed);
+
+ doThrow(new IllegalStateException("Cannot delete listener rule: target group floci-tg-1 not found"))
+ .when(provisioner).delete(eq(created), eq(REGION));
+
+ service.rollbackFailedExecution(stack, REGION, true, failed, null, Set.of());
+
+ assertEquals("ROLLBACK_FAILED", stack.getStatus());
+ assertEquals("DELETE_FAILED", created.getStatus());
+ assertEquals(
+ "Cannot delete listener rule: target group floci-tg-1 not found",
+ created.getStatusReason());
+ verify(provisioner).delete(created, REGION);
+ }
+
+ private static StackResource resource(String logicalId, String physicalId, String resourceType, String status) {
+ StackResource resource = new StackResource();
+ resource.setLogicalId(logicalId);
+ resource.setPhysicalId(physicalId);
+ resource.setResourceType(resourceType);
+ resource.setStatus(status);
+ return resource;
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return AccountAwareStorageBackend.inMemory(ACCOUNT);
+ }
+ }
+}