Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudformation): bound operation executor (#3761)

Niels Kootstra91de9e452e64
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
index 6415f8cb21..cfcb72271b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
@@ -37,10 +37,12 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import io.github.hectorvent.floci.core.resource.ExplorerResource;
import io.github.hectorvent.floci.core.resource.ResourceProvider;
import io.github.hectorvent.floci.core.resource.SupportedResourceType;
@@ -57,11 +59,21 @@ public class CloudFormationService implements ResourceProvider {
private static final Logger LOG = Logger.getLogger(CloudFormationService.class);
+ private static final int MAX_OPERATION_THREADS = 16;
+ private static final int MAX_QUEUED_OPERATIONS = 128;
+
private final ConcurrentHashMap<String, Stack> stacks = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, DeletedStackEntry> deletedStacks = new ConcurrentHashMap<>();
// Account-scoped exports registry: account:region:exportName -> exportValue
private final ConcurrentHashMap<String, String> exports = new ConcurrentHashMap<>();
- private final ExecutorService executor = Executors.newCachedThreadPool();
+ private final ExecutorService executor = newOperationExecutor();
+
+ static ThreadPoolExecutor newOperationExecutor() {
+ return new ThreadPoolExecutor(
+ MAX_OPERATION_THREADS, MAX_OPERATION_THREADS, 0L, TimeUnit.MILLISECONDS,
+ new ArrayBlockingQueue<>(MAX_QUEUED_OPERATIONS),
+ new ThreadPoolExecutor.AbortPolicy());
+ }
private final CloudFormationResourceProvisioner provisioner;
private final S3Service s3Service;
@@ -643,6 +655,13 @@ public class CloudFormationService implements ResourceProvider {
private record ClaimedExecution(ChangeSet changeSet, boolean isCreate) {}
+ private record ChangeSetState(String status, String executionStatus) {}
+
+ private record StackMutationSnapshot(String status, Instant lastUpdatedTime,
+ List<StackEvent> events,
+ Map<String, ChangeSet> changeSets,
+ Map<ChangeSet, ChangeSetState> changeSetStates) {}
+
// compute() holds the stack's per-key lock for the whole claim, so only one racing execution can win.
private Future<?> claimAndSubmitExecution(String stackNameOrArn, String changeSetName, String region,
String accountId, boolean requireAvailable) {
@@ -650,6 +669,7 @@ public class CloudFormationService implements ResourceProvider {
String resolvedChangeSetName = resolveChangeSetName(changeSetName, region, accountId);
ClaimedExecution[] claimed = new ClaimedExecution[1];
+ StackMutationSnapshot[] snapshot = new StackMutationSnapshot[1];
Stack stack = stacks.compute(stackKey(accountId, canonicalStackName, region), (k, existing) -> {
if (existing == null) {
throw new AwsException("ValidationError",
@@ -667,6 +687,7 @@ public class CloudFormationService implements ResourceProvider {
if (!eligible) {
throw invalidChangeSetStatus(cs);
}
+ snapshot[0] = snapshot(existing);
boolean isCreate = "CREATE".equalsIgnoreCase(cs.getChangeSetType()) ||
"CREATE_IN_PROGRESS".equals(existing.getStatus());
cs.setExecutionStatus("EXECUTE_IN_PROGRESS");
@@ -687,9 +708,18 @@ public class CloudFormationService implements ResourceProvider {
claimed[0] = new ClaimedExecution(cs, isCreate);
return existing;
});
- persistStack(stack);
-
- return submitExecution(stack, claimed[0].changeSet(), claimed[0].isCreate(), region, accountId);
+ try {
+ persistStack(stack);
+ return submitExecution(stack, claimed[0].changeSet(), claimed[0].isCreate(), region, accountId);
+ } catch (AwsException e) {
+ if ("LimitExceededException".equals(e.getErrorCode())) {
+ Stack restored = restoreStack(accountId, canonicalStackName, region, snapshot[0]);
+ if (restored != null) {
+ persistStack(restored);
+ }
+ }
+ throw e;
+ }
}
private AwsException invalidChangeSetStatus(ChangeSet cs) {
@@ -704,11 +734,11 @@ public class CloudFormationService implements ResourceProvider {
String templateBody = cs.getTemplateBody();
Map<String, String> params = cs.getParameters() != null ? cs.getParameters() : Map.of();
- return executor.submit(() -> runUnderAccount(accountId, () -> {
+ return submitOperation(() -> runUnderAccount(accountId, () -> {
executeTemplate(stack, templateBody, params, isCreate, region, accountId);
String status = stack.getStatus();
cs.setExecutionStatus(status != null && (status.contains("ROLLBACK") || status.endsWith("_FAILED"))
- ? "EXECUTE_FAILED" : "EXECUTE_COMPLETE");
+ ? "EXECUTE_FAILED" : "EXECUTE_COMPLETE");
persistStack(stack);
}));
}
@@ -796,11 +826,69 @@ public class CloudFormationService implements ResourceProvider {
"Stack [" + stack.getStackId()
+ "] cannot be deleted while TerminationProtection is enabled", 400);
}
+ StackMutationSnapshot snapshot = snapshot(stack);
stack.setStatus("DELETE_IN_PROGRESS");
addEvent(stack, stack.getStackName(), stack.getStackId(),
"AWS::CloudFormation::Stack", "DELETE_IN_PROGRESS", null);
- return executor.submit(() -> runUnderAccount(accountId, () -> deleteStackResources(stack, region)));
+ try {
+ return submitOperation(() -> runUnderAccount(accountId,
+ () -> deleteStackResources(stack, region, accountId)));
+ } catch (AwsException e) {
+ if ("LimitExceededException".equals(e.getErrorCode())) {
+ Stack restored = restoreStack(accountId, stack.getStackName(), region, snapshot);
+ if (restored != null) {
+ persistStack(restored);
+ }
+ }
+ throw e;
+ }
+ }
+
+ private StackMutationSnapshot snapshot(Stack stack) {
+ Map<ChangeSet, ChangeSetState> states = new IdentityHashMap<>();
+ for (ChangeSet changeSet : stack.getChangeSets().values()) {
+ states.put(changeSet, new ChangeSetState(changeSet.getStatus(), changeSet.getExecutionStatus()));
+ }
+ return new StackMutationSnapshot(stack.getStatus(), stack.getLastUpdatedTime(),
+ new ArrayList<>(stack.getEvents()), new LinkedHashMap<>(stack.getChangeSets()), states);
+ }
+
+ private void restore(Stack stack, StackMutationSnapshot snapshot) {
+ if (snapshot == null) {
+ return;
+ }
+ stack.setStatus(snapshot.status());
+ stack.setLastUpdatedTime(snapshot.lastUpdatedTime());
+ stack.setEvents(new ArrayList<>(snapshot.events()));
+ stack.setChangeSets(new LinkedHashMap<>(snapshot.changeSets()));
+ snapshot.changeSetStates().forEach((changeSet, state) -> {
+ changeSet.setStatus(state.status());
+ changeSet.setExecutionStatus(state.executionStatus());
+ });
+ }
+
+ private Stack restoreStack(String accountId, String stackName, String region,
+ StackMutationSnapshot snapshot) {
+ return stacks.compute(stackKey(accountId, stackName, region), (key, current) -> {
+ if (current != null) {
+ restore(current, snapshot);
+ }
+ return current;
+ });
+ }
+
+ private Future<?> submitOperation(Runnable operation) {
+ try {
+ return executor.submit(operation);
+ } catch (RejectedExecutionException e) {
+ throw operationLimitExceeded();
+ }
+ }
+
+ static AwsException operationLimitExceeded() {
+ return new AwsException("LimitExceededException",
+ "Too many CloudFormation operations are in progress.", 400);
}
// ── GetTemplate ───────────────────────────────────────────────────────────
@@ -1537,30 +1625,22 @@ public class CloudFormationService implements ResourceProvider {
return false;
}
- private void deleteResourcePhysically(StackResource resource, String region) throws Exception {
+ private void deleteResourcePhysically(StackResource resource, String region, String accountId)
+ throws Exception {
if ("AWS::CloudFormation::Stack".equals(resource.getResourceType())) {
- Future<?> future = deleteStack(resource.getPhysicalId(), region, regionResolver.getAccountId());
- if (future != null) {
- try {
- future.get();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw e;
- } catch (ExecutionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof Exception ex) {
- throw ex;
- }
- throw e;
- }
+ Stack child = resolveStack(resource.getPhysicalId(), region, accountId);
+ if (child == null) {
+ return;
}
- Stack child = resolveStack(resource.getPhysicalId(), region);
- if (child != null && "DELETE_FAILED".equals(child.getStatus())) {
- String reason = child.getStatusReason() != null
- ? child.getStatusReason()
- : "Nested stack deletion failed";
- throw new IllegalStateException(reason);
+ if (child.isEnableTerminationProtection()) {
+ throw new AwsException("ValidationError",
+ "Stack [" + child.getStackId()
+ + "] cannot be deleted while TerminationProtection is enabled", 400);
}
+ child.setStatus("DELETE_IN_PROGRESS");
+ addEvent(child, child.getStackName(), child.getStackId(),
+ "AWS::CloudFormation::Stack", "DELETE_IN_PROGRESS", null);
+ deleteStackResources(child, region, accountId);
} else {
provisioner.delete(resource, region);
}
@@ -1645,7 +1725,7 @@ public class CloudFormationService implements ResourceProvider {
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_IN_PROGRESS",
"Resource creation cancelled during update rollback");
- deleteResourcePhysically(resource, region);
+ deleteResourcePhysically(resource, region, ownerAccount(stack));
}
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_COMPLETE",
@@ -1807,7 +1887,7 @@ public class CloudFormationService implements ResourceProvider {
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_IN_PROGRESS", null);
try {
- deleteResourcePhysically(resource, region);
+ deleteResourcePhysically(resource, region, ownerAccount(stack));
completeResourceDeletion(stack, resource);
} catch (Exception e) {
if (isAlreadyDeleted(e)) {
@@ -1865,7 +1945,7 @@ public class CloudFormationService implements ResourceProvider {
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_IN_PROGRESS", null);
try {
- deleteResourcePhysically(resource, region);
+ deleteResourcePhysically(resource, region, ownerAccount(stack));
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_COMPLETE", null);
stack.getResources().remove(resource.getLogicalId());
@@ -1923,7 +2003,7 @@ public class CloudFormationService implements ResourceProvider {
return ordered;
}
- private void deleteStackResources(Stack stack, String region) {
+ private void deleteStackResources(Stack stack, String region, String accountId) {
try {
List<StackResource> resources = resourcesInCreationOrder(stack, region);
Collections.reverse(resources); // Dependents go before what they depend on
@@ -1955,7 +2035,7 @@ public class CloudFormationService implements ResourceProvider {
addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),
resource.getResourceType(), "DELETE_IN_PROGRESS", null);
try {
- deleteResourcePhysically(resource, region);
+ deleteResourcePhysically(resource, region, accountId);
completeResourceDeletion(stack, resource);
} catch (Exception e) {
if (isAlreadyDeleted(e)) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java
index 84bb3ca9b6..bd7f2ddf86 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java
@@ -6,12 +6,20 @@ import io.restassured.response.Response;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
import static io.restassured.RestAssured.given;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
/**
* Verifies the resource-level {@code DeletionPolicy} attribute (issue #1555): {@code Retain} keeps a
@@ -27,6 +35,50 @@ class CloudFormationDeletionPolicyIntegrationTest {
private static final String CUSTOM_AUTH =
"AWS4-HMAC-SHA256 Credential=111122223333/20260205/eu-west-1/cloudformation/aws4_request";
+ @Test
+ void deletingNestedStacksDoesNotExhaustOperationWorkers() throws InterruptedException {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String childTemplateKey = "child-delete-workers-" + suffix + ".json";
+ given().when().put("/nested-stack-templates").then().statusCode(200);
+ given()
+ .contentType("application/json")
+ .body("{\"Resources\":{}}")
+ .when()
+ .put("/nested-stack-templates/" + childTemplateKey)
+ .then()
+ .statusCode(200);
+
+ List<String> stackIds = new ArrayList<>();
+ List<String> stackNames = new ArrayList<>();
+ String childUrl = "http://localhost/nested-stack-templates/" + childTemplateKey;
+ for (int i = 0; i < 16; i++) {
+ String stackName = "nested-delete-worker-" + suffix + "-" + i;
+ String template = "{\"Resources\":{\"Child\":{\"Type\":\"AWS::CloudFormation::Stack\","
+ + "\"Properties\":{\"TemplateURL\":\"" + childUrl + "\"}}}}";
+ String stackId = createStack(stackName, template);
+ awaitStackStatus(stackId, "CREATE_COMPLETE");
+ stackNames.add(stackName);
+ stackIds.add(stackId);
+ }
+
+ assertTimeoutPreemptively(Duration.ofSeconds(20), () -> {
+ try (ExecutorService clients = Executors.newFixedThreadPool(16)) {
+ List<CompletableFuture<Void>> deletes = stackNames.stream()
+ .map(name -> CompletableFuture.runAsync(() -> deleteStack(name), clients))
+ .toList();
+ CompletableFuture.allOf(deletes.toArray(CompletableFuture[]::new)).join();
+ for (String stackId : stackIds) {
+ try {
+ awaitStackStatus(stackId, "DELETE_COMPLETE");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("Interrupted while waiting for nested stack deletion", e);
+ }
+ }
+ }
+ });
+ }
+
@BeforeAll
static void configureRestAssured() {
RestAssuredJsonUtils.configureAwsContentTypes();
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceExecutorTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceExecutorTest.java
new file mode 100644
index 0000000000..89b13a31b6
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceExecutorTest.java
@@ -0,0 +1,49 @@
+package io.github.hectorvent.floci.services.cloudformation;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class CloudFormationServiceExecutorTest {
+
+ @Test
+ void operationExecutorRejectsWorkAfterThreadsAndQueueAreFull() throws Exception {
+ ThreadPoolExecutor executor = CloudFormationService.newOperationExecutor();
+ CountDownLatch release = new CountDownLatch(1);
+ try {
+ for (int i = 0; i < executor.getMaximumPoolSize(); i++) {
+ executor.execute(() -> await(release));
+ }
+ int queueCapacity = executor.getQueue().remainingCapacity();
+ for (int i = 0; i < queueCapacity; i++) {
+ executor.execute(() -> { });
+ }
+
+ assertThrows(RejectedExecutionException.class, () -> executor.execute(() -> { }));
+ } finally {
+ release.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void rejectedCloudFormationOperationUsesAwsErrorCode() {
+ AwsException exception = CloudFormationService.operationLimitExceeded();
+
+ assertEquals("LimitExceededException", exception.getErrorCode());
+ }
+
+ private static void await(CountDownLatch release) {
+ try {
+ release.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+}