Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): bound operation executor (#3761)
Niels Kootstra91de9e452e64
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.javaindex 6415f8cb21..cfcb72271b 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java@@ -37,10 +37,12 @@ import java.util.regex.Matcher;import java.util.regex.Pattern;import java.util.concurrent.CompletableFuture;import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.ExecutionException;import java.util.concurrent.ExecutorService;-import java.util.concurrent.Executors;import java.util.concurrent.Future;+import java.util.concurrent.ArrayBlockingQueue;+import java.util.concurrent.RejectedExecutionException;+import java.util.concurrent.ThreadPoolExecutor;+import java.util.concurrent.TimeUnit;import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.core.resource.ResourceProvider;import io.github.hectorvent.floci.core.resource.SupportedResourceType;@@ -57,11 +59,21 @@ public class CloudFormationService implements ResourceProvider {private static final Logger LOG = Logger.getLogger(CloudFormationService.class);+ private static final int MAX_OPERATION_THREADS = 16;+ private static final int MAX_QUEUED_OPERATIONS = 128;+private final ConcurrentHashMap<String, Stack> stacks = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, DeletedStackEntry> deletedStacks = new ConcurrentHashMap<>();// Account-scoped exports registry: account:region:exportName -> exportValueprivate final ConcurrentHashMap<String, String> exports = new ConcurrentHashMap<>();- private final ExecutorService executor = Executors.newCachedThreadPool();+ private final ExecutorService executor = newOperationExecutor();++ static ThreadPoolExecutor newOperationExecutor() {+ return new ThreadPoolExecutor(+ MAX_OPERATION_THREADS, MAX_OPERATION_THREADS, 0L, TimeUnit.MILLISECONDS,+ new ArrayBlockingQueue<>(MAX_QUEUED_OPERATIONS),+ new ThreadPoolExecutor.AbortPolicy());+ }private final CloudFormationResourceProvisioner provisioner;private final S3Service s3Service;@@ -643,6 +655,13 @@ public class CloudFormationService implements ResourceProvider {private record ClaimedExecution(ChangeSet changeSet, boolean isCreate) {}+ private record ChangeSetState(String status, String executionStatus) {}++ private record StackMutationSnapshot(String status, Instant lastUpdatedTime,+ List<StackEvent> events,+ Map<String, ChangeSet> changeSets,+ Map<ChangeSet, ChangeSetState> changeSetStates) {}+// compute() holds the stack's per-key lock for the whole claim, so only one racing execution can win.private Future<?> claimAndSubmitExecution(String stackNameOrArn, String changeSetName, String region,String accountId, boolean requireAvailable) {@@ -650,6 +669,7 @@ public class CloudFormationService implements ResourceProvider {String resolvedChangeSetName = resolveChangeSetName(changeSetName, region, accountId);ClaimedExecution[] claimed = new ClaimedExecution[1];+ StackMutationSnapshot[] snapshot = new StackMutationSnapshot[1];Stack stack = stacks.compute(stackKey(accountId, canonicalStackName, region), (k, existing) -> {if (existing == null) {throw new AwsException("ValidationError",@@ -667,6 +687,7 @@ public class CloudFormationService implements ResourceProvider {if (!eligible) {throw invalidChangeSetStatus(cs);}+ snapshot[0] = snapshot(existing);boolean isCreate = "CREATE".equalsIgnoreCase(cs.getChangeSetType()) ||"CREATE_IN_PROGRESS".equals(existing.getStatus());cs.setExecutionStatus("EXECUTE_IN_PROGRESS");@@ -687,9 +708,18 @@ public class CloudFormationService implements ResourceProvider {claimed[0] = new ClaimedExecution(cs, isCreate);return existing;});- persistStack(stack);-- return submitExecution(stack, claimed[0].changeSet(), claimed[0].isCreate(), region, accountId);+ try {+ persistStack(stack);+ return submitExecution(stack, claimed[0].changeSet(), claimed[0].isCreate(), region, accountId);+ } catch (AwsException e) {+ if ("LimitExceededException".equals(e.getErrorCode())) {+ Stack restored = restoreStack(accountId, canonicalStackName, region, snapshot[0]);+ if (restored != null) {+ persistStack(restored);+ }+ }+ throw e;+ }}private AwsException invalidChangeSetStatus(ChangeSet cs) {@@ -704,11 +734,11 @@ public class CloudFormationService implements ResourceProvider {String templateBody = cs.getTemplateBody();Map<String, String> params = cs.getParameters() != null ? cs.getParameters() : Map.of();- return executor.submit(() -> runUnderAccount(accountId, () -> {+ return submitOperation(() -> runUnderAccount(accountId, () -> {executeTemplate(stack, templateBody, params, isCreate, region, accountId);String status = stack.getStatus();cs.setExecutionStatus(status != null && (status.contains("ROLLBACK") || status.endsWith("_FAILED"))- ? "EXECUTE_FAILED" : "EXECUTE_COMPLETE");+ ? "EXECUTE_FAILED" : "EXECUTE_COMPLETE");persistStack(stack);}));}@@ -796,11 +826,69 @@ public class CloudFormationService implements ResourceProvider {"Stack [" + stack.getStackId()+ "] cannot be deleted while TerminationProtection is enabled", 400);}+ StackMutationSnapshot snapshot = snapshot(stack);stack.setStatus("DELETE_IN_PROGRESS");addEvent(stack, stack.getStackName(), stack.getStackId(),"AWS::CloudFormation::Stack", "DELETE_IN_PROGRESS", null);- return executor.submit(() -> runUnderAccount(accountId, () -> deleteStackResources(stack, region)));+ try {+ return submitOperation(() -> runUnderAccount(accountId,+ () -> deleteStackResources(stack, region, accountId)));+ } catch (AwsException e) {+ if ("LimitExceededException".equals(e.getErrorCode())) {+ Stack restored = restoreStack(accountId, stack.getStackName(), region, snapshot);+ if (restored != null) {+ persistStack(restored);+ }+ }+ throw e;+ }+ }++ private StackMutationSnapshot snapshot(Stack stack) {+ Map<ChangeSet, ChangeSetState> states = new IdentityHashMap<>();+ for (ChangeSet changeSet : stack.getChangeSets().values()) {+ states.put(changeSet, new ChangeSetState(changeSet.getStatus(), changeSet.getExecutionStatus()));+ }+ return new StackMutationSnapshot(stack.getStatus(), stack.getLastUpdatedTime(),+ new ArrayList<>(stack.getEvents()), new LinkedHashMap<>(stack.getChangeSets()), states);+ }++ private void restore(Stack stack, StackMutationSnapshot snapshot) {+ if (snapshot == null) {+ return;+ }+ stack.setStatus(snapshot.status());+ stack.setLastUpdatedTime(snapshot.lastUpdatedTime());+ stack.setEvents(new ArrayList<>(snapshot.events()));+ stack.setChangeSets(new LinkedHashMap<>(snapshot.changeSets()));+ snapshot.changeSetStates().forEach((changeSet, state) -> {+ changeSet.setStatus(state.status());+ changeSet.setExecutionStatus(state.executionStatus());+ });+ }++ private Stack restoreStack(String accountId, String stackName, String region,+ StackMutationSnapshot snapshot) {+ return stacks.compute(stackKey(accountId, stackName, region), (key, current) -> {+ if (current != null) {+ restore(current, snapshot);+ }+ return current;+ });+ }++ private Future<?> submitOperation(Runnable operation) {+ try {+ return executor.submit(operation);+ } catch (RejectedExecutionException e) {+ throw operationLimitExceeded();+ }+ }++ static AwsException operationLimitExceeded() {+ return new AwsException("LimitExceededException",+ "Too many CloudFormation operations are in progress.", 400);}// ── GetTemplate ───────────────────────────────────────────────────────────@@ -1537,30 +1625,22 @@ public class CloudFormationService implements ResourceProvider {return false;}- private void deleteResourcePhysically(StackResource resource, String region) throws Exception {+ private void deleteResourcePhysically(StackResource resource, String region, String accountId)+ throws Exception {if ("AWS::CloudFormation::Stack".equals(resource.getResourceType())) {- Future<?> future = deleteStack(resource.getPhysicalId(), region, regionResolver.getAccountId());- if (future != null) {- try {- future.get();- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- throw e;- } catch (ExecutionException e) {- Throwable cause = e.getCause();- if (cause instanceof Exception ex) {- throw ex;- }- throw e;- }+ Stack child = resolveStack(resource.getPhysicalId(), region, accountId);+ if (child == null) {+ return;}- Stack child = resolveStack(resource.getPhysicalId(), region);- if (child != null && "DELETE_FAILED".equals(child.getStatus())) {- String reason = child.getStatusReason() != null- ? child.getStatusReason()- : "Nested stack deletion failed";- throw new IllegalStateException(reason);+ if (child.isEnableTerminationProtection()) {+ throw new AwsException("ValidationError",+ "Stack [" + child.getStackId()+ + "] cannot be deleted while TerminationProtection is enabled", 400);}+ child.setStatus("DELETE_IN_PROGRESS");+ addEvent(child, child.getStackName(), child.getStackId(),+ "AWS::CloudFormation::Stack", "DELETE_IN_PROGRESS", null);+ deleteStackResources(child, region, accountId);} else {provisioner.delete(resource, region);}@@ -1645,7 +1725,7 @@ public class CloudFormationService implements ResourceProvider {addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_IN_PROGRESS","Resource creation cancelled during update rollback");- deleteResourcePhysically(resource, region);+ deleteResourcePhysically(resource, region, ownerAccount(stack));}addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_COMPLETE",@@ -1807,7 +1887,7 @@ public class CloudFormationService implements ResourceProvider {addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_IN_PROGRESS", null);try {- deleteResourcePhysically(resource, region);+ deleteResourcePhysically(resource, region, ownerAccount(stack));completeResourceDeletion(stack, resource);} catch (Exception e) {if (isAlreadyDeleted(e)) {@@ -1865,7 +1945,7 @@ public class CloudFormationService implements ResourceProvider {addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_IN_PROGRESS", null);try {- deleteResourcePhysically(resource, region);+ deleteResourcePhysically(resource, region, ownerAccount(stack));addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_COMPLETE", null);stack.getResources().remove(resource.getLogicalId());@@ -1923,7 +2003,7 @@ public class CloudFormationService implements ResourceProvider {return ordered;}- private void deleteStackResources(Stack stack, String region) {+ private void deleteStackResources(Stack stack, String region, String accountId) {try {List<StackResource> resources = resourcesInCreationOrder(stack, region);Collections.reverse(resources); // Dependents go before what they depend on@@ -1955,7 +2035,7 @@ public class CloudFormationService implements ResourceProvider {addEvent(stack, resource.getLogicalId(), resource.getPhysicalId(),resource.getResourceType(), "DELETE_IN_PROGRESS", null);try {- deleteResourcePhysically(resource, region);+ deleteResourcePhysically(resource, region, accountId);completeResourceDeletion(stack, resource);} catch (Exception e) {if (isAlreadyDeleted(e)) {diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.javaindex 84bb3ca9b6..bd7f2ddf86 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletionPolicyIntegrationTest.java@@ -6,12 +6,20 @@ import io.restassured.response.Response;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.Test;+import java.time.Duration;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import static io.restassured.RestAssured.given;import static org.hamcrest.MatcherAssert.assertThat;import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.not;import static org.junit.jupiter.api.Assertions.fail;+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;/*** Verifies the resource-level {@code DeletionPolicy} attribute (issue #1555): {@code Retain} keeps a@@ -27,6 +35,50 @@ class CloudFormationDeletionPolicyIntegrationTest {private static final String CUSTOM_AUTH ="AWS4-HMAC-SHA256 Credential=111122223333/20260205/eu-west-1/cloudformation/aws4_request";+ @Test+ void deletingNestedStacksDoesNotExhaustOperationWorkers() throws InterruptedException {+ String suffix = Long.toString(System.nanoTime(), 36);+ String childTemplateKey = "child-delete-workers-" + suffix + ".json";+ given().when().put("/nested-stack-templates").then().statusCode(200);+ given()+ .contentType("application/json")+ .body("{\"Resources\":{}}")+ .when()+ .put("/nested-stack-templates/" + childTemplateKey)+ .then()+ .statusCode(200);++ List<String> stackIds = new ArrayList<>();+ List<String> stackNames = new ArrayList<>();+ String childUrl = "http://localhost/nested-stack-templates/" + childTemplateKey;+ for (int i = 0; i < 16; i++) {+ String stackName = "nested-delete-worker-" + suffix + "-" + i;+ String template = "{\"Resources\":{\"Child\":{\"Type\":\"AWS::CloudFormation::Stack\","+ + "\"Properties\":{\"TemplateURL\":\"" + childUrl + "\"}}}}";+ String stackId = createStack(stackName, template);+ awaitStackStatus(stackId, "CREATE_COMPLETE");+ stackNames.add(stackName);+ stackIds.add(stackId);+ }++ assertTimeoutPreemptively(Duration.ofSeconds(20), () -> {+ try (ExecutorService clients = Executors.newFixedThreadPool(16)) {+ List<CompletableFuture<Void>> deletes = stackNames.stream()+ .map(name -> CompletableFuture.runAsync(() -> deleteStack(name), clients))+ .toList();+ CompletableFuture.allOf(deletes.toArray(CompletableFuture[]::new)).join();+ for (String stackId : stackIds) {+ try {+ awaitStackStatus(stackId, "DELETE_COMPLETE");+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError("Interrupted while waiting for nested stack deletion", e);+ }+ }+ }+ });+ }+@BeforeAllstatic void configureRestAssured() {RestAssuredJsonUtils.configureAwsContentTypes();diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceExecutorTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceExecutorTest.javanew file mode 100644index 0000000000..89b13a31b6--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationServiceExecutorTest.java@@ -0,0 +1,49 @@+package io.github.hectorvent.floci.services.cloudformation;++import io.github.hectorvent.floci.core.common.AwsException;+import org.junit.jupiter.api.Test;++import java.util.concurrent.CountDownLatch;+import java.util.concurrent.RejectedExecutionException;+import java.util.concurrent.ThreadPoolExecutor;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;++class CloudFormationServiceExecutorTest {++ @Test+ void operationExecutorRejectsWorkAfterThreadsAndQueueAreFull() throws Exception {+ ThreadPoolExecutor executor = CloudFormationService.newOperationExecutor();+ CountDownLatch release = new CountDownLatch(1);+ try {+ for (int i = 0; i < executor.getMaximumPoolSize(); i++) {+ executor.execute(() -> await(release));+ }+ int queueCapacity = executor.getQueue().remainingCapacity();+ for (int i = 0; i < queueCapacity; i++) {+ executor.execute(() -> { });+ }++ assertThrows(RejectedExecutionException.class, () -> executor.execute(() -> { }));+ } finally {+ release.countDown();+ executor.shutdownNow();+ }+ }++ @Test+ void rejectedCloudFormationOperationUsesAwsErrorCode() {+ AwsException exception = CloudFormationService.operationLimitExceeded();++ assertEquals("LimitExceededException", exception.getErrorCode());+ }++ private static void await(CountDownLatch release) {+ try {+ release.await();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+}