Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudformation): claim change set atomically before execution (#3564)

Niels Kootstra03d33350a7da
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
index c86fb2e2f0..6ed211d358 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java
@@ -620,19 +620,7 @@ public class CloudFormationService implements ResourceProvider {
* that failure surfaces on those paths, and floci must still expose it there.
*/
public Future<?> executeChangeSetForRequest(String stackName, String changeSetName, String region) {
- Stack stack = getStackOrThrow(stackName, region, regionResolver.getAccountId());
- ChangeSet cs = stack.getChangeSets().get(resolveChangeSetName(
- changeSetName, region, regionResolver.getAccountId()));
- if (cs == null) {
- throw new AwsException("ChangeSetNotFoundException",
- "ChangeSet [" + changeSetName + "] does not exist", 400);
- }
- if (!"AVAILABLE".equals(cs.getExecutionStatus())) {
- throw new AwsException("InvalidChangeSetStatus",
- "ChangeSet [" + cs.getChangeSetId() + "] cannot be executed in its current "
- + "status of [" + cs.getStatus() + "]", 400);
- }
- return executeChangeSet(stackName, changeSetName, region, regionResolver.getAccountId());
+ return claimAndSubmitExecution(stackName, changeSetName, region, regionResolver.getAccountId(), true);
}
/**
@@ -643,33 +631,86 @@ public class CloudFormationService implements ResourceProvider {
* are materialized under a synthetic request scope bound to {@code accountId} so a single-stack
* deployment lands in the caller's account, and a StackSet instance lands in its target account.
*
- * <p>Unlike {@link #executeChangeSetForRequest}, this does not refuse a non-{@code AVAILABLE}
- * change set: {@code CreateStack}/{@code UpdateStack} call this directly right after creating
- * their own change set, and must still reach {@code CREATE_FAILED} (or its update equivalent)
- * when that change set failed, for example from a failed SAM transform.
+ * <p>Unlike {@link #executeChangeSetForRequest}, this does not require the change set to be
+ * {@code AVAILABLE}: {@code CreateStack}/{@code UpdateStack} call this directly right after
+ * creating their own change set, and must still reach {@code CREATE_FAILED} (or its update
+ * equivalent) when that change set failed, for example from a failed SAM transform. It still
+ * refuses one that is already executing or has already executed.
*/
public Future<?> executeChangeSet(String stackName, String changeSetName, String region, String accountId) {
- Stack stack = getStackOrThrow(stackName, region, accountId);
- ChangeSet cs = stack.getChangeSets().get(resolveChangeSetName(changeSetName, region, accountId));
- if (cs == null) {
- throw new AwsException("ChangeSetNotFoundException",
- "ChangeSet [" + changeSetName + "] does not exist", 400);
- }
+ return claimAndSubmitExecution(stackName, changeSetName, region, accountId, false);
+ }
- boolean isCreate = "CREATE".equalsIgnoreCase(cs.getChangeSetType()) ||
- "CREATE_IN_PROGRESS".equals(stack.getStatus());
+ private record ClaimedExecution(ChangeSet changeSet, boolean isCreate) {}
- stack.setStatus(isCreate ? "CREATE_IN_PROGRESS" : "UPDATE_IN_PROGRESS");
- stack.setLastUpdatedTime(now());
- addEvent(stack, stack.getStackName(), stack.getStackId(),
- "AWS::CloudFormation::Stack", isCreate ? "CREATE_IN_PROGRESS" : "UPDATE_IN_PROGRESS", null);
+ // compute() holds the stack's per-key lock for the whole claim, so only one racing execution can win.
+ private Future<?> claimAndSubmitExecution(String stackNameOrArn, String changeSetName, String region,
+ String accountId, boolean requireAvailable) {
+ String canonicalStackName = getStackOrThrow(stackNameOrArn, region, accountId).getStackName();
+ String resolvedChangeSetName = resolveChangeSetName(changeSetName, region, accountId);
+
+ ClaimedExecution[] claimed = new ClaimedExecution[1];
+ Stack stack = stacks.compute(stackKey(accountId, canonicalStackName, region), (k, existing) -> {
+ if (existing == null) {
+ throw new AwsException("ValidationError",
+ "Stack with id " + stackNameOrArn + " does not exist", 400);
+ }
+ ChangeSet cs = existing.getChangeSets().get(resolvedChangeSetName);
+ if (cs == null) {
+ throw new AwsException("ChangeSetNotFoundException",
+ "ChangeSet [" + changeSetName + "] does not exist", 400);
+ }
+ String executionStatus = cs.getExecutionStatus();
+ boolean eligible = requireAvailable
+ ? "AVAILABLE".equals(executionStatus)
+ : executionStatus == null || !executionStatus.startsWith("EXECUTE_");
+ if (!eligible) {
+ throw invalidChangeSetStatus(cs);
+ }
+ boolean isCreate = "CREATE".equalsIgnoreCase(cs.getChangeSetType()) ||
+ "CREATE_IN_PROGRESS".equals(existing.getStatus());
+ cs.setExecutionStatus("EXECUTE_IN_PROGRESS");
+ if (requireAvailable) {
+ // CloudFormation deletes every other change set on the stack once one of them executes.
+ existing.getChangeSets().values().removeIf(other -> other != cs);
+ } else {
+ for (ChangeSet other : existing.getChangeSets().values()) {
+ if (other != cs && "AVAILABLE".equals(other.getExecutionStatus())) {
+ other.setExecutionStatus("OBSOLETE");
+ }
+ }
+ }
+ existing.setStatus(isCreate ? "CREATE_IN_PROGRESS" : "UPDATE_IN_PROGRESS");
+ existing.setLastUpdatedTime(now());
+ addEvent(existing, existing.getStackName(), existing.getStackId(),
+ "AWS::CloudFormation::Stack", isCreate ? "CREATE_IN_PROGRESS" : "UPDATE_IN_PROGRESS", null);
+ claimed[0] = new ClaimedExecution(cs, isCreate);
+ return existing;
+ });
persistStack(stack);
+ return submitExecution(stack, claimed[0].changeSet(), claimed[0].isCreate(), region, accountId);
+ }
+
+ private AwsException invalidChangeSetStatus(ChangeSet cs) {
+ String detail = "FAILED".equals(cs.getStatus())
+ ? "status of [" + cs.getStatus() + "]"
+ : "execution status of [" + cs.getExecutionStatus() + "]";
+ return new AwsException("InvalidChangeSetStatus",
+ "ChangeSet [" + cs.getChangeSetId() + "] cannot be executed in its current " + detail, 400);
+ }
+
+ private Future<?> submitExecution(Stack stack, ChangeSet cs, boolean isCreate, String region, String accountId) {
String templateBody = cs.getTemplateBody();
Map<String, String> params = cs.getParameters() != null ? cs.getParameters() : Map.of();
- return executor.submit(() -> runUnderAccount(accountId,
- () -> executeTemplate(stack, templateBody, params, isCreate, region, accountId)));
+ return executor.submit(() -> runUnderAccount(accountId, () -> {
+ executeTemplate(stack, templateBody, params, isCreate, region, accountId);
+ String status = stack.getStatus();
+ cs.setExecutionStatus(status != null && (status.contains("ROLLBACK") || status.endsWith("_FAILED"))
+ ? "EXECUTE_FAILED" : "EXECUTE_COMPLETE");
+ persistStack(stack);
+ }));
}
/**
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationExecuteChangeSetConcurrencyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationExecuteChangeSetConcurrencyIntegrationTest.java
new file mode 100644
index 0000000000..87043f7b4d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationExecuteChangeSetConcurrencyIntegrationTest.java
@@ -0,0 +1,307 @@
+package io.github.hectorvent.floci.services.cloudformation;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@QuarkusTest
+class CloudFormationExecuteChangeSetConcurrencyIntegrationTest {
+
+ private final List<String> stacksToDelete = new ArrayList<>();
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @AfterEach
+ void deleteStacks() {
+ for (String stackName : stacksToDelete) {
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DeleteStack")
+ .formParam("StackName", stackName)
+ .when()
+ .post("/");
+ }
+ stacksToDelete.clear();
+ }
+
+ @Test
+ void executeChangeSet_concurrentCalls_onlyOneRunsAndProvisionsResources() throws Exception {
+ String stackName = uniqueStackName("race");
+ createChangeSet(stackName, "initial", "CREATE", topicTemplate("Topic"))
+ .then().statusCode(200).body(containsString("<Id>"));
+
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicReference<Response> response1 = new AtomicReference<>();
+ AtomicReference<Response> response2 = new AtomicReference<>();
+ AtomicReference<Throwable> unexpected = new AtomicReference<>();
+
+ Thread caller1 = new Thread(() -> race(start, () -> response1.set(executeChangeSet(stackName, "initial")), unexpected));
+ Thread caller2 = new Thread(() -> race(start, () -> response2.set(executeChangeSet(stackName, "initial")), unexpected));
+ caller1.start();
+ caller2.start();
+ start.countDown();
+ caller1.join(TimeUnit.SECONDS.toMillis(10));
+ caller2.join(TimeUnit.SECONDS.toMillis(10));
+ assertFalse(caller1.isAlive() || caller2.isAlive(), "a thread never finished");
+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());
+
+ List<Response> responses = List.of(response1.get(), response2.get());
+ assertEquals(1, responses.stream().filter(r -> r.getStatusCode() == 200).count(),
+ () -> "expected exactly one call accepted: " + bodies(responses));
+ assertEquals(1, responses.stream().filter(r -> r.getStatusCode() == 400).count(),
+ () -> "expected exactly one call rejected: " + bodies(responses));
+
+ String rejectedBody = responses.get(0).getStatusCode() == 400
+ ? responses.get(0).asString() : responses.get(1).asString();
+ assertThat(rejectedBody, containsString("<Code>InvalidChangeSetStatus</Code>"));
+ assertTrue(rejectedBody.contains("execution status of [EXECUTE_IN_PROGRESS]")
+ || rejectedBody.contains("execution status of [EXECUTE_COMPLETE]"), rejectedBody);
+
+ waitForStackStatus(stackName, "CREATE_COMPLETE");
+ assertEquals(1, countTopicsWithPrefix(stackName + "-Topic-"), "expected exactly one topic provisioned");
+ }
+
+ @Test
+ void executeChangeSet_afterCompletion_rejectsSecondExecution() {
+ String stackName = uniqueStackName("reexec");
+ createChangeSet(stackName, "initial", "CREATE", topicTemplate("Topic")).then().statusCode(200);
+ executeChangeSet(stackName, "initial").then().statusCode(200);
+ waitForStackStatus(stackName, "CREATE_COMPLETE");
+
+ assertEquals("EXECUTE_COMPLETE", describeChangeSetExecutionStatus(stackName, "initial"));
+
+ Response second = executeChangeSet(stackName, "initial");
+ second.then().statusCode(400).body(containsString("<Code>InvalidChangeSetStatus</Code>"));
+ assertThat(second.asString(), containsString("execution status of [EXECUTE_COMPLETE]"));
+
+ assertEquals(1, countTopicsWithPrefix(stackName + "-Topic-"));
+ }
+
+ @Test
+ void executeChangeSet_deletesOtherChangeSetsOnTheStack() {
+ String stackName = uniqueStackName("delete-siblings");
+ createChangeSet(stackName, "initial", "CREATE", topicTemplate("Base")).then().statusCode(200);
+ executeChangeSet(stackName, "initial").then().statusCode(200);
+ waitForStackStatus(stackName, "CREATE_COMPLETE");
+
+ createChangeSet(stackName, "update-a", "UPDATE", twoTopicTemplate("Base", "TopicA")).then().statusCode(200);
+ createChangeSet(stackName, "update-b", "UPDATE", twoTopicTemplate("Base", "TopicB")).then().statusCode(200);
+
+ executeChangeSet(stackName, "update-a").then().statusCode(200);
+ waitForStackStatus(stackName, "UPDATE_COMPLETE");
+
+ describeChangeSet(stackName, "update-b").then().statusCode(400)
+ .body(containsString("<Code>ChangeSetNotFoundException</Code>"));
+ executeChangeSet(stackName, "update-b").then().statusCode(400)
+ .body(containsString("<Code>ChangeSetNotFoundException</Code>"));
+
+ assertEquals("EXECUTE_COMPLETE", describeChangeSetExecutionStatus(stackName, "update-a"));
+
+ assertEquals(1, countTopicsWithPrefix(stackName + "-TopicA-"));
+ assertEquals(0, countTopicsWithPrefix(stackName + "-TopicB-"));
+ }
+
+ @Test
+ void executeChangeSet_afterCreateStack_rejectsInitialCreateChangeSet() {
+ String stackName = uniqueStackName("internal");
+ createStack(stackName, topicTemplate("Topic"));
+
+ Response rejected = executeChangeSet(stackName, "initial-create");
+ rejected.then().statusCode(400).body(containsString("<Code>InvalidChangeSetStatus</Code>"));
+ assertThat(rejected.asString(), containsString("execution status of [EXECUTE_"));
+ }
+
+ @Test
+ void updateStack_marksPendingChangeSetsObsolete() {
+ String stackName = uniqueStackName("obsolete");
+ createStack(stackName, topicTemplate("Base"));
+ waitForStackStatus(stackName, "CREATE_COMPLETE");
+ createChangeSet(stackName, "pending", "UPDATE", twoTopicTemplate("Base", "TopicA")).then().statusCode(200);
+
+ updateStack(stackName, twoTopicTemplate("Base", "TopicB"));
+ waitForStackStatus(stackName, "UPDATE_COMPLETE");
+
+ assertEquals("OBSOLETE", describeChangeSetExecutionStatus(stackName, "pending"));
+ Response rejected = executeChangeSet(stackName, "pending");
+ rejected.then().statusCode(400).body(containsString("<Code>InvalidChangeSetStatus</Code>"));
+ assertThat(rejected.asString(), containsString("execution status of [OBSOLETE]"));
+ assertEquals(0, countTopicsWithPrefix(stackName + "-TopicA-"));
+ }
+
+ private String uniqueStackName(String prefix) {
+ String name = "cfn-execute-" + prefix + "-" + Long.toString(System.nanoTime(), 36);
+ stacksToDelete.add(name);
+ return name;
+ }
+
+ private static String topicTemplate(String logicalId) {
+ return """
+ Resources:
+ %s:
+ Type: AWS::SNS::Topic
+ """.formatted(logicalId);
+ }
+
+ private static String twoTopicTemplate(String logicalId1, String logicalId2) {
+ return """
+ Resources:
+ %s:
+ Type: AWS::SNS::Topic
+ %s:
+ Type: AWS::SNS::Topic
+ """.formatted(logicalId1, logicalId2);
+ }
+
+ private static Response createChangeSet(String stackName, String changeSetName, String changeSetType, String template) {
+ return given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "CreateChangeSet")
+ .formParam("StackName", stackName)
+ .formParam("ChangeSetName", changeSetName)
+ .formParam("ChangeSetType", changeSetType)
+ .formParam("TemplateBody", template)
+ .when()
+ .post("/");
+ }
+
+ private static void createStack(String stackName, String template) {
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "CreateStack")
+ .formParam("StackName", stackName)
+ .formParam("TemplateBody", template)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
+ private static void updateStack(String stackName, String template) {
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "UpdateStack")
+ .formParam("StackName", stackName)
+ .formParam("TemplateBody", template)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
+ private static Response executeChangeSet(String stackName, String changeSetName) {
+ return given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "ExecuteChangeSet")
+ .formParam("StackName", stackName)
+ .formParam("ChangeSetName", changeSetName)
+ .when()
+ .post("/");
+ }
+
+ private static Response describeChangeSet(String stackName, String changeSetName) {
+ return given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeChangeSet")
+ .formParam("StackName", stackName)
+ .formParam("ChangeSetName", changeSetName)
+ .when()
+ .post("/");
+ }
+
+ private static String describeChangeSetExecutionStatus(String stackName, String changeSetName) {
+ return describeChangeSet(stackName, changeSetName)
+ .then()
+ .statusCode(200)
+ .extract()
+ .xmlPath()
+ .getString("DescribeChangeSetResponse.DescribeChangeSetResult.ExecutionStatus");
+ }
+
+ private static long countTopicsWithPrefix(String prefix) {
+ List<String> arns = given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "ListTopics")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract()
+ .xmlPath()
+ .getList("ListTopicsResponse.ListTopicsResult.Topics.member.TopicArn", String.class);
+ return arns.stream()
+ .filter(arn -> arn.substring(arn.lastIndexOf(':') + 1).startsWith(prefix))
+ .count();
+ }
+
+ private static void race(CountDownLatch start, Runnable action, AtomicReference<Throwable> unexpected) {
+ await(start);
+ try {
+ action.run();
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private static String bodies(List<Response> responses) {
+ return responses.get(0).asString() + " | " + responses.get(1).asString();
+ }
+
+ private static void waitForStackStatus(String stackName, String status) {
+ String expected = "<StackStatus>" + status + "</StackStatus>";
+ String body = "";
+ long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos();
+ while (System.nanoTime() < deadline) {
+ body = given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeStacks")
+ .formParam("StackName", stackName)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract()
+ .asString();
+ if (body.contains(expected)) {
+ return;
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("Interrupted while waiting for stack " + stackName
+ + " to reach " + expected, e);
+ }
+ }
+ assertThat(body, containsString(expected));
+ }
+}