Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): claim change set atomically before execution (#3564)
Niels Kootstra03d33350a7da
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.javaindex c86fb2e2f0..6ed211d358 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java@@ -620,19 +620,7 @@ public class CloudFormationService implements ResourceProvider {* that failure surfaces on those paths, and floci must still expose it there.*/public Future<?> executeChangeSetForRequest(String stackName, String changeSetName, String region) {- Stack stack = getStackOrThrow(stackName, region, regionResolver.getAccountId());- ChangeSet cs = stack.getChangeSets().get(resolveChangeSetName(- changeSetName, region, regionResolver.getAccountId()));- if (cs == null) {- throw new AwsException("ChangeSetNotFoundException",- "ChangeSet [" + changeSetName + "] does not exist", 400);- }- if (!"AVAILABLE".equals(cs.getExecutionStatus())) {- throw new AwsException("InvalidChangeSetStatus",- "ChangeSet [" + cs.getChangeSetId() + "] cannot be executed in its current "- + "status of [" + cs.getStatus() + "]", 400);- }- return executeChangeSet(stackName, changeSetName, region, regionResolver.getAccountId());+ return claimAndSubmitExecution(stackName, changeSetName, region, regionResolver.getAccountId(), true);}/**@@ -643,33 +631,86 @@ public class CloudFormationService implements ResourceProvider {* are materialized under a synthetic request scope bound to {@code accountId} so a single-stack* deployment lands in the caller's account, and a StackSet instance lands in its target account.*- * <p>Unlike {@link #executeChangeSetForRequest}, this does not refuse a non-{@code AVAILABLE}- * change set: {@code CreateStack}/{@code UpdateStack} call this directly right after creating- * their own change set, and must still reach {@code CREATE_FAILED} (or its update equivalent)- * when that change set failed, for example from a failed SAM transform.+ * <p>Unlike {@link #executeChangeSetForRequest}, this does not require the change set to be+ * {@code AVAILABLE}: {@code CreateStack}/{@code UpdateStack} call this directly right after+ * creating their own change set, and must still reach {@code CREATE_FAILED} (or its update+ * equivalent) when that change set failed, for example from a failed SAM transform. It still+ * refuses one that is already executing or has already executed.*/public Future<?> executeChangeSet(String stackName, String changeSetName, String region, String accountId) {- Stack stack = getStackOrThrow(stackName, region, accountId);- ChangeSet cs = stack.getChangeSets().get(resolveChangeSetName(changeSetName, region, accountId));- if (cs == null) {- throw new AwsException("ChangeSetNotFoundException",- "ChangeSet [" + changeSetName + "] does not exist", 400);- }+ return claimAndSubmitExecution(stackName, changeSetName, region, accountId, false);+ }- boolean isCreate = "CREATE".equalsIgnoreCase(cs.getChangeSetType()) ||- "CREATE_IN_PROGRESS".equals(stack.getStatus());+ private record ClaimedExecution(ChangeSet changeSet, boolean isCreate) {}- stack.setStatus(isCreate ? "CREATE_IN_PROGRESS" : "UPDATE_IN_PROGRESS");- stack.setLastUpdatedTime(now());- addEvent(stack, stack.getStackName(), stack.getStackId(),- "AWS::CloudFormation::Stack", isCreate ? "CREATE_IN_PROGRESS" : "UPDATE_IN_PROGRESS", null);+ // compute() holds the stack's per-key lock for the whole claim, so only one racing execution can win.+ private Future<?> claimAndSubmitExecution(String stackNameOrArn, String changeSetName, String region,+ String accountId, boolean requireAvailable) {+ String canonicalStackName = getStackOrThrow(stackNameOrArn, region, accountId).getStackName();+ String resolvedChangeSetName = resolveChangeSetName(changeSetName, region, accountId);++ ClaimedExecution[] claimed = new ClaimedExecution[1];+ Stack stack = stacks.compute(stackKey(accountId, canonicalStackName, region), (k, existing) -> {+ if (existing == null) {+ throw new AwsException("ValidationError",+ "Stack with id " + stackNameOrArn + " does not exist", 400);+ }+ ChangeSet cs = existing.getChangeSets().get(resolvedChangeSetName);+ if (cs == null) {+ throw new AwsException("ChangeSetNotFoundException",+ "ChangeSet [" + changeSetName + "] does not exist", 400);+ }+ String executionStatus = cs.getExecutionStatus();+ boolean eligible = requireAvailable+ ? "AVAILABLE".equals(executionStatus)+ : executionStatus == null || !executionStatus.startsWith("EXECUTE_");+ if (!eligible) {+ throw invalidChangeSetStatus(cs);+ }+ boolean isCreate = "CREATE".equalsIgnoreCase(cs.getChangeSetType()) ||+ "CREATE_IN_PROGRESS".equals(existing.getStatus());+ cs.setExecutionStatus("EXECUTE_IN_PROGRESS");+ if (requireAvailable) {+ // CloudFormation deletes every other change set on the stack once one of them executes.+ existing.getChangeSets().values().removeIf(other -> other != cs);+ } else {+ for (ChangeSet other : existing.getChangeSets().values()) {+ if (other != cs && "AVAILABLE".equals(other.getExecutionStatus())) {+ other.setExecutionStatus("OBSOLETE");+ }+ }+ }+ existing.setStatus(isCreate ? "CREATE_IN_PROGRESS" : "UPDATE_IN_PROGRESS");+ existing.setLastUpdatedTime(now());+ addEvent(existing, existing.getStackName(), existing.getStackId(),+ "AWS::CloudFormation::Stack", isCreate ? "CREATE_IN_PROGRESS" : "UPDATE_IN_PROGRESS", null);+ claimed[0] = new ClaimedExecution(cs, isCreate);+ return existing;+ });persistStack(stack);+ return submitExecution(stack, claimed[0].changeSet(), claimed[0].isCreate(), region, accountId);+ }++ private AwsException invalidChangeSetStatus(ChangeSet cs) {+ String detail = "FAILED".equals(cs.getStatus())+ ? "status of [" + cs.getStatus() + "]"+ : "execution status of [" + cs.getExecutionStatus() + "]";+ return new AwsException("InvalidChangeSetStatus",+ "ChangeSet [" + cs.getChangeSetId() + "] cannot be executed in its current " + detail, 400);+ }++ private Future<?> submitExecution(Stack stack, ChangeSet cs, boolean isCreate, String region, String accountId) {String templateBody = cs.getTemplateBody();Map<String, String> params = cs.getParameters() != null ? cs.getParameters() : Map.of();- return executor.submit(() -> runUnderAccount(accountId,- () -> executeTemplate(stack, templateBody, params, isCreate, region, accountId)));+ return executor.submit(() -> runUnderAccount(accountId, () -> {+ executeTemplate(stack, templateBody, params, isCreate, region, accountId);+ String status = stack.getStatus();+ cs.setExecutionStatus(status != null && (status.contains("ROLLBACK") || status.endsWith("_FAILED"))+ ? "EXECUTE_FAILED" : "EXECUTE_COMPLETE");+ persistStack(stack);+ }));}/**diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationExecuteChangeSetConcurrencyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationExecuteChangeSetConcurrencyIntegrationTest.javanew file mode 100644index 0000000000..87043f7b4d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationExecuteChangeSetConcurrencyIntegrationTest.java@@ -0,0 +1,307 @@+package io.github.hectorvent.floci.services.cloudformation;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import static io.restassured.RestAssured.given;+import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.containsString;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++@QuarkusTest+class CloudFormationExecuteChangeSetConcurrencyIntegrationTest {++ private final List<String> stacksToDelete = new ArrayList<>();++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @AfterEach+ void deleteStacks() {+ for (String stackName : stacksToDelete) {+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteStack")+ .formParam("StackName", stackName)+ .when()+ .post("/");+ }+ stacksToDelete.clear();+ }++ @Test+ void executeChangeSet_concurrentCalls_onlyOneRunsAndProvisionsResources() throws Exception {+ String stackName = uniqueStackName("race");+ createChangeSet(stackName, "initial", "CREATE", topicTemplate("Topic"))+ .then().statusCode(200).body(containsString("<Id>"));++ CountDownLatch start = new CountDownLatch(1);+ AtomicReference<Response> response1 = new AtomicReference<>();+ AtomicReference<Response> response2 = new AtomicReference<>();+ AtomicReference<Throwable> unexpected = new AtomicReference<>();++ Thread caller1 = new Thread(() -> race(start, () -> response1.set(executeChangeSet(stackName, "initial")), unexpected));+ Thread caller2 = new Thread(() -> race(start, () -> response2.set(executeChangeSet(stackName, "initial")), unexpected));+ caller1.start();+ caller2.start();+ start.countDown();+ caller1.join(TimeUnit.SECONDS.toMillis(10));+ caller2.join(TimeUnit.SECONDS.toMillis(10));+ assertFalse(caller1.isAlive() || caller2.isAlive(), "a thread never finished");+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());++ List<Response> responses = List.of(response1.get(), response2.get());+ assertEquals(1, responses.stream().filter(r -> r.getStatusCode() == 200).count(),+ () -> "expected exactly one call accepted: " + bodies(responses));+ assertEquals(1, responses.stream().filter(r -> r.getStatusCode() == 400).count(),+ () -> "expected exactly one call rejected: " + bodies(responses));++ String rejectedBody = responses.get(0).getStatusCode() == 400+ ? responses.get(0).asString() : responses.get(1).asString();+ assertThat(rejectedBody, containsString("<Code>InvalidChangeSetStatus</Code>"));+ assertTrue(rejectedBody.contains("execution status of [EXECUTE_IN_PROGRESS]")+ || rejectedBody.contains("execution status of [EXECUTE_COMPLETE]"), rejectedBody);++ waitForStackStatus(stackName, "CREATE_COMPLETE");+ assertEquals(1, countTopicsWithPrefix(stackName + "-Topic-"), "expected exactly one topic provisioned");+ }++ @Test+ void executeChangeSet_afterCompletion_rejectsSecondExecution() {+ String stackName = uniqueStackName("reexec");+ createChangeSet(stackName, "initial", "CREATE", topicTemplate("Topic")).then().statusCode(200);+ executeChangeSet(stackName, "initial").then().statusCode(200);+ waitForStackStatus(stackName, "CREATE_COMPLETE");++ assertEquals("EXECUTE_COMPLETE", describeChangeSetExecutionStatus(stackName, "initial"));++ Response second = executeChangeSet(stackName, "initial");+ second.then().statusCode(400).body(containsString("<Code>InvalidChangeSetStatus</Code>"));+ assertThat(second.asString(), containsString("execution status of [EXECUTE_COMPLETE]"));++ assertEquals(1, countTopicsWithPrefix(stackName + "-Topic-"));+ }++ @Test+ void executeChangeSet_deletesOtherChangeSetsOnTheStack() {+ String stackName = uniqueStackName("delete-siblings");+ createChangeSet(stackName, "initial", "CREATE", topicTemplate("Base")).then().statusCode(200);+ executeChangeSet(stackName, "initial").then().statusCode(200);+ waitForStackStatus(stackName, "CREATE_COMPLETE");++ createChangeSet(stackName, "update-a", "UPDATE", twoTopicTemplate("Base", "TopicA")).then().statusCode(200);+ createChangeSet(stackName, "update-b", "UPDATE", twoTopicTemplate("Base", "TopicB")).then().statusCode(200);++ executeChangeSet(stackName, "update-a").then().statusCode(200);+ waitForStackStatus(stackName, "UPDATE_COMPLETE");++ describeChangeSet(stackName, "update-b").then().statusCode(400)+ .body(containsString("<Code>ChangeSetNotFoundException</Code>"));+ executeChangeSet(stackName, "update-b").then().statusCode(400)+ .body(containsString("<Code>ChangeSetNotFoundException</Code>"));++ assertEquals("EXECUTE_COMPLETE", describeChangeSetExecutionStatus(stackName, "update-a"));++ assertEquals(1, countTopicsWithPrefix(stackName + "-TopicA-"));+ assertEquals(0, countTopicsWithPrefix(stackName + "-TopicB-"));+ }++ @Test+ void executeChangeSet_afterCreateStack_rejectsInitialCreateChangeSet() {+ String stackName = uniqueStackName("internal");+ createStack(stackName, topicTemplate("Topic"));++ Response rejected = executeChangeSet(stackName, "initial-create");+ rejected.then().statusCode(400).body(containsString("<Code>InvalidChangeSetStatus</Code>"));+ assertThat(rejected.asString(), containsString("execution status of [EXECUTE_"));+ }++ @Test+ void updateStack_marksPendingChangeSetsObsolete() {+ String stackName = uniqueStackName("obsolete");+ createStack(stackName, topicTemplate("Base"));+ waitForStackStatus(stackName, "CREATE_COMPLETE");+ createChangeSet(stackName, "pending", "UPDATE", twoTopicTemplate("Base", "TopicA")).then().statusCode(200);++ updateStack(stackName, twoTopicTemplate("Base", "TopicB"));+ waitForStackStatus(stackName, "UPDATE_COMPLETE");++ assertEquals("OBSOLETE", describeChangeSetExecutionStatus(stackName, "pending"));+ Response rejected = executeChangeSet(stackName, "pending");+ rejected.then().statusCode(400).body(containsString("<Code>InvalidChangeSetStatus</Code>"));+ assertThat(rejected.asString(), containsString("execution status of [OBSOLETE]"));+ assertEquals(0, countTopicsWithPrefix(stackName + "-TopicA-"));+ }++ private String uniqueStackName(String prefix) {+ String name = "cfn-execute-" + prefix + "-" + Long.toString(System.nanoTime(), 36);+ stacksToDelete.add(name);+ return name;+ }++ private static String topicTemplate(String logicalId) {+ return """+ Resources:+ %s:+ Type: AWS::SNS::Topic+ """.formatted(logicalId);+ }++ private static String twoTopicTemplate(String logicalId1, String logicalId2) {+ return """+ Resources:+ %s:+ Type: AWS::SNS::Topic+ %s:+ Type: AWS::SNS::Topic+ """.formatted(logicalId1, logicalId2);+ }++ private static Response createChangeSet(String stackName, String changeSetName, String changeSetType, String template) {+ return given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateChangeSet")+ .formParam("StackName", stackName)+ .formParam("ChangeSetName", changeSetName)+ .formParam("ChangeSetType", changeSetType)+ .formParam("TemplateBody", template)+ .when()+ .post("/");+ }++ private static void createStack(String stackName, String template) {+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateStack")+ .formParam("StackName", stackName)+ .formParam("TemplateBody", template)+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ private static void updateStack(String stackName, String template) {+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "UpdateStack")+ .formParam("StackName", stackName)+ .formParam("TemplateBody", template)+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ private static Response executeChangeSet(String stackName, String changeSetName) {+ return given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ExecuteChangeSet")+ .formParam("StackName", stackName)+ .formParam("ChangeSetName", changeSetName)+ .when()+ .post("/");+ }++ private static Response describeChangeSet(String stackName, String changeSetName) {+ return given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeChangeSet")+ .formParam("StackName", stackName)+ .formParam("ChangeSetName", changeSetName)+ .when()+ .post("/");+ }++ private static String describeChangeSetExecutionStatus(String stackName, String changeSetName) {+ return describeChangeSet(stackName, changeSetName)+ .then()+ .statusCode(200)+ .extract()+ .xmlPath()+ .getString("DescribeChangeSetResponse.DescribeChangeSetResult.ExecutionStatus");+ }++ private static long countTopicsWithPrefix(String prefix) {+ List<String> arns = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ListTopics")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract()+ .xmlPath()+ .getList("ListTopicsResponse.ListTopicsResult.Topics.member.TopicArn", String.class);+ return arns.stream()+ .filter(arn -> arn.substring(arn.lastIndexOf(':') + 1).startsWith(prefix))+ .count();+ }++ private static void race(CountDownLatch start, Runnable action, AtomicReference<Throwable> unexpected) {+ await(start);+ try {+ action.run();+ } catch (Throwable t) {+ unexpected.set(t);+ }+ }++ private static void await(CountDownLatch latch) {+ try {+ latch.await();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ }++ private static String bodies(List<Response> responses) {+ return responses.get(0).asString() + " | " + responses.get(1).asString();+ }++ private static void waitForStackStatus(String stackName, String status) {+ String expected = "<StackStatus>" + status + "</StackStatus>";+ String body = "";+ long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos();+ while (System.nanoTime() < deadline) {+ body = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", stackName)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract()+ .asString();+ if (body.contains(expected)) {+ return;+ }+ try {+ Thread.sleep(100);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError("Interrupted while waiting for stack " + stackName+ + " to reach " + expected, e);+ }+ }+ assertThat(body, containsString(expected));+ }+}