Light, fluffy, and always free - The AWS Local Emulator alternative
feat(cloudformation): list a deleted stack and report its DeletionTime (#4246)
Allen Sanborn67f06c3ac918
diff --git a/docs/services/cloudformation.md b/docs/services/cloudformation.mdindex d09a4eb8e7..3449277d89 100644--- a/docs/services/cloudformation.md+++ b/docs/services/cloudformation.md@@ -382,6 +382,25 @@ Lambda. floci supports two shapes:ResponseURL callback fires. The wait is bounded by an async custom-resource timeout (3 minutes bydefault); a resource that never completes fails the stack rather than hanging.+## Deleted Stacks++A deleted stack is kept for a short window afterwards rather than being forgotten at+`DELETE_COMPLETE`, because the client that asked for the delete goes on polling for it. Within that+window AWS's asymmetry is reproduced:++- `DescribeStacks` **by stack id** (the ARN) answers with the stack, its `DELETE_COMPLETE` status+ and its `DeletionTime`. `DescribeStackEvents` by stack id answers likewise.+- `DescribeStacks` **by name** returns `ValidationError`, as it does on AWS. The CDK CLI depends on+ the pair: it monitors a delete by stack id, and a stack id that stops resolving ends the rest of+ a `cdk destroy --all` run.+- `ListStacks` includes the stack as `DELETE_COMPLETE` with its `DeletionTime`, and a+ `StackStatusFilter` of `DELETE_COMPLETE` selects it. A name reused after a delete is listed+ twice, once per stack id.++Once the window passes, all of those behave as though the stack had never existed. AWS keeps a+deleted stack for 90 days; locally it only has to outlive the delete polling of the client that+asked for it, so the window defaults to 30 seconds (see [Configuration](#configuration)).+## Deletion PoliciesA resource's [`DeletionPolicy`](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-attribute-deletionpolicy.html)@@ -456,6 +475,7 @@ provisioning outcomes rather than always returning `SUCCEEDED`.| Variable | Default | Description ||---|---|---|| `FLOCI_SERVICES_CLOUDFORMATION_ENABLED` | `true` | Enable or disable the service |+| `FLOCI_SERVICES_CLOUDFORMATION_DELETED_STACK_RETENTION_SECONDS` | `30` | How long a deleted stack stays describable by its stack id and listed as `DELETE_COMPLETE` (see [Deleted Stacks](#deleted-stacks)) |## Examplesdiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationQueryHandler.javaindex 474193e8f5..a5f4c2202f 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationQueryHandler.java@@ -559,8 +559,11 @@ public class CloudFormationQueryHandler {.elem("StackId", s.getStackId()).elem("StackName", s.getStackName()).elem("StackStatus", s.getStatus())- .elem("CreationTime", ISO.format(s.getCreationTime()))- .end("member");+ .elem("CreationTime", ISO.format(s.getCreationTime()));+ if (s.getDeletionTime() != null) {+ xml.elem("DeletionTime", ISO.format(s.getDeletionTime()));+ }+ xml.end("member");}xml.end("StackSummaries").end("ListStacksResult").raw(AwsQueryResponse.responseMetadata())@@ -601,6 +604,9 @@ public class CloudFormationQueryHandler {if (s.getLastUpdatedTime() != null) {xml.elem("LastUpdatedTime", ISO.format(s.getLastUpdatedTime()));}+ if (s.getDeletionTime() != null) {+ xml.elem("DeletionTime", ISO.format(s.getDeletionTime()));+ }if (s.getStatusReason() != null) {xml.elem("StackStatusReason", s.getStatusReason());}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.javaindex f9f6bdf1f7..29e43de84f 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java@@ -1085,10 +1085,38 @@ public class CloudFormationService implements ResourceProvider {// ── ListStacks ────────────────────────────────────────────────────────────+ /**+ * Summaries for the region's stacks, the ones still within the deleted-stack retention window+ * included. AWS keeps a deleted stack in ListStacks as {@code DELETE_COMPLETE} long after+ * DescribeStacks has stopped answering for it by name, which is how a client reconciles what+ * it deployed against what is left. A name reused after a delete lists twice, once per stack+ * id, as it does on AWS.+ *+ * <p>Reduced by stack id, because a stack being deleted is briefly in both maps: it is+ * retained before it is removed from the live one, so that it is never in neither.+ *+ * <p>The maps are read one after the other, the live one first and in full, rather than as one+ * concatenated pipeline. A delete retains before it removes, so reading in that order leaves a+ * stack mid-handover in at least one of the two views. Building both views up front breaks+ * that: a {@link ConcurrentHashMap} view reflects the table at some point at or since its own+ * creation and is not guaranteed to reflect a modification made after it, so a retained view+ * created before the retain need never show the stack, while a live view created after it can+ * already miss it. Traversing the retained view second does not rescue this, because it is+ * creation and not traversal that bounds the guarantee.+ */public List<Stack> listStacks(String region) {String accountId = currentAccount();- return stacks.values().stream()+ Instant current = now();+ Map<String, Stack> byStackId = new LinkedHashMap<>();+ stacks.values().stream().filter(s -> accountId.equals(ownerAccount(s)) && region.equals(s.getRegion()))+ .forEach(s -> byStackId.putIfAbsent(s.getStackId(), s));+ deletedStacks.values().stream()+ .filter(entry -> !entry.isExpired(current))+ .map(DeletedStackEntry::stack)+ .filter(s -> accountId.equals(ownerAccount(s)) && region.equals(s.getRegion()))+ .forEach(s -> byStackId.putIfAbsent(s.getStackId(), s));+ return byStackId.values().stream().sorted(Comparator.comparing(Stack::getCreationTime)).toList();}@@ -2062,15 +2090,24 @@ public class CloudFormationService implements ResourceProvider {throw new IllegalStateException(reason);}+ // Before the status, never after. Status is the volatile publishing write, so a reader+ // that observes DELETE_COMPLETE is guaranteed to observe every write that preceded it.+ // Assigned afterwards, a concurrent DescribeStacks can report the terminal status with+ // no DeletionTime at all.+ stack.setDeletionTime(now());stack.setStatus("DELETE_COMPLETE");addEvent(stack, stack.getStackName(), stack.getStackId(),"AWS::CloudFormation::Stack", "DELETE_COMPLETE", null);removeStackExports(stack, region);- stacks.remove(stackKey(ownerAccount(stack), stack.getStackName(), region));- unpersistStack(ownerAccount(stack), stack.getStackName(), region);+ // Retained before it leaves the live map, never after: the other order leaves a window+ // in which the stack is in neither, and a ListStacks landing there loses it entirely.+ // Overlapping instead is harmless, since both maps hold this same Stack and listStacks+ // reduces by stack id.deletedStacks.put(stack.getStackId(), new DeletedStackEntry(stack,now().plusSeconds(config.services().cloudformation().deletedStackRetentionSeconds())));+ stacks.remove(stackKey(ownerAccount(stack), stack.getStackName(), region));+ unpersistStack(ownerAccount(stack), stack.getStackName(), region);LOG.infov("Stack {0} deleted", stack.getStackName());} catch (Exception e) {diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/model/Stack.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/model/Stack.javaindex faf1e90fa0..681b29828a 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/model/Stack.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/model/Stack.java@@ -45,6 +45,8 @@ public class Stack {private String statusReason;private Instant creationTime = Instant.now();private Instant lastUpdatedTime;+ /** Set once the stack reaches DELETE_COMPLETE; null for every stack that is still there. */+ private Instant deletionTime;private String templateBody;private String originalTemplateBody;private List<String> capabilities = new ArrayList<>();@@ -79,6 +81,8 @@ public class Stack {public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }public Instant getLastUpdatedTime() { return lastUpdatedTime; }public void setLastUpdatedTime(Instant lastUpdatedTime) { this.lastUpdatedTime = lastUpdatedTime; }+ public Instant getDeletionTime() { return deletionTime; }+ public void setDeletionTime(Instant deletionTime) { this.deletionTime = deletionTime; }public String getTemplateBody() { return templateBody; }public void setTemplateBody(String templateBody) { this.templateBody = templateBody; }public String getOriginalTemplateBody() { return originalTemplateBody; }diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex fd956db40b..3025786fa9 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -479,6 +479,10 @@ floci:timeout-sweep-interval-seconds: 1cloudformation:enabled: true+ # How long a deleted stack stays describable by its stack id (ARN) and listed as+ # DELETE_COMPLETE. AWS keeps one for 90 days; locally it only has to outlive the delete+ # polling of the client that asked for it.+ deleted-stack-retention-seconds: 30# Fail an AWS::Lambda::Function whose S3 code cannot be read, as real CloudFormation# does. Set true to fall back to the built-in stub handler instead (issue #2648).allow-stub-lambda-code: falsediff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletedStackListingRaceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletedStackListingRaceTest.javanew file mode 100644index 0000000000..13a21e5a03--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationDeletedStackListingRaceTest.java@@ -0,0 +1,177 @@+package io.github.hectorvent.floci.services.cloudformation;++import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.cloudformation.model.Stack;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicReference;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;++/**+ * A stack being deleted moves from the live map to the retained-deleted one. Handing it over in+ * the wrong order leaves a window in which it is in neither, and a ListStacks landing there loses+ * a stack that both exists and is retained.+ */+@QuarkusTest+@TestProfile(CloudFormationDeletedStackListingRaceTest.LongRetentionProfile.class)+class CloudFormationDeletedStackListingRaceTest {++ /**+ * Retention long enough that it cannot end this test. The shipped 30 seconds are measured on+ * the test {@code MutableClock}, which advances a millisecond on every read, and the pollers+ * below read it thousands of times: left at the default they expire the retained record+ * themselves and lose the stack for a reason that has nothing to do with the handoff.+ */+ public static final class LongRetentionProfile implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.of("floci.services.cloudformation.deleted-stack-retention-seconds", "86400");+ }+ }++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "000000000000";+ /**+ * The window is one map removal plus a storage delete wide, so a single reader on a single+ * delete rarely lands in it. Several readers across many deletes make it dependable: on the+ * unfixed ordering this fails on every run, and it still takes only a few seconds.+ */+ private static final int ATTEMPTS = 60;+ private static final int POLLERS = 4;+ /** Generous: a poller checks the stop flag every pass, so overrunning this means it is stuck. */+ private static final long POLLER_SHUTDOWN_MILLIS = 10_000;++ @Inject+ CloudFormationService cloudFormationService;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void listStacks_whileADeleteHandsTheStackOver_neverLosesIt() throws Exception {+ for (int attempt = 0; attempt < ATTEMPTS; attempt++) {+ String stackName = "list-race-stack-" + attempt;+ String stackId = createStack(stackName, attempt);++ AtomicBoolean stop = new AtomicBoolean();+ AtomicReference<String> lost = new AtomicReference<>();+ List<Thread> pollers = new ArrayList<>();+ for (int p = 0; p < POLLERS; p++) {+ Thread poller = new Thread(() -> {+ while (!stop.get() && !Thread.currentThread().isInterrupted()) {+ Stack listed = RequestScopes.callAs(ACCOUNT, () ->+ cloudFormationService.listStacks(REGION).stream()+ .filter(s -> stackId.equals(s.getStackId()))+ .findFirst()+ .orElse(null));+ if (listed == null) {+ lost.compareAndSet(null, "ListStacks lost " + stackId + " mid-delete");+ return;+ }+ }+ }, "list-race-poller-" + p);+ // Daemon as a backstop: a poller left running by some future edit that throws+ // where nothing throws today must not be able to hold the test JVM open.+ poller.setDaemon(true);+ pollers.add(poller);+ poller.start();+ }++ List<String> stillRunning = new ArrayList<>();+ try {+ deleteStack(stackName);+ awaitDeleted(stackId);+ } finally {+ // In a finally so a failing delete stops the pollers on its way out rather than+ // leaving them spinning, which would bury the failure that caused it.+ stop.set(true);+ for (Thread poller : pollers) {+ poller.join(POLLER_SHUTDOWN_MILLIS);+ if (poller.isAlive()) {+ poller.interrupt();+ stillRunning.add(poller.getName());+ }+ }+ }++ // After the try, never inside the finally: an assertion there would replace whatever+ // the delete threw with its own failure.+ assertTrue(stillRunning.isEmpty(), "pollers did not stop: " + stillRunning);+ assertNull(lost.get(), lost.get());+ }+ }++ private String createStack(String stackName, int attempt) {+ String template = """+ {+ "Resources": {+ "Q1": { "Type": "AWS::SQS::Queue", "Properties": { "QueueName": "list-race-q1-%1$d" } },+ "Q2": { "Type": "AWS::SQS::Queue", "Properties": { "QueueName": "list-race-q2-%1$d" } },+ "T1": { "Type": "AWS::SNS::Topic", "Properties": { "TopicName": "list-race-t1-%1$d" } }+ }+ }+ """.formatted(attempt);++ String response = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateStack")+ .formParam("StackName", stackName)+ .formParam("TemplateBody", template)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<StackId>"))+ .extract().asString();++ return response.substring(response.indexOf("<StackId>") + "<StackId>".length(),+ response.indexOf("</StackId>"));+ }++ private void deleteStack(String stackName) {+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteStack")+ .formParam("StackName", stackName)+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ private void awaitDeleted(String stackId) throws InterruptedException {+ long deadline = System.currentTimeMillis() + 10_000;+ while (System.currentTimeMillis() < deadline) {+ String body = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", stackId)+ .when()+ .post("/")+ .then()+ .extract().asString();+ if (body.contains("<StackStatus>DELETE_COMPLETE</StackStatus>")) {+ return;+ }+ Thread.sleep(2);+ }+ fail("stack " + stackId + " never reached DELETE_COMPLETE");+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.javaindex 662dfbe298..d8d6f6c116 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.services.ec2.Ec2Service;import io.github.hectorvent.floci.testing.MutableClock;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;+import io.restassured.specification.RequestSpecification;import jakarta.inject.Inject;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.BeforeEach;@@ -2213,6 +2214,179 @@ class CloudFormationIntegrationTest {.body(containsString("does not exist"));}+ @Test+ void describeDeletedStack_byArn_reportsItsDeletionTime() throws Exception {+ String stackArn = createAndDeleteStack("deleted-deletion-time-stack",+ "deleted-deletion-time-test-bucket");++ String xml = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", stackArn)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().asString();++ assertThat(xml, containsString("<StackStatus>DELETE_COMPLETE</StackStatus>"));+ // The service clock, not the wall clock: the test clock starts the run at 2026-01-01.+ assertThat(xml, containsString("<DeletionTime>2026-01-01T"));+ }++ @Test+ void describeLiveStack_reportsNoDeletionTime() {+ String template = """+ {+ "Resources": {+ "MyBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "live-no-deletion-time-test-bucket" }+ }+ }+ }+ """;++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateStack")+ .formParam("StackName", "live-no-deletion-time-stack")+ .formParam("TemplateBody", template)+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", "live-no-deletion-time-stack")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(not(containsString("<DeletionTime>")));+ }++ @Test+ void listStacks_includesADeletedStackAsDeleteComplete() throws Exception {+ String stackArn = createAndDeleteStack("listed-deleted-stack",+ "listed-deleted-test-bucket");++ String member = listStacksMemberFor(listStacksXml(null), stackArn);++ assertThat(member, containsString("<StackName>listed-deleted-stack</StackName>"));+ assertThat(member, containsString("<StackStatus>DELETE_COMPLETE</StackStatus>"));+ assertThat(member, containsString("<DeletionTime>2026-01-01T"));+ }++ @Test+ void listStacks_filteredToDeleteComplete_returnsTheDeletedStackAndNoLiveOne() throws Exception {+ String stackArn = createAndDeleteStack("listed-filtered-deleted-stack",+ "listed-filtered-deleted-test-bucket");++ String xml = listStacksXml("DELETE_COMPLETE");++ assertThat(xml, containsString("<StackId>" + stackArn + "</StackId>"));+ assertThat(xml, not(containsString("<StackStatus>CREATE_COMPLETE</StackStatus>")));+ }++ @Test+ void listStacks_afterTheRetentionWindow_dropsTheDeletedStack() throws Exception {+ String stackArn = createAndDeleteStack("listed-expiring-deleted-stack",+ "listed-expiring-deleted-test-bucket");++ assertThat(listStacksXml(null), containsString("<StackId>" + stackArn + "</StackId>"));++ clock.advance(Duration.ofSeconds(31));++ assertThat(listStacksXml(null), not(containsString("<StackId>" + stackArn + "</StackId>")));+ }++ /**+ * Creates a single-bucket stack, deletes it, and answers its stack ARN once the delete has+ * reached DELETE_COMPLETE. DeleteStack hands the work to a background executor, so the+ * DELETE_COMPLETE event is what says the retained record is in place.+ */+ private String createAndDeleteStack(String stackName, String bucketName) throws Exception {+ String template = """+ {+ "Resources": {+ "MyBucket": {+ "Type": "AWS::S3::Bucket",+ "Properties": { "BucketName": "%s" }+ }+ }+ }+ """.formatted(bucketName);++ String createResponse = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateStack")+ .formParam("StackName", stackName)+ .formParam("TemplateBody", template)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<StackId>"))+ .extract().asString();++ String stackArn = createResponse.substring(+ createResponse.indexOf("<StackId>") + "<StackId>".length(),+ createResponse.indexOf("</StackId>"));++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteStack")+ .formParam("StackName", stackName)+ .when()+ .post("/")+ .then()+ .statusCode(200);++ long deadline = System.currentTimeMillis() + 10_000;+ while (System.currentTimeMillis() < deadline) {+ String events = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStackEvents")+ .formParam("StackName", stackArn)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().asString();++ if (events.contains("<ResourceStatus>DELETE_COMPLETE</ResourceStatus>")) {+ return stackArn;+ }+ Thread.sleep(200);+ }+ throw new AssertionError("stack " + stackName + " never reached DELETE_COMPLETE");+ }++ private String listStacksXml(String statusFilter) {+ RequestSpecification request = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ListStacks");+ if (statusFilter != null) {+ request = request.formParam("StackStatusFilter.member.1", statusFilter);+ }+ return request+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().asString();+ }++ /** The one ListStacks summary carrying this stack id, so an assertion cannot match another. */+ private static String listStacksMemberFor(String xml, String stackArn) {+ int idAt = xml.indexOf("<StackId>" + stackArn + "</StackId>");+ assertTrue(idAt >= 0, "stack " + stackArn + " is not listed:\n" + xml);+ return xml.substring(xml.lastIndexOf("<member>", idAt), xml.indexOf("</member>", idAt));+ }+@Testvoid deleteChangeSet_nonExistentChangeSet_returnsError() {String template = """diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 39065f41b4..7148773cc8 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -262,6 +262,9 @@ floci:timeout-sweep-interval-seconds: 1cloudformation:enabled: true+ # Pinned to the shipped default: the retention tests advance the test clock past this+ # window by name, so the value they assume is spelled out rather than inherited.+ deleted-stack-retention-seconds: 30# Fail an AWS::Lambda::Function whose S3 code cannot be read, as real CloudFormation# does. Set true to fall back to the built-in stub handler instead (issue #2648).allow-stub-lambda-code: false