Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): restore SSM parameters on failed updates (#5752)
Jackson Price945f5d39951f
diff --git a/docs/services/cloudformation.md b/docs/services/cloudformation.mdindex e2fc9ddce0..0aceb4880f 100644--- a/docs/services/cloudformation.md+++ b/docs/services/cloudformation.md@@ -131,6 +131,31 @@ an atomic ownership check against another stack or a direct Cognito API writer rFailed in-place restoration preserves the original name and scopes for a `ContinueUpdateRollback`retry. Deleting the current server keeps this snapshot if deletion fails.+## SSM parameters++Like AWS's resource handler, `AWS::SSM::Parameter` is created with one `PutParameter` that carries+the template's tags and does not overwrite. A name that already exists outside the stack fails the+resource with `ParameterAlreadyExists`; the stack never overwrites, adopts or deletes that+parameter. Floci applies the `Name`, `Type`, `Value` and `Tags` properties.++An update that keeps `Name` overwrites the parameter. When a later resource fails the update, the+rollback writes the prior value and settings back, as a new SSM version when they changed (the+version history is not rewound, as on AWS), and returns the tags the previous or the failed+template set to their prior values; other tags are left alone. As on AWS, the update first checks+that the parameter exists: one deleted outside the stack fails it with `ParameterNotFound` before+anything is written. A rollback that finds the parameter gone fails the same way, leaving the stack+in `UPDATE_ROLLBACK_FAILED`.++Giving `Name` a new value replaces the parameter. The previous parameter stays until the update+commits, and is then deleted unless `UpdateReplacePolicy` is `Retain`; a failed update deletes the+replacement instead. A restore or delete that fails during the rollback leaves the stack in+`UPDATE_ROLLBACK_FAILED`, and `ContinueUpdateRollback` retries it (see+[Failed update rollbacks](#failed-update-rollbacks)). Deleting a parameter that is already gone+counts as deleted. Known differences: removing an explicit `Name` keeps the parameter under that+name, where AWS replaces it with one under a generated name; and an update in place of a parameter+deleted outside the stack ends in `UPDATE_ROLLBACK_COMPLETE`, since nothing was written, where AWS's+rollback repeats the existence check and likely ends in `UPDATE_ROLLBACK_FAILED`.+## Supported Resource TypesResource types provisioned during `CreateStack` / `UpdateStack` / `DeleteStack`. Each delegates to@@ -652,8 +677,10 @@ Deviations from AWS to be aware of:removals as well.- After `DeleteStack` completes, a retained resource's `DELETE_SKIPPED` record is only visiblethrough `DescribeStackEvents` for the deleted stack, not `DescribeStackResources`.-- `UpdateReplacePolicy`, and the `RetainExceptOnCreate` request parameter of `CreateStack` /- `UpdateStack`, are not implemented.+- `UpdateReplacePolicy` is honored only by the resource types that delete a displaced resource once+ the update commits, such as an SSM parameter or a log stream; other types keep the displaced+ resource, or delete it during the update, whatever the policy. The `RetainExceptOnCreate` request+ parameter of `CreateStack` / `UpdateStack` is not implemented.## StackSetsdiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.javaindex c2f08f6604..728abac648 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java@@ -210,6 +210,14 @@ public final class CfnRollback {*/public static final String SQS_UPDATE_SNAPSHOT_ATTR = "__FlociSqsUpdateSnapshot";+ /**+ * Holds the value, settings and managed tags an SSM parameter carried before an in-place+ * update changed them, with the resource's attributes from before, so a failed stack update+ * can put them back. Written by {@code SsmCfnProvisioner} before its put and spent by its+ * {@code rollbackUpdate}; a rename leaves its rollback to {@link #REPLACEMENT_CLEANUP_ATTR}.+ */+ public static final String SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR = "__FlociSsmParameterUpdateSnapshot";+/*** Holds the settings a Cognito user pool or user pool client had before an in-place update* changed them, so a failed stack update can put them back. Written bydiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/SsmCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/SsmCfnProvisioner.javaindex f0afd60c1e..1f7bb6d49d 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/SsmCfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/SsmCfnProvisioner.java@@ -1,23 +1,43 @@package io.github.hectorvent.floci.services.cloudformation.provisioners;+import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.ssm.SsmService;+import io.github.hectorvent.floci.services.ssm.model.Parameter;import jakarta.enterprise.context.ApplicationScoped;+import java.util.ArrayList;import java.util.Arrays;+import java.util.Collection;+import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.Objects;import java.util.Set;import java.util.TreeSet;-/** Provisions {@code AWS::SSM::Parameter}. */+/**+ * Provisions {@code AWS::SSM::Parameter}. The parameter name is the physical id and {@code Ref}.+ *+ * <p>A create, and the replacement a changed name makes, is one {@code PutParameter} that carries+ * the template's tags and does not overwrite, as AWS's handler sends it: a name that already exists+ * outside the stack fails the resource instead of being overwritten or adopted. An update that keeps+ * the name overwrites the parameter and drives the template's tags, after keeping what it changes on+ * the resource, so a failed stack update can put it back. A replacement leaves the displaced+ * parameter to the {@link ReplacementCleanup} record: deleted once the update commits, unless+ * {@code UpdateReplacePolicy} is {@code Retain}, and named again when the update rolls back.+ */@ApplicationScopedpublic class SsmCfnProvisioner implements CfnResourceProvisioner {private static final int PARAMETER_NAME_MAX_LENGTH = 2048;private static final String SSM_TEMPLATE_TAG_KEYS_ATTR = "FlociSsmTemplateTagKeys";+ private static final ObjectMapper MAPPER = new ObjectMapper();private final SsmService ssmService;@@ -32,6 +52,14 @@ public class SsmCfnProvisioner implements CfnResourceProvisioner {@Overridepublic void provision(StackResource r, JsonNode props, ProvisionContext ctx) {+ // A snapshot describes the update in flight; one an earlier update left behind is stale.+ r.getAttributes().remove(CfnRollback.SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR);+ if (ctx.isUpdate()) {+ // Rolling this update back undoes it alone; only ContinueUpdateRollback retries the+ // delete the last rollback could not make.+ ReplacementCleanup.endRollbackRetry(r);+ }+ Map<String, String> attributesBefore = new HashMap<>(r.getAttributes());String name = ctx.stablePhysicalName(ctx.resolveOptional(props, "Name"),r.getLogicalId(), PARAMETER_NAME_MAX_LENGTH, false);String value = ctx.resolveOptional(props, "Value");@@ -44,8 +72,14 @@ public class SsmCfnProvisioner implements CfnResourceProvisioner {}Map<String, String> tags = ctx.resolveTags(props, "Tags");SsmService.validateTagKeys(tags);- ssmService.putParameter(name, value, type, null, true, ctx.region());- reconcileTags(name, r.getAttributes().get(SSM_TEMPLATE_TAG_KEYS_ATTR), tags, ctx.region());+ // A dispatcher stub created nothing, so a parameter that takes over its id is a create.+ if (ctx.reusesPriorEntity(name) && !CfnResourceDispatcher.isStub(name, attributesBefore)) {+ snapshotBeforeUpdate(r, name, attributesBefore, tags.keySet(), ctx.region());+ ssmService.putParameter(name, value, type, null, true, ctx.region());+ reconcileTags(name, templateTagKeys(attributesBefore), tags, ctx.region());+ } else {+ ssmService.putParameter(name, value, type, null, false, tags, ctx.region());+ }r.setPhysicalId(name);r.getAttributes().put("Name", name);r.getAttributes().put("Type", type);@@ -57,21 +91,62 @@ public class SsmCfnProvisioner implements CfnResourceProvisioner {} else {r.getAttributes().put(SSM_TEMPLATE_TAG_KEYS_ATTR, String.join(",", new TreeSet<>(tags.keySet())));}+ ReplacementCleanup.record(r, ctx, attributesBefore);}/**- * Applies the template's tags and removes only the keys the previous template set and this one- * drops, so a tag added outside the template stays, as in AWS. The overwriting put keeps the- * parameter's existing tags, and a parameter provisioned before the keys were recorded has none- * to drop.+ * Keeps what an update in place is about to change, so {@link #rollbackUpdate} can put it back:+ * the parameter's value and settings, the tags of the keys the previous template or this one+ * manages, and the resource's attributes. Reading it first also fails the update of a parameter+ * deleted outside the stack with {@code ParameterNotFound} before anything is written, as AWS's+ * handler checks that the parameter exists.*/- private void reconcileTags(String name, String previousTemplateKeys, Map<String, String> desired,+ private void snapshotBeforeUpdate(StackResource r, String name, Map<String, String> attributesBefore,+ Set<String> desiredTagKeys, String region) {+ ObjectNode snapshot = MAPPER.createObjectNode();+ snapshot.put("region", region);+ snapshot.put("name", name);+ ObjectNode attributes = snapshot.putObject("attributes");+ attributesBefore.forEach((key, attribute) -> {+ if (!key.startsWith("__Floci") && attribute != null) {+ attributes.put(key, attribute);+ }+ });+ Parameter current = ssmService.getParameter(name, region);+ snapshot.put("value", current.getValue());+ snapshot.put("type", current.getType());+ snapshot.put("description", current.getDescription());+ snapshot.put("keyId", current.getKeyId());+ snapshot.put("allowedPattern", current.getAllowedPattern());+ snapshot.put("tier", current.getTier());+ if (current.getPolicies() != null) {+ snapshot.putArray("policies").addAll(current.getPolicies());+ }+ // A managed key the parameter did not carry is listed without a value, so the rollback removes it.+ Set<String> managedKeys = new TreeSet<>(templateTagKeys(attributesBefore));+ managedKeys.addAll(desiredTagKeys);+ Map<String, String> currentTags = current.getTags() == null ? Map.of() : current.getTags();+ ArrayNode tagKeys = snapshot.putArray("tagKeys");+ ObjectNode tags = snapshot.putObject("tags");+ for (String key : managedKeys) {+ tagKeys.add(key);+ if (currentTags.containsKey(key)) {+ tags.put(key, currentTags.get(key));+ }+ }+ r.getAttributes().put(CfnRollback.SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR, snapshot.toString());+ }++ /**+ * Applies {@code desired} and removes the {@code managedKeys} it does not hold, so a tag added+ * outside the template stays, as in AWS. The overwriting put keeps the parameter's existing tags,+ * and a parameter provisioned before the keys were recorded has none to drop.+ */+ private void reconcileTags(String name, Collection<String> managedKeys, Map<String, String> desired,String region) {- List<String> dropped = previousTemplateKeys == null || previousTemplateKeys.isEmpty()- ? List.of()- : Arrays.stream(previousTemplateKeys.split(","))- .filter(key -> !desired.containsKey(key))- .toList();+ List<String> dropped = managedKeys.stream()+ .filter(key -> !desired.containsKey(key))+ .toList();if (!dropped.isEmpty()) {ssmService.removeTagsFromResource(name, dropped, region);}@@ -80,6 +155,12 @@ public class SsmCfnProvisioner implements CfnResourceProvisioner {}}+ /** The tag keys the template that provisioned the resource set. */+ private static List<String> templateTagKeys(Map<String, String> attributes) {+ String keys = attributes.get(SSM_TEMPLATE_TAG_KEYS_ATTR);+ return keys == null || keys.isEmpty() ? List.of() : Arrays.asList(keys.split(","));+ }+/** AWS's form is {@code parameter/<name>} whether or not the name starts with a slash. */private static String parameterArn(String name, ProvisionContext ctx) {String path = name.startsWith("/") ? name : "/" + name;@@ -88,6 +169,111 @@ public class SsmCfnProvisioner implements CfnResourceProvisioner {@Overridepublic void delete(String resourceType, String physicalId, String region) {- ssmService.deleteParameter(physicalId, region);+ // ParameterNotFound is SSM's own code for a parameter that is already gone.+ CfnDeletes.safeDelete("SSM parameter", physicalId,+ () -> ssmService.deleteParameter(physicalId, region), "ParameterNotFound");+ }++ /**+ * An update in place that failed after its snapshot may already have changed the parameter, so+ * the engine keeps this attempt for {@link #rollbackUpdate} instead of putting the previous+ * resource back over it. A failure before the snapshot changed nothing and is left to the engine.+ */+ @Override+ public boolean retainsFailedUpdateState(StackResource resource) {+ return resource.getAttributes().containsKey(CfnRollback.SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR);+ }++ @Override+ public boolean hasReplacementUpdate(StackResource resource) {+ return ReplacementCleanup.hasReplacement(resource);+ }++ @Override+ public String updateCleanupPhysicalId(StackResource resource) {+ return ReplacementCleanup.cleanupPhysicalId(resource);+ }++ /**+ * The engine calls this for every resource once a stack update commits, but {@link #clearUpdate}+ * only after a replacement's cleanup, so an in-place update's snapshot is dropped here: the update+ * committed and nothing will roll it back.+ */+ @Override+ public UpdateCleanupResult completeUpdate(StackResource resource) {+ resource.getAttributes().remove(CfnRollback.SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR);+ return ReplacementCleanup.complete(resource, this::delete);+ }++ @Override+ public void clearUpdate(StackResource resource) {+ resource.getAttributes().remove(CfnRollback.SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR);+ ReplacementCleanup.clear(resource);+ }++ /**+ * A replacement is undone through the cleanup record, and an update in place from the snapshot+ * taken before it. Like AWS's rollback, which runs its update handler, the restore needs the+ * parameter to exist: one deleted outside the stack fails it with {@code ParameterNotFound}. The+ * value and settings go back with one overwrite, which SSM keeps as a new version, unless the+ * parameter already holds them; the managed tags go back to the values they had, and the resource+ * gets back the attributes it had. The snapshot is spent only once the restore succeeded, so a+ * restore that throws leaves it for ContinueUpdateRollback to retry. With neither on the resource+ * there is nothing to restore from.+ */+ @Override+ public boolean rollbackUpdate(StackResource resource) {+ if (ReplacementCleanup.rollback(resource, this::delete)) {+ return true;+ }+ String raw = resource.getAttributes().get(CfnRollback.SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR);+ if (raw == null) {+ return false;+ }+ JsonNode snapshot;+ try {+ snapshot = MAPPER.readTree(raw);+ } catch (JsonProcessingException e) {+ throw new IllegalStateException("Could not read the SSM parameter update snapshot for "+ + resource.getLogicalId(), e);+ }+ String region = snapshot.path("region").asText();+ String name = snapshot.path("name").asText();+ restore(ssmService.getParameter(name, region), snapshot, name, region);+ resource.getAttributes().keySet().removeIf(key -> !key.startsWith("__Floci"));+ snapshot.path("attributes").properties()+ .forEach(attribute -> resource.getAttributes().put(attribute.getKey(), attribute.getValue().asText()));+ resource.getAttributes().remove(CfnRollback.SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR);+ return true;+ }++ /** Puts the snapshot's value, settings and managed tags back on the parameter. */+ private void restore(Parameter current, JsonNode snapshot, String name, String region) {+ String value = snapshot.path("value").asText();+ String type = snapshot.path("type").textValue();+ String description = snapshot.path("description").textValue();+ String keyId = snapshot.path("keyId").textValue();+ String allowedPattern = snapshot.path("allowedPattern").textValue();+ String tier = snapshot.path("tier").textValue();+ List<JsonNode> policies = null;+ if (snapshot.has("policies")) {+ policies = new ArrayList<>();+ snapshot.path("policies").forEach(policies::add);+ }+ boolean unchanged = value.equals(current.getValue()) && Objects.equals(type, current.getType())+ && Objects.equals(description, current.getDescription())+ && Objects.equals(keyId, current.getKeyId())+ && Objects.equals(allowedPattern, current.getAllowedPattern())+ && Objects.equals(tier, current.getTier())+ && Objects.equals(policies, current.getPolicies());+ if (!unchanged) {+ ssmService.putParameter(name, value, type, description, true, null, keyId, allowedPattern, tier,+ policies, region);+ }+ List<String> managedKeys = new ArrayList<>();+ snapshot.path("tagKeys").forEach(key -> managedKeys.add(key.asText()));+ Map<String, String> tags = new HashMap<>();+ snapshot.path("tags").properties().forEach(tag -> tags.put(tag.getKey(), tag.getValue().asText()));+ reconcileTags(name, managedKeys, tags, region);}}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.javaindex d6ca819486..1901d0fcfc 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java@@ -10002,10 +10002,10 @@ class CloudFormationIntegrationTest {}@Test- void updateStack_unsupportedResourceRollbackRemainsExplicitlyFailed() {+ void updateStack_ssmRollbackRestoresTheValueAndTemplate() {String suffix = Long.toString(System.nanoTime(), 36);- String stackName = "unsupported-update-rollback-" + suffix;- String parameterName = "/cfn/unsupported-update-rollback/" + suffix;+ String stackName = "ssm-update-rollback-" + suffix;+ String parameterName = "/cfn/ssm-update-rollback/" + suffix;String template = """{"Resources": {@@ -10031,7 +10031,7 @@ class CloudFormationIntegrationTest {"Type": "AWS::SecretsManager::Secret","DependsOn": "Parameter","Properties": {- "Name": "unsupported-update-rollback-secret-%s",+ "Name": "ssm-update-rollback-secret-%s","SecretString": "explicit","GenerateSecretString": {"PasswordLength": 32}}@@ -10066,8 +10066,7 @@ class CloudFormationIntegrationTest {.post("/").then().statusCode(200)- .body(containsString("<StackStatus>UPDATE_ROLLBACK_FAILED</StackStatus>"))- .body(containsString("Parameter"));+ .body(containsString("<StackStatus>UPDATE_ROLLBACK_COMPLETE</StackStatus>"));given().contentType("application/x-www-form-urlencoded")@@ -10078,7 +10077,7 @@ class CloudFormationIntegrationTest {.then().statusCode(200).body(containsString("<LogicalResourceId>Parameter</LogicalResourceId>"))- .body(containsString("<ResourceStatus>UPDATE_FAILED</ResourceStatus>"))+ .body(not(containsString("<ResourceStatus>UPDATE_FAILED</ResourceStatus>"))).body(not(containsString("<LogicalResourceId>BadSecret</LogicalResourceId>")));given()@@ -10089,7 +10088,7 @@ class CloudFormationIntegrationTest {.post("/").then().statusCode(200)- .body("Parameter.Value", equalTo("updated"));+ .body("Parameter.Value", equalTo("initial"));given().contentType("application/x-www-form-urlencoded")@@ -10099,7 +10098,8 @@ class CloudFormationIntegrationTest {.post("/").then().statusCode(200)- .body(containsString("updated"));+ .body(containsString("initial"))+ .body(not(containsString("BadSecret")));given().contentType("application/x-www-form-urlencoded")diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationSsmRollbackIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationSsmRollbackIntegrationTest.javanew file mode 100644index 0000000000..0c5c35d4d7--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationSsmRollbackIntegrationTest.java@@ -0,0 +1,607 @@+package io.github.hectorvent.floci.services.cloudformation;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.XmlParser;+import io.github.hectorvent.floci.services.ssm.SsmService;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.mockito.InjectSpy;+import io.restassured.response.Response;+import io.restassured.specification.RequestSpecification;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import org.mockito.Mockito;++import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.stream.Collectors;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;++/**+ * {@code AWS::SSM::Parameter} through a stack's update lifecycle: a later resource that fails the+ * update rolls an in-place change or a rename back, a committed rename deletes or retains the+ * parameter it displaced, a name held outside the stack is never overwritten or adopted, and a+ * rollback that fails is finished by ContinueUpdateRollback. A failing update adds a Secrets+ * Manager secret that cannot be created, after every parameter.+ */+@QuarkusTest+class CloudFormationSsmRollbackIntegrationTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();++ @InjectSpy+ SsmService ssmService;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ /** One {@code AWS::SSM::Parameter} of the template, under its logical id. */+ private record Param(String logicalId, String name, String value, Map<String, String> tags, boolean retain) {+ Param(String logicalId, String name, String value) {+ this(logicalId, name, value, Map.of(), false);+ }+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void failedUpdateRestoresTheParameterItsTagsAndItsIdentity(boolean rename) throws Exception {+ String stack = unique("ssm-rollback");+ String original = "/" + stack;+ String target = rename ? original + "-new" : original;+ deploy("CreateStack", stack, template(false,+ new Param("Parameter", original, "before", Map.of("team", "old", "dropped", "restore"), false)));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ ssm("AddTagsToResource", Map.of("ResourceType", "Parameter", "ResourceId", original,+ "Tags", List.of(Map.of("Key", "external", "Value", "keep")))).then().statusCode(200);+ // Settings the template does not manage, which the in-place write resets.+ ssm("PutParameter", Map.of("Name", original, "Value", "before", "Type", "String", "Overwrite", true,+ "Description", "set outside the stack", "AllowedPattern", "^[a-z]+$")).then().statusCode(200);++ deploy("UpdateStack", stack, template(true,+ new Param("Parameter", target, "after", Map.of("team", "new", "added", "remove"), false)));++ CfnStackWaits.StackState state = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", state.status(), state.reason());+ assertEquals("before", value(original));+ assertEquals(Map.of("team", "old", "dropped", "restore", "external", "keep"), tags(original));+ Map<String, Object> described = describe(original);+ assertEquals("set outside the stack", described.get("Description"));+ assertEquals("^[a-z]+$", described.get("AllowedPattern"));+ assertEquals(Map.of("ParameterName", original), outputs(stack));+ assertEquals(original, physicalIds(stack).get("Parameter"));+ if (rename) {+ assertMissing(target);+ }+ deleteStack(stack);+ assertMissing(original);+ }++ @Test+ void severalParametersRollBackTogether() throws Exception {+ String stack = unique("ssm-several");+ String base = "/" + stack;+ deploy("CreateStack", stack, template(false,+ new Param("InPlace", base + "/in-place", "one"),+ new Param("Renamed", base + "/renamed", "one"),+ new Param("Untouched", base + "/untouched", "one")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());++ deploy("UpdateStack", stack, template(true,+ new Param("InPlace", base + "/in-place", "two"),+ new Param("Renamed", base + "/renamed-new", "two"),+ new Param("Untouched", base + "/untouched", "one"),+ new Param("Added", base + "/added", "two")));++ CfnStackWaits.StackState state = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", state.status(), state.reason());+ assertEquals("one", value(base + "/in-place"));+ assertEquals("one", value(base + "/renamed"));+ assertMissing(base + "/renamed-new");+ assertMissing(base + "/added");+ assertEquals(1L, version(base + "/untouched"));+ assertEquals(Map.of("InPlaceName", base + "/in-place", "RenamedName", base + "/renamed",+ "UntouchedName", base + "/untouched"), outputs(stack));+ deleteStack(stack);+ assertMissing(base + "/in-place");+ assertMissing(base + "/renamed");+ assertMissing(base + "/untouched");+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void aCommittedRenameAppliesItsReplacePolicy(boolean retain) throws Exception {+ String stack = unique("ssm-replace");+ String original = "/" + stack;+ String target = original + "-new";+ deploy("CreateStack", stack, template(false, new Param("Parameter", original, "before", Map.of(), retain)));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());++ deploy("UpdateStack", stack, template(false, new Param("Parameter", target, "after", Map.of(), retain)));++ assertEquals("UPDATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ assertEquals("after", value(target));+ List<String> cleanup = events(stack).stream()+ .filter(event -> original.equals(event.get("PhysicalResourceId")))+ .map(event -> event.get("ResourceStatus"))+ .toList();+ if (retain) {+ assertEquals("before", value(original));+ assertFalse(cleanup.contains("DELETE_IN_PROGRESS"), cleanup.toString());+ } else {+ assertMissing(original);+ assertTrue(cleanup.containsAll(List.of("DELETE_IN_PROGRESS", "DELETE_COMPLETE")), cleanup.toString());+ }+ deleteStack(stack);+ assertMissing(target);+ if (retain) {+ assertEquals("before", value(original));+ ssm("DeleteParameter", Map.of("Name", original)).then().statusCode(200);+ }+ }++ @Test+ void aRenameOntoAParameterHeldOutsideTheStackFailsWithoutAdoptingIt() throws Exception {+ String stack = unique("ssm-collision");+ String original = "/" + stack;+ String target = original + "-foreign";+ deploy("CreateStack", stack, template(false, new Param("Parameter", original, "before")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ ssm("PutParameter", Map.of("Name", target, "Value", "foreign", "Type", "String")).then().statusCode(200);++ deploy("UpdateStack", stack, template(false, new Param("Parameter", target, "after")));++ CfnStackWaits.StackState state = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", state.status(), state.reason());+ assertTrue(updateFailure(stack).contains("already exists"), updateFailure(stack));+ assertEquals("before", value(original));+ assertEquals("foreign", value(target));+ assertEquals(1L, version(target));+ deleteStack(stack);+ assertMissing(original);+ assertEquals("foreign", value(target));+ ssm("DeleteParameter", Map.of("Name", target)).then().statusCode(200);+ }++ @Test+ void aCreateOntoAParameterHeldOutsideTheStackFailsWithoutOverwritingIt() throws Exception {+ String stack = unique("ssm-create-collision");+ String name = "/" + stack;+ ssm("PutParameter", Map.of("Name", name, "Value", "foreign", "Type", "String")).then().statusCode(200);++ deploy("CreateStack", stack, template(false, new Param("Parameter", name, "managed")));++ assertEquals("ROLLBACK_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ assertEquals("foreign", value(name));+ assertEquals(1L, version(name));+ deleteStack(stack);+ assertEquals("foreign", value(name));+ ssm("DeleteParameter", Map.of("Name", name)).then().statusCode(200);+ }++ @Test+ void aSecondUpdateRollsBackToTheLastCommittedValue() throws Exception {+ String stack = unique("ssm-two-updates");+ String name = "/" + stack;+ deploy("CreateStack", stack, template(false, new Param("Parameter", name, "one")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ deploy("UpdateStack", stack, template(false, new Param("Parameter", name, "two")));+ assertEquals("UPDATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());++ deploy("UpdateStack", stack, template(true, new Param("Parameter", name, "three")));++ assertEquals("UPDATE_ROLLBACK_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ assertEquals("two", value(name));+ deleteStack(stack);+ assertMissing(name);+ }++ @Test+ void continueUpdateRollbackRetriesARestoreThatFailed() throws Exception {+ String stack = unique("ssm-retry-restore");+ String name = "/" + stack;+ deploy("CreateStack", stack, template(false, new Param("Parameter", name, "before")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ failRestoringOnce(name, "before");++ deploy("UpdateStack", stack, template(true, new Param("Parameter", name, "after")));++ CfnStackWaits.StackState failed = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_FAILED", failed.status(), failed.reason());+ assertTrue(failed.reason().contains("Parameter"), failed.reason());+ assertEquals("after", value(name));++ continueUpdateRollback(stack, null);++ CfnStackWaits.StackState retried = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", retried.status(), retried.reason());+ assertEquals("before", value(name));+ deleteStack(stack);+ assertMissing(name);+ }++ @Test+ void continueUpdateRollbackRetriesAReplacementDeleteThatFailed() throws Exception {+ String stack = unique("ssm-retry-delete");+ String original = "/" + stack;+ String target = original + "-new";+ deploy("CreateStack", stack, template(false, new Param("Parameter", original, "before")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ failDeleting(target);+ try {+ deploy("UpdateStack", stack, template(true, new Param("Parameter", target, "after")));++ CfnStackWaits.StackState failed = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_FAILED", failed.status(), failed.reason());+ assertEquals(original, physicalIds(stack).get("Parameter"));+ assertEquals("after", value(target));+ } finally {+ Mockito.doCallRealMethod().when(ssmService).deleteParameter(eq(target), anyString());+ }++ continueUpdateRollback(stack, null);++ CfnStackWaits.StackState retried = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", retried.status(), retried.reason());+ assertMissing(target);+ assertEquals("before", value(original));+ deleteStack(stack);+ assertMissing(original);+ }++ /**+ * A skipped resource keeps the state the failed update left it in, and the next update starts+ * from that state: it neither fails over the skipped rollback nor rolls back to before it.+ */+ @Test+ void aSkippedRestoreDoesNotHoldBackTheNextUpdate() throws Exception {+ String stack = unique("ssm-skip-restore");+ String name = "/" + stack;+ deploy("CreateStack", stack, template(false, new Param("Parameter", name, "one")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ failRestoringOnce(name, "one");+ deploy("UpdateStack", stack, template(true, new Param("Parameter", name, "two")));+ assertEquals("UPDATE_ROLLBACK_FAILED", CfnStackWaits.awaitTerminal(stack).status());++ continueUpdateRollback(stack, "Parameter");++ CfnStackWaits.StackState skipped = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", skipped.status(), skipped.reason());+ assertEquals("two", value(name));++ deploy("UpdateStack", stack, template(false, new Param("Parameter", name, "three")));+ CfnStackWaits.StackState updated = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_COMPLETE", updated.status(), updated.reason());+ assertEquals("three", value(name));++ deploy("UpdateStack", stack, template(true, new Param("Parameter", name, "four")));+ assertEquals("UPDATE_ROLLBACK_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ assertEquals("three", value(name));+ deleteStack(stack);+ assertMissing(name);+ }++ @Test+ void aSkippedReplacementDeleteIsLeftToTheNextCommittedUpdate() throws Exception {+ String stack = unique("ssm-skip-delete");+ String original = "/" + stack;+ String target = original + "-new";+ deploy("CreateStack", stack, template(false, new Param("Parameter", original, "one")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ failDeleting(target);+ try {+ deploy("UpdateStack", stack, template(true, new Param("Parameter", target, "two")));+ assertEquals("UPDATE_ROLLBACK_FAILED", CfnStackWaits.awaitTerminal(stack).status());+ continueUpdateRollback(stack, "Parameter");+ CfnStackWaits.StackState skipped = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", skipped.status(), skipped.reason());+ } finally {+ Mockito.doCallRealMethod().when(ssmService).deleteParameter(eq(target), anyString());+ }+ assertEquals(original, physicalIds(stack).get("Parameter"));+ assertEquals("two", value(target));++ deploy("UpdateStack", stack, template(false, new Param("Parameter", original, "three")));++ CfnStackWaits.StackState updated = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_COMPLETE", updated.status(), updated.reason());+ assertEquals("three", value(original));+ assertMissing(target);+ deleteStack(stack);+ assertMissing(original);+ }++ @Test+ void deleteStackAfterAFailedRenameRollbackDeletesBothParameters() throws Exception {+ String stack = unique("ssm-delete-failed");+ String original = "/" + stack;+ String target = original + "-new";+ deploy("CreateStack", stack, template(false, new Param("Parameter", original, "before")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ failDeleting(target);+ try {+ deploy("UpdateStack", stack, template(true, new Param("Parameter", target, "after")));+ assertEquals("UPDATE_ROLLBACK_FAILED", CfnStackWaits.awaitTerminal(stack).status());+ } finally {+ Mockito.doCallRealMethod().when(ssmService).deleteParameter(eq(target), anyString());+ }++ deleteStack(stack);++ assertMissing(original);+ assertMissing(target);+ }++ /** As on AWS the update still completes: the displaced parameter is reported and left in place. */+ @Test+ void aDisplacedParameterThatCannotBeDeletedIsReportedAndAbandoned() throws Exception {+ String stack = unique("ssm-cleanup-failed");+ String original = "/" + stack;+ String target = original + "-new";+ deploy("CreateStack", stack, template(false, new Param("Parameter", original, "before")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ failDeleting(original);+ try {+ deploy("UpdateStack", stack, template(false, new Param("Parameter", target, "after")));++ CfnStackWaits.StackState state = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_COMPLETE", state.status(), state.reason());+ assertEquals("The following resource(s) could not be deleted during update cleanup: [Parameter ("+ + original + ")].", state.reason());+ assertTrue(events(stack).stream().anyMatch(event -> original.equals(event.get("PhysicalResourceId"))+ && "DELETE_FAILED".equals(event.get("ResourceStatus"))));+ } finally {+ Mockito.doCallRealMethod().when(ssmService).deleteParameter(eq(original), anyString());+ }+ assertEquals("before", value(original));++ deploy("UpdateStack", stack, template(false, new Param("Parameter", target, "next")));+ assertEquals("UPDATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ assertEquals("before", value(original));+ deleteStack(stack);+ assertMissing(target);+ assertEquals("before", value(original));+ ssm("DeleteParameter", Map.of("Name", original)).then().statusCode(200);+ }++ @Test+ void deleteStackToleratesAParameterDeletedOutsideTheStack() throws Exception {+ String stack = unique("ssm-gone-delete");+ String name = "/" + stack;+ deploy("CreateStack", stack, template(false, new Param("Parameter", name, "value")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ ssm("DeleteParameter", Map.of("Name", name)).then().statusCode(200);++ deleteStack(stack);+ }++ /**+ * As on AWS, an update in place checks that the parameter exists, so one deleted outside the stack+ * fails the update before anything is written, and the rollback has nothing to undo. A rename+ * never reads the prior parameter, so it succeeds.+ */+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void anUpdateOfAParameterDeletedOutsideTheStack(boolean rename) throws Exception {+ String stack = unique("ssm-gone-update");+ String original = "/" + stack;+ String target = rename ? original + "-new" : original;+ deploy("CreateStack", stack, template(false, new Param("Parameter", original, "before")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ ssm("DeleteParameter", Map.of("Name", original)).then().statusCode(200);++ deploy("UpdateStack", stack, template(false, new Param("Parameter", target, "after")));++ CfnStackWaits.StackState state = CfnStackWaits.awaitTerminal(stack);+ if (rename) {+ assertEquals("UPDATE_COMPLETE", state.status(), state.reason());+ assertEquals("after", value(target));+ } else {+ assertEquals("UPDATE_ROLLBACK_COMPLETE", state.status(), state.reason());+ assertTrue(updateFailure(stack).contains("not found"), updateFailure(stack));+ }+ assertMissing(original);+ deleteStack(stack);+ assertMissing(target);+ }++ /**+ * An update that fails before it writes leaves the stack's resource as it was, its DeletionPolicy+ * included: dropping Retain in the failed template does not let DeleteStack delete the parameter.+ */+ @Test+ void aFailedUpdateOfADeletedParameterKeepsItsDeletionPolicy() throws Exception {+ String stack = unique("ssm-gone-retain");+ String name = "/" + stack;+ deploy("CreateStack", stack, retainTemplate(name, "before", true));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ ssm("DeleteParameter", Map.of("Name", name)).then().statusCode(200);++ deploy("UpdateStack", stack, retainTemplate(name, "after", false));++ CfnStackWaits.StackState state = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", state.status(), state.reason());+ ssm("PutParameter", Map.of("Name", name, "Value", "recreated", "Type", "String")).then().statusCode(200);+ deleteStack(stack);+ assertEquals("recreated", value(name));+ ssm("DeleteParameter", Map.of("Name", name)).then().statusCode(200);+ }++ /**+ * The rollback reads the parameter before it writes, as AWS's update handler does, so one that is+ * gone by then fails the rollback instead of being re-created; the retry restores it once it is back.+ */+ @Test+ void aRollbackThatFindsTheParameterGoneFailsUntilItIsBack() throws Exception {+ String stack = unique("ssm-gone-rollback");+ String name = "/" + stack;+ deploy("CreateStack", stack, template(false, new Param("Parameter", name, "before")));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ // The update's own read sees the parameter; the rollback's read finds it gone.+ Mockito.doCallRealMethod()+ .doThrow(new AwsException("ParameterNotFound", "Parameter " + name + " not found.", 400))+ .when(ssmService).getParameter(eq(name), anyString());+ try {+ deploy("UpdateStack", stack, template(true, new Param("Parameter", name, "after")));++ CfnStackWaits.StackState failed = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_FAILED", failed.status(), failed.reason());+ assertTrue(updateFailure(stack).contains("not found"), updateFailure(stack));+ } finally {+ Mockito.doCallRealMethod().when(ssmService).getParameter(eq(name), anyString());+ }+ assertEquals("after", value(name));++ continueUpdateRollback(stack, null);++ CfnStackWaits.StackState retried = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", retried.status(), retried.reason());+ assertEquals("before", value(name));+ deleteStack(stack);+ assertMissing(name);+ }++ /** The first restore of {@code name} to {@code priorValue} fails; a retry reaches the service. */+ private void failRestoringOnce(String name, String priorValue) {+ Mockito.doThrow(new AwsException("InternalServerError", "simulated restore failure", 500))+ .doCallRealMethod()+ .when(ssmService)+ .putParameter(eq(name), eq(priorValue), any(), any(), eq(true), any(), any(), any(), any(), any(),+ anyString());+ }++ private void failDeleting(String name) {+ Mockito.doThrow(new AwsException("InternalServerError", "simulated delete failure", 500))+ .when(ssmService)+ .deleteParameter(eq(name), anyString());+ }++ private static String template(boolean fail, Param... params) throws JsonProcessingException {+ Map<String, Object> resources = new LinkedHashMap<>();+ Map<String, Object> outputs = new LinkedHashMap<>();+ for (Param param : params) {+ resources.put(param.logicalId(), Map.of("Type", "AWS::SSM::Parameter",+ "UpdateReplacePolicy", param.retain() ? "Retain" : "Delete",+ "Properties", Map.of("Name", param.name(), "Value", param.value(), "Type", "String",+ "Tags", param.tags())));+ outputs.put(param.logicalId() + "Name", Map.of("Value", Map.of("Ref", param.logicalId())));+ }+ if (fail) {+ resources.put("Failure", Map.of("Type", "AWS::SecretsManager::Secret",+ "DependsOn", List.of(params).stream().map(Param::logicalId).toList(),+ "Properties", Map.of("SecretString", "explicit",+ "GenerateSecretString", Map.of("PasswordLength", 32))));+ }+ return MAPPER.writeValueAsString(Map.of("Resources", resources, "Outputs", outputs));+ }++ private static String retainTemplate(String name, String value, boolean retain) throws JsonProcessingException {+ Map<String, Object> parameter = new LinkedHashMap<>();+ parameter.put("Type", "AWS::SSM::Parameter");+ if (retain) {+ parameter.put("DeletionPolicy", "Retain");+ }+ parameter.put("Properties", Map.of("Name", name, "Value", value, "Type", "String"));+ return MAPPER.writeValueAsString(Map.of("Resources", Map.of("Parameter", parameter)));+ }++ private static void deploy(String action, String stack, String template) {+ cfn(action, stack).formParam("TemplateBody", template).post("/").then().statusCode(200);+ }++ private static void continueUpdateRollback(String stack, String skip) {+ RequestSpecification request = cfn("ContinueUpdateRollback", stack);+ if (skip != null) {+ request.formParam("ResourcesToSkip.member.1", skip);+ }+ request.post("/").then().statusCode(200);+ }++ private static void deleteStack(String stack) {+ cfn("DeleteStack", stack).post("/").then().statusCode(200);+ CfnStackWaits.awaitStackDeleted(stack);+ }++ private static Map<String, String> outputs(String stack) {+ String body = cfn("DescribeStacks", stack).post("/").then().statusCode(200).extract().asString();+ return XmlParser.extractPairs(body, "Outputs", "OutputKey", "OutputValue");+ }++ private static Map<String, String> physicalIds(String stack) {+ String body = cfn("DescribeStackResources", stack).post("/").then().statusCode(200).extract().asString();+ return XmlParser.extractPairs(body, "StackResources", "LogicalResourceId", "PhysicalResourceId");+ }++ private static List<Map<String, String>> events(String stack) {+ String body = cfn("DescribeStackEvents", stack).post("/").then().statusCode(200).extract().asString();+ return XmlParser.extractGroups(body, "member");+ }++ /** Why the {@code Parameter} resource failed the update, from its UPDATE_FAILED event. */+ private static String updateFailure(String stack) {+ return events(stack).stream()+ .filter(event -> "Parameter".equals(event.get("LogicalResourceId"))+ && "UPDATE_FAILED".equals(event.get("ResourceStatus")))+ .map(event -> event.get("ResourceStatusReason"))+ .findFirst()+ .orElse("");+ }++ private static RequestSpecification cfn(String action, String stack) {+ return given().contentType("application/x-www-form-urlencoded")+ .formParam("Action", action).formParam("StackName", stack);+ }++ private static Response ssm(String operation, Map<String, ?> body) throws JsonProcessingException {+ return given().contentType("application/x-amz-json-1.1")+ .header("X-Amz-Target", "AmazonSSM." + operation).body(MAPPER.writeValueAsString(body)).post("/");+ }++ private static String value(String name) throws JsonProcessingException {+ return ssm("GetParameter", Map.of("Name", name)).then().statusCode(200)+ .extract().jsonPath().getString("Parameter.Value");+ }++ private static long version(String name) throws JsonProcessingException {+ return ssm("GetParameter", Map.of("Name", name)).then().statusCode(200)+ .extract().jsonPath().getLong("Parameter.Version");+ }++ private static Map<String, String> tags(String name) throws JsonProcessingException {+ List<Map<String, String>> tags = ssm("ListTagsForResource",+ Map.of("ResourceType", "Parameter", "ResourceId", name))+ .then().statusCode(200).extract().jsonPath().getList("TagList");+ return tags.stream().collect(Collectors.toMap(tag -> tag.get("Key"), tag -> tag.get("Value")));+ }++ private static Map<String, Object> describe(String name) throws JsonProcessingException {+ List<Map<String, Object>> parameters = ssm("DescribeParameters", Map.of("ParameterFilters",+ List.of(Map.of("Key", "Name", "Option", "Equals", "Values", List.of(name)))))+ .then().statusCode(200).extract().jsonPath().getList("Parameters");+ assertEquals(1, parameters.size(), parameters.toString());+ return parameters.get(0);+ }++ private static void assertMissing(String name) throws JsonProcessingException {+ Response response = ssm("GetParameter", Map.of("Name", name));+ assertEquals(400, response.statusCode(), response.asString());+ assertTrue(response.asString().contains("ParameterNotFound"), response.asString());+ }++ private static String unique(String prefix) {+ return prefix + "-" + Long.toString(System.nanoTime(), 36);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationUpdateRollbackParametersTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationUpdateRollbackParametersTest.javaindex da694aa683..a7aa0fd4e1 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationUpdateRollbackParametersTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationUpdateRollbackParametersTest.java@@ -129,7 +129,7 @@ class CloudFormationUpdateRollbackParametersTest {void failedUpdateWithFailedResourceRollback_stillRetainsLastSuccessfulParameterValues() {String suffix = Long.toString(System.nanoTime(), 36);String stackName = "rollback-failed-params-stack-" + suffix;- String parameterName = "/cfn/test/rollback-failed-params-" + suffix;+ String alarmName = "rollback-failed-params-alarm-" + suffix;String template = """{"Parameters": {@@ -137,19 +137,21 @@ class CloudFormationUpdateRollbackParametersTest {},"Resources": {"Parameter": {- "Type": "AWS::SSM::Parameter",+ "Type": "AWS::CloudWatch::Alarm","Properties": {- "Name": "%s",- "Type": "String",- "Value": {"Fn::Sub": "value-${Suffix}"}+ "AlarmName": "%s",+ "Namespace": "RollbackTest", "MetricName": "Sample", "Statistic": "Average",+ "Period": 60, "EvaluationPeriods": 1, "Threshold": 1,+ "ComparisonOperator": "GreaterThanThreshold",+ "AlarmDescription": {"Fn::Sub": "value-${Suffix}"}}}%s}}""";- String initialTemplate = template.formatted(parameterName, "");- String failingTemplate = template.formatted(parameterName, """+ String initialTemplate = template.formatted(alarmName, "");+ String failingTemplate = template.formatted(alarmName, ""","BadSecret": {"Type": "AWS::SecretsManager::Secret",@@ -174,9 +176,8 @@ class CloudFormationUpdateRollbackParametersTest {.then().statusCode(200);- // The update changes Suffix (which the Parameter resource's Value depends on) and also- // introduces BadSecret, which fails to create. Rollback must then also fail: rollback isn't- // implemented for AWS::SSM::Parameter, so the stack lands in UPDATE_ROLLBACK_FAILED.+ // This fixture deliberately uses the still-unsupported metric-alarm rollback to exercise+ // parameter restoration when another resource prevents rollback from completing.given().contentType("application/x-www-form-urlencoded").formParam("Action", "UpdateStack")diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.javaindex 57955b4873..d2ad4adf15 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.java@@ -160,7 +160,8 @@ class LeafCfnProvisionerTest {{"Name": "/app/db", "Value": "secret", "Type": "String"}"""), ctx);- verify(ssm).putParameter("/app/db", "secret", "String", null, true, REGION);+ // A create never overwrites: a parameter already under that name is not the stack's.+ verify(ssm).putParameter("/app/db", "secret", "String", null, false, Map.of(), REGION);assertEquals("/app/db", r.getPhysicalId());assertEquals(Map.of("Name", "/app/db", "Type", "String", "Value", "secret","Arn", "arn:aws:ssm:us-east-1:000000000000:parameter/app/db"), r.getAttributes());@@ -186,7 +187,8 @@ class LeafCfnProvisionerTest {assertTrue(r.getPhysicalId().startsWith("my-stack-Param-"),"generated names stay stack-scoped: " + r.getPhysicalId());// Absent Value and Type fall back rather than reaching the service as null.- verify(ssm).putParameter(anyString(), eq(""), eq("String"), eq(null), eq(true), eq(REGION));+ verify(ssm).putParameter(anyString(), eq(""), eq("String"), eq(null), eq(false), eq(Map.of()),+ eq(REGION));}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/SsmCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/SsmCfnProvisionerTest.javaindex 96e7ebd238..2d190ebf14 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/SsmCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/SsmCfnProvisionerTest.java@@ -3,37 +3,51 @@ package io.github.hectorvent.floci.services.cloudformation.provisioners;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.ssm.SsmService;+import io.github.hectorvent.floci.services.ssm.model.Parameter;import org.junit.jupiter.api.Test;+import org.mockito.InOrder;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.Set;+import java.util.stream.Collectors;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyBoolean;import static org.mockito.ArgumentMatchers.anyMap;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.argThat;import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.inOrder;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;/**- * The SSM parameter CFN provisioner in isolation, for the {@code Tags} property: the parameter is- * written without tags, then the template's tags are added, and an update removes only the keys the- * previous template had, keeping tags added outside the template.+ * The SSM parameter CFN provisioner in isolation. A create carries the template's tags in a put+ * that never overwrites; an update in place removes only the keys the previous template had,+ * keeping tags added outside the template; and a failed stack update puts back what an update in+ * place or a rename changed.*/class SsmCfnProvisionerTest {private static final String REGION = "us-east-1";private static final String NAME = "/app/param";+ private static final String RENAMED = NAME + "-new";private static final String TEMPLATE_TAG_KEYS_ATTR = "FlociSsmTemplateTagKeys";+ private static final String ARN = "arn:aws:ssm:us-east-1:000000000000:parameter/app/param";private final SsmService ssm = mock(SsmService.class);private final SsmCfnProvisioner provisioner = new SsmCfnProvisioner(ssm);@@ -50,12 +64,30 @@ class SsmCfnProvisionerTest {return new ProvisionContext(engine, REGION, "000000000000", "my-stack");}- private ProvisionContext updateCtx() {+ /** An update of the parameter the stack holds as {@link #NAME}, which SSM holds as {@code prior}. */+ private ProvisionContext updateCtx(Parameter prior) {+ when(ssm.getParameter(NAME, REGION)).thenReturn(prior);ProvisionContext create = ctx();return new ProvisionContext(create.engine(), create.region(), create.accountId(),create.stackName(), NAME);}+ private ProvisionContext updateCtx() {+ return updateCtx(stored("old", Map.of()));+ }++ /** SSM now holds what the update in place wrote: its value, with the settings the write resets. */+ private void afterTheUpdate(Map<String, String> tags) {+ when(ssm.getParameter(NAME, REGION)).thenReturn(stored("v", tags));+ }++ private static Parameter stored(String value, Map<String, String> tags) {+ Parameter parameter = new Parameter(NAME, value, "String");+ parameter.setTags(new HashMap<>(tags));+ return parameter;+ }++ /** The resource as the stack holds it: created with value {@code old} and the given template tag keys. */private static StackResource parameter(String physicalId, String templateTagKeys) {StackResource r = new StackResource();r.setLogicalId("MyParameter");@@ -63,6 +95,7 @@ class SsmCfnProvisionerTest {r.setAttributes(new HashMap<>());if (physicalId != null) {r.setPhysicalId(physicalId);+ r.getAttributes().putAll(Map.of("Name", physicalId, "Type", "String", "Value", "old", "Arn", ARN));}if (templateTagKeys != null) {r.getAttributes().put(TEMPLATE_TAG_KEYS_ATTR, templateTagKeys);@@ -84,15 +117,22 @@ class SsmCfnProvisionerTest {return props;}+ private static Map<String, String> visibleAttributes(StackResource r) {+ return r.getAttributes().entrySet().stream()+ .filter(entry -> !entry.getKey().startsWith("__Floci"))+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));+ }+@Test- void parameterTagsFromTheTemplateAreAddedAfterThePut() {+ void aCreateCarriesTheTemplateTagsInAPutThatDoesNotOverwrite() {StackResource r = parameter(null, null);provisioner.provision(r, props("team", "a", "env", "dev"), ctx());- verify(ssm).putParameter(NAME, "v", "String", null, true, REGION);- verify(ssm).addTagsToResource(NAME, Map.of("team", "a", "env", "dev"), REGION);+ verify(ssm).putParameter(NAME, "v", "String", null, false, Map.of("team", "a", "env", "dev"), REGION);+ verify(ssm, never()).addTagsToResource(anyString(), anyMap(), anyString());verify(ssm, never()).removeTagsFromResource(anyString(), any(), anyString());+ verify(ssm, never()).getParameter(anyString(), anyString());assertEquals("env,team", r.getAttributes().get(TEMPLATE_TAG_KEYS_ATTR));}@@ -102,7 +142,7 @@ class SsmCfnProvisionerTest {provisioner.provision(r, props(), ctx());- verify(ssm).putParameter(NAME, "v", "String", null, true, REGION);+ verify(ssm).putParameter(NAME, "v", "String", null, false, Map.of(), REGION);verify(ssm, never()).addTagsToResource(anyString(), anyMap(), anyString());verify(ssm, never()).removeTagsFromResource(anyString(), any(), anyString());assertFalse(r.getAttributes().containsKey(TEMPLATE_TAG_KEYS_ATTR));@@ -114,6 +154,7 @@ class SsmCfnProvisionerTest {provisioner.provision(r, props("team", "b"), updateCtx());+ verify(ssm).putParameter(NAME, "v", "String", null, true, REGION);verify(ssm).removeTagsFromResource(NAME, List.of("env"), REGION);verify(ssm).addTagsToResource(NAME, Map.of("team", "b"), REGION);assertEquals("team", r.getAttributes().get(TEMPLATE_TAG_KEYS_ATTR));@@ -138,4 +179,230 @@ class SsmCfnProvisionerTest {verify(ssm, never()).removeTagsFromResource(anyString(), any(), anyString());verify(ssm).addTagsToResource(NAME, Map.of("team", "b"), REGION);}++ /** The snapshot carries the prior value, so it lives under a key Fn::GetAtt cannot name. */+ @Test+ void anUpdateInPlaceKeepsWhatItChangesBeforeItWrites() {+ StackResource r = parameter(NAME, null);++ provisioner.provision(r, props(), updateCtx());++ InOrder order = inOrder(ssm);+ order.verify(ssm).getParameter(NAME, REGION);+ order.verify(ssm).putParameter(NAME, "v", "String", null, true, REGION);+ assertTrue(provisioner.retainsFailedUpdateState(r));+ assertFalse(provisioner.hasReplacementUpdate(r));+ assertEquals(Map.of("Name", NAME, "Type", "String", "Value", "v", "Arn", ARN), visibleAttributes(r));+ }++ @Test+ void rollingBackAnUpdateInPlaceRestoresTheParameterItsTagsAndItsAttributes() {+ Parameter prior = stored("old", Map.of("team", "a", "env", "dev", "external", "x"));+ prior.setDescription("described outside the stack");+ prior.setAllowedPattern("^[a-z]+$");+ prior.setTier("Advanced");+ List<JsonNode> policies = List.of(mapper.createObjectNode().put("Type", "Expiration"));+ prior.setPolicies(policies);+ StackResource r = parameter(NAME, "env,team");+ provisioner.provision(r, props("team", "b", "added", "y"), updateCtx(prior));+ afterTheUpdate(Map.of("team", "b", "env", "dev", "added", "y", "external", "x"));++ assertTrue(provisioner.rollbackUpdate(r));++ verify(ssm).putParameter(NAME, "old", "String", "described outside the stack", true, null, null,+ "^[a-z]+$", "Advanced", policies, REGION);+ verify(ssm).removeTagsFromResource(NAME, List.of("added"), REGION);+ verify(ssm).addTagsToResource(NAME, Map.of("env", "dev", "team", "a"), REGION);+ verify(ssm, never()).addTagsToResource(eq(NAME), argThat(tags -> tags.containsKey("external")), eq(REGION));+ assertEquals(NAME, r.getPhysicalId());+ assertEquals(Map.of("Name", NAME, "Type", "String", "Value", "old", "Arn", ARN,+ TEMPLATE_TAG_KEYS_ATTR, "env,team"), visibleAttributes(r));+ assertFalse(provisioner.retainsFailedUpdateState(r));+ }++ @Test+ void aRestoreThatFailsKeepsTheSnapshotForTheRetry() {+ StackResource r = parameter(NAME, null);+ provisioner.provision(r, props(), updateCtx());+ afterTheUpdate(Map.of());+ when(ssm.putParameter(eq(NAME), eq("old"), eq("String"), any(), eq(true), any(), any(), any(), any(),+ any(), eq(REGION)))+ .thenThrow(new AwsException("InternalServerError", "simulated", 500))+ .thenReturn(3L);++ assertThrows(AwsException.class, () -> provisioner.rollbackUpdate(r));+ assertTrue(provisioner.retainsFailedUpdateState(r));+ assertEquals("v", r.getAttributes().get("Value"));++ assertTrue(provisioner.rollbackUpdate(r));+ assertFalse(provisioner.retainsFailedUpdateState(r));+ assertEquals("old", r.getAttributes().get("Value"));+ }++ /**+ * A snapshot an earlier update left, as a skipped ContinueUpdateRollback does, describes no+ * update in flight: a provision that fails before it writes leaves nothing to restore, and one+ * that writes snapshots the parameter as it is now.+ */+ @Test+ void aSnapshotAnEarlierUpdateLeftIsDroppedBeforeAnythingElse() {+ StackResource r = parameter(NAME, null);+ provisioner.provision(r, props(), updateCtx(stored("stale", Map.of())));+ StackResource skipped = parameter(NAME, null);+ skipped.getAttributes().put(CfnRollback.SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR,+ r.getAttributes().get(CfnRollback.SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR));++ StackResource refused = parameter(NAME, null);+ refused.getAttributes().putAll(skipped.getAttributes());+ assertThrows(AwsException.class, () -> provisioner.provision(refused, props("", "x"), updateCtx()));+ assertFalse(provisioner.retainsFailedUpdateState(refused));++ provisioner.provision(skipped, props(), updateCtx(stored("current", Map.of())));+ afterTheUpdate(Map.of());+ assertTrue(provisioner.rollbackUpdate(skipped));+ verify(ssm).putParameter(eq(NAME), eq("current"), eq("String"), any(), eq(true), any(), any(), any(),+ any(), any(), eq(REGION));+ verify(ssm, never()).putParameter(eq(NAME), eq("stale"), anyString(), any(), anyBoolean(), any(), any(),+ any(), any(), any(), anyString());+ }++ /** AWS's handler checks the parameter exists before an update in place writes it. */+ @Test+ void anUpdateInPlaceOfAParameterDeletedOutsideTheStackFailsBeforeItWrites() {+ when(ssm.getParameter(NAME, REGION)).thenThrow(new AwsException("ParameterNotFound", "gone", 400));+ StackResource r = parameter(NAME, null);+ ProvisionContext create = ctx();+ ProvisionContext update = new ProvisionContext(create.engine(), create.region(), create.accountId(),+ create.stackName(), NAME);++ AwsException failure = assertThrows(AwsException.class, () -> provisioner.provision(r, props(), update));++ assertEquals("ParameterNotFound", failure.getErrorCode());+ verify(ssm, never()).putParameter(anyString(), anyString(), anyString(), any(), anyBoolean(), anyString());+ assertFalse(provisioner.retainsFailedUpdateState(r));+ }++ /** A parameter deleted after the update wrote it is not re-created by the rollback, as on AWS. */+ @Test+ void aRestoreOfAParameterDeletedAfterTheUpdateFailsAndKeepsTheSnapshot() {+ StackResource r = parameter(NAME, null);+ provisioner.provision(r, props(), updateCtx());+ when(ssm.getParameter(NAME, REGION)).thenThrow(new AwsException("ParameterNotFound", "gone", 400));++ assertEquals("ParameterNotFound",+ assertThrows(AwsException.class, () -> provisioner.rollbackUpdate(r)).getErrorCode());++ verify(ssm, never()).putParameter(anyString(), anyString(), anyString(), any(), anyBoolean(), any(), any(),+ any(), any(), any(), anyString());+ assertTrue(provisioner.retainsFailedUpdateState(r));+ }++ /** An update that failed before its write left nothing to write back, so the rollback adds no version. */+ @Test+ void aRestoreWritesNothingWhenTheParameterStillHoldsItsPriorState() {+ StackResource r = parameter(NAME, "team");+ provisioner.provision(r, props("team", "b"), updateCtx(stored("old", Map.of("team", "a"))));++ assertTrue(provisioner.rollbackUpdate(r));++ verify(ssm, never()).putParameter(anyString(), anyString(), anyString(), any(), anyBoolean(), any(), any(),+ any(), any(), any(), anyString());+ verify(ssm).addTagsToResource(NAME, Map.of("team", "a"), REGION);+ }++ /** A stub created nothing, so the parameter that takes over its id is created, not updated. */+ @Test+ void aParameterThatTakesOverAStubIdIsCreated() {+ String stubId = "MyParameter-1a2b3c4d";+ StackResource r = parameter(null, null);+ r.setPhysicalId(stubId);+ r.getAttributes().put("Arn", CfnResourceDispatcher.STUB_ARN_PREFIX + "MyParameter");+ ProvisionContext create = ctx();+ ProvisionContext update = new ProvisionContext(create.engine(), create.region(), create.accountId(),+ create.stackName(), stubId);+ ObjectNode unnamed = props("team", "a");+ unnamed.remove("Name");++ provisioner.provision(r, unnamed, update);++ verify(ssm, never()).getParameter(anyString(), anyString());+ verify(ssm).putParameter(stubId, "v", "String", null, false, Map.of("team", "a"), REGION);+ assertFalse(provisioner.retainsFailedUpdateState(r));+ assertTrue(provisioner.rollbackUpdate(r));+ verify(ssm).deleteParameter(stubId, REGION);+ assertEquals(stubId, r.getPhysicalId());+ }++ @Test+ void aCommittedUpdateInPlaceSpendsItsSnapshot() {+ StackResource r = parameter(NAME, null);+ provisioner.provision(r, props(), updateCtx());++ assertFalse(provisioner.completeUpdate(r).applicable());++ assertFalse(provisioner.retainsFailedUpdateState(r));+ assertFalse(r.getAttributes().containsKey(CfnRollback.SSM_PARAMETER_UPDATE_SNAPSHOT_ATTR));+ }++ /** A rename never writes the prior parameter, so it neither reads it nor needs it to exist. */+ @Test+ void aRenameCreatesTheReplacementWithoutReadingOrOverwriting() {+ StackResource r = parameter(NAME, "team");++ provisioner.provision(r, props("team", "b").put("Name", RENAMED), updateCtx());++ verify(ssm, never()).getParameter(anyString(), anyString());+ verify(ssm).putParameter(RENAMED, "v", "String", null, false, Map.of("team", "b"), REGION);+ verify(ssm, never()).removeTagsFromResource(anyString(), any(), anyString());+ assertEquals(RENAMED, r.getPhysicalId());+ assertTrue(provisioner.hasReplacementUpdate(r));+ assertEquals(NAME, provisioner.updateCleanupPhysicalId(r));+ assertFalse(provisioner.retainsFailedUpdateState(r));+ }++ @Test+ void rollingBackARenameDeletesTheReplacementAndNamesThePriorParameterAgain() {+ StackResource r = parameter(NAME, "team");+ provisioner.provision(r, props("team", "b").put("Name", RENAMED), updateCtx());++ assertTrue(provisioner.rollbackUpdate(r));++ verify(ssm).deleteParameter(RENAMED, REGION);+ verify(ssm, never()).deleteParameter(NAME, REGION);+ assertEquals(NAME, r.getPhysicalId());+ assertEquals(Map.of("Name", NAME, "Type", "String", "Value", "old", "Arn", ARN,+ TEMPLATE_TAG_KEYS_ATTR, "team"), visibleAttributes(r));+ assertFalse(provisioner.hasReplacementUpdate(r));+ }++ @Test+ void aCommittedRenameDeletesTheDisplacedParameter() {+ StackResource r = parameter(NAME, null);+ provisioner.provision(r, props().put("Name", RENAMED), updateCtx());++ UpdateCleanupResult cleanup = provisioner.completeUpdate(r);+ provisioner.clearUpdate(r);++ assertTrue(cleanup.complete());+ verify(ssm).deleteParameter(NAME, REGION);+ assertFalse(provisioner.hasReplacementUpdate(r));+ assertNull(provisioner.updateCleanupPhysicalId(r));+ }++ @Test+ void deleteToleratesOnlyAParameterThatIsAlreadyGone() {+ doThrow(new AwsException("ParameterNotFound", "gone", 400)).when(ssm).deleteParameter(NAME, REGION);+ provisioner.delete("AWS::SSM::Parameter", NAME, REGION);++ doThrow(new AwsException("AccessDeniedException", "denied", 400)).when(ssm).deleteParameter(NAME, REGION);+ assertThrows(AwsException.class, () -> provisioner.delete("AWS::SSM::Parameter", NAME, REGION));+ }++ @Test+ void aResourceWithNothingRecordedHasNoRollback() {+ StackResource r = parameter(NAME, null);++ assertFalse(provisioner.rollbackUpdate(r));+ assertEquals(Set.of("Name", "Type", "Value", "Arn"), r.getAttributes().keySet());+ }}