Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudformation): roll back a changed custom resource (#4941)

Mario Toffiae8a300f4d55b
diff --git a/docs/services/cloudformation.md b/docs/services/cloudformation.md
index 65996f1869..bdb5a55d66 100644
--- a/docs/services/cloudformation.md
+++ b/docs/services/cloudformation.md
@@ -416,7 +416,24 @@ Lambda. floci supports two shapes:
default); a resource that never completes fails the stack rather than hanging.
The handler's `Data` keys resolve through `Fn::GetAtt`, except keys that begin with `__Floci`. Floci
-reserves that prefix for its own resource state, which no template can read.
+reserves that prefix for its own resource state and ignores such keys.
+
+A failed stack update rolls back a custom resource whose handler was sent an `Update`, as
+CloudFormation does. This holds whether the handler applied the update and a later resource failed,
+or the handler answered `FAILED` itself. The handler is sent a second `Update` under the same
+`PhysicalResourceId`, with the old properties as `ResourceProperties` and the attempted ones as
+`OldResourceProperties`, so it can undo the change. A handler that answers `FAILED` to that rollback
+leaves the resource `UPDATE_FAILED` with its `Reason`, and the stack `UPDATE_ROLLBACK_FAILED`. The
+old properties still become the resource's own, as on AWS: a later `DeleteStack` sends them in its
+`Delete`, and the next update sends them as `OldResourceProperties`. An update whose handler returned
+a new `PhysicalResourceId` replaced the resource. Its rollback sends no `Update`: the new id gets a
+`Delete` with the attempted properties, and the resource goes back to the old id. CloudFormation sends
+that `Delete` in its `UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS` phase. Floci has no rollback
+cleanup phase and sends it during the rollback. As in that phase, a `Delete` the handler fails is
+logged and does not fail the rollback. When the rollback's own `Update` returns a new id, the id it
+was sent gets a `Delete` straight away for the same reason. An update that changes `ServiceToken`
+reaches no handler: it fails with `Modifying service token is not allowed.`, as on AWS, and its
+rollback sends nothing.
## Deleted Stacks
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java
index 247ca84ba2..853dd49614 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java
@@ -70,6 +70,15 @@ public final class CfnRollback {
*/
public static final String EVENT_INVOKE_CONFIG_SNAPSHOT_ATTR = "__FlociEventInvokeConfigSnapshot";
+ /**
+ * Holds what a custom resource had before an update sent its handler an {@code Update}: the
+ * physical id, the old and the attempted properties, the region and the {@code Data} attributes,
+ * so a failed stack update can send the handler the old properties back, whether the handler
+ * applied the update or failed it. Written by {@code CustomResourceCfnProvisioner} before the
+ * invoke and spent by its {@code rollbackUpdate}.
+ */
+ public static final String CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR = "__FlociCustomResourceUpdateSnapshot";
+
/**
* Holds the body and tags a dashboard carried before an in-place update changed them, or the
* fact that it did not exist, so a failed stack update can put it back. Written by
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CustomResourceCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CustomResourceCfnProvisioner.java
index a458fc77a7..ae0f55bcb7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CustomResourceCfnProvisioner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CustomResourceCfnProvisioner.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.cloudformation.provisioners;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -9,6 +10,7 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.docker.ContainerReachableEndpoint;
import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;
import io.github.hectorvent.floci.services.cloudformation.CustomResourceResponseStore;
+import io.github.hectorvent.floci.services.cloudformation.model.StackEvent;
import io.github.hectorvent.floci.services.cloudformation.model.StackResource;
import io.github.hectorvent.floci.services.lambda.LambdaService;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
@@ -22,6 +24,7 @@ import java.time.Duration;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
/**
* Provisions {@code AWS::CloudFormation::CustomResource} and every {@code Custom::*} type, moved out
@@ -46,6 +49,8 @@ public class CustomResourceCfnProvisioner implements CfnResourceProvisioner {
private static final String CR_SERVICE_TOKEN_ATTR = "__FlociServiceToken";
private static final String CR_PROPERTIES_ATTR = "__FlociResourceProperties";
private static final String CR_STACK_ID_ATTR = "__FlociStackId";
+ /** Prefix of every attribute Floci keeps for itself; the others are the handler's {@code Data}. */
+ private static final String RESERVED_ATTR_PREFIX = "__Floci";
/**
* How long to wait for the Lambda's ResponseURL callback after the synchronous invoke returns.
* The invoke already blocks until the handler finishes, so this only covers a PUT that lands
@@ -75,6 +80,8 @@ public class CustomResourceCfnProvisioner implements CfnResourceProvisioner {
@Override
public void provision(StackResource r, JsonNode props, ProvisionContext ctx) {
+ // A snapshot describes the update in flight; one an earlier update left behind is stale.
+ r.getAttributes().remove(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR);
CloudFormationTemplateEngine engine = ctx.engine();
String region = ctx.region();
if (props == null || !props.has("ServiceToken")) {
@@ -116,17 +123,23 @@ public class CustomResourceCfnProvisioner implements CfnResourceProvisioner {
return;
}
+ // CloudFormation fails this update before sending anything, so no handler owes a rollback.
+ String priorServiceToken = isUpdate ? r.getAttributes().get(CR_SERVICE_TOKEN_ATTR) : null;
+ if (priorServiceToken != null && !priorServiceToken.equals(serviceToken)) {
+ throw new AwsException("ValidationError", "Modifying service token is not allowed.", 400);
+ }
+
String stackId = stackIdOrMinted(engine.stackId(), region, ctx.accountId(), ctx.stackName());
+ // Kept before the handler runs, so the rollback of a failed Update sends the same stack id.
+ r.getAttributes().put(CR_STACK_ID_ATTR, stackId);
+ // A resource stashed without its properties has none to roll back to.
+ if (oldResourceProperties != null) {
+ snapshotBeforeUpdate(r, oldResourceProperties, resourceProperties, region);
+ }
JsonNode response = invokeCustomResourceHandler(serviceToken, requestType, r.getLogicalId(),
r.getResourceType(), priorPhysicalId, resourceProperties, oldResourceProperties,
region, stackId);
-
- String status = response.path("Status").asText("FAILED");
- if (!"SUCCESS".equals(status)) {
- throw new AwsException("CustomResourceFailed",
- "Custom resource handler reported FAILED: "
- + response.path("Reason").asText("(no reason given)"), 400);
- }
+ requireSuccess(response);
String returnedPhysicalId = response.path("PhysicalResourceId").asText(null);
if (returnedPhysicalId != null && !returnedPhysicalId.isBlank()) {
@@ -138,27 +151,25 @@ public class CustomResourceCfnProvisioner implements CfnResourceProvisioner {
}
// Data.* become Fn::GetAtt attributes on the custom resource.
- JsonNode data = response.path("Data");
- if (data.isObject()) {
- data.fields().forEachRemaining(e ->
- r.getAttributes().put(e.getKey(), nodeToAttributeValue(e.getValue())));
- }
+ putData(r, response.path("Data"));
// Stash what a later Delete invocation needs (delete() only gets the StackResource).
r.getAttributes().put(CR_SERVICE_TOKEN_ATTR, serviceToken);
r.getAttributes().put(CR_PROPERTIES_ATTR, resourceProperties.toString());
- r.getAttributes().put(CR_STACK_ID_ATTR, stackId);
}
@Override
public void delete(StackResource r, String region) {
+ ObjectNode stashed = readStashedProperties(r);
+ deleteWith(r, stashed != null ? stashed : objectMapper.createObjectNode(), region);
+ }
+
+ private void deleteWith(StackResource r, ObjectNode resourceProperties, String region) {
String serviceToken = r.getAttributes().get(CR_SERVICE_TOKEN_ATTR);
if (serviceToken == null || serviceToken.isBlank()) {
LOG.debugv("Custom resource {0} has no stored ServiceToken; skipping Delete", r.getLogicalId());
return;
}
- ObjectNode stashed = readStashedProperties(r);
- ObjectNode resourceProperties = stashed != null ? stashed : objectMapper.createObjectNode();
// A resource stashed before the stack id was recorded has none, and delete() has no stack.
String stackId = stackIdOrMinted(r.getAttributes().get(CR_STACK_ID_ATTR), region,
accountFromArn(serviceToken), "");
@@ -175,6 +186,118 @@ public class CustomResourceCfnProvisioner implements CfnResourceProvisioner {
}
}
+ @Override
+ public boolean rollbackUpdate(StackResource resource) {
+ return rollbackUpdate(resource, event -> { });
+ }
+
+ /**
+ * An Update that reached the handler and then failed keeps its copy for {@link #rollbackUpdate},
+ * because CloudFormation sends that handler the old properties back too.
+ */
+ @Override
+ public boolean retainsFailedUpdateState(StackResource resource) {
+ return resource.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR);
+ }
+
+ /**
+ * Undoes an update that reached the handler, as CloudFormation does, whether the handler applied
+ * it or failed it. An update that replaced the resource gets a Delete for the replacement, and
+ * the resource points back at the one it displaced. One that kept the physical id gets a second
+ * Update carrying the old properties, with the attempted ones as OldResourceProperties. A
+ * handler that fails that Update fails the rollback, and the snapshot stays for another attempt.
+ */
+ @Override
+ public boolean rollbackUpdate(StackResource resource, Consumer<StackEvent> progress) {
+ String raw = resource.getAttributes().get(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR);
+ if (raw == null) {
+ return true;
+ }
+ JsonNode snapshot;
+ try {
+ snapshot = objectMapper.readTree(raw);
+ } catch (JsonProcessingException e) {
+ throw new IllegalStateException("Could not read the custom resource update snapshot for "
+ + resource.getLogicalId(), e);
+ }
+ String priorPhysicalId = snapshot.path("physicalId").asText();
+ ObjectNode oldProperties = (ObjectNode) snapshot.get("properties");
+ ObjectNode attemptedProperties = (ObjectNode) snapshot.get("attempted");
+ String region = snapshot.path("region").asText();
+ String currentPhysicalId = resource.getPhysicalId();
+ // CloudFormation holds the rollback target as the resource's properties even when the handler
+ // fails it: a later Delete, or the next update's OldResourceProperties, carries them.
+ resource.getAttributes().put(CR_PROPERTIES_ATTR, oldProperties.toString());
+ if (!priorPhysicalId.equals(currentPhysicalId)) {
+ deleteWith(resource, attemptedProperties, region);
+ resource.setPhysicalId(priorPhysicalId);
+ replaceData(resource, snapshot.path("attributes"));
+ } else {
+ ProvisionContext.report(progress, currentPhysicalId, "UPDATE_IN_PROGRESS", null);
+ String serviceToken = resource.getAttributes().get(CR_SERVICE_TOKEN_ATTR);
+ String stackId = stackIdOrMinted(resource.getAttributes().get(CR_STACK_ID_ATTR), region,
+ accountFromArn(serviceToken), "");
+ JsonNode response = invokeCustomResourceHandler(serviceToken, "Update", resource.getLogicalId(),
+ resource.getResourceType(), currentPhysicalId, oldProperties, attemptedProperties,
+ region, stackId);
+ requireSuccess(response);
+ String returnedPhysicalId = response.path("PhysicalResourceId").asText(null);
+ if (returnedPhysicalId != null && !returnedPhysicalId.isBlank()
+ && !returnedPhysicalId.equals(currentPhysicalId)) {
+ // Floci has no rollback cleanup phase, so the id the rollback displaced goes now.
+ deleteWith(resource, attemptedProperties, region);
+ resource.setPhysicalId(returnedPhysicalId);
+ }
+ replaceData(resource, response.path("Data"));
+ }
+ resource.getAttributes().remove(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR);
+ return true;
+ }
+
+ /**
+ * Keeps what the resource has before an Update reaches its handler, so {@link #rollbackUpdate}
+ * can send the old properties back: the physical id, the old and the attempted properties, the
+ * region and the {@code Data} attributes.
+ */
+ private void snapshotBeforeUpdate(StackResource r, ObjectNode oldResourceProperties,
+ ObjectNode attemptedProperties, String region) {
+ ObjectNode snapshot = objectMapper.createObjectNode();
+ snapshot.put("physicalId", r.getPhysicalId());
+ snapshot.set("properties", oldResourceProperties);
+ snapshot.set("attempted", attemptedProperties);
+ snapshot.put("region", region);
+ ObjectNode attributes = snapshot.putObject("attributes");
+ r.getAttributes().forEach((key, value) -> {
+ if (!key.startsWith(RESERVED_ATTR_PREFIX)) {
+ attributes.put(key, value);
+ }
+ });
+ r.getAttributes().put(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR, snapshot.toString());
+ }
+
+ private static void requireSuccess(JsonNode response) {
+ if (!"SUCCESS".equals(response.path("Status").asText("FAILED"))) {
+ throw new AwsException("CustomResourceFailed",
+ "Custom resource handler reported FAILED: "
+ + response.path("Reason").asText("(no reason given)"), 400);
+ }
+ }
+
+ private static void putData(StackResource r, JsonNode data) {
+ if (data.isObject()) {
+ data.fields().forEachRemaining(e -> {
+ if (!e.getKey().startsWith(RESERVED_ATTR_PREFIX)) {
+ r.getAttributes().put(e.getKey(), nodeToAttributeValue(e.getValue()));
+ }
+ });
+ }
+ }
+
+ private static void replaceData(StackResource r, JsonNode data) {
+ r.getAttributes().keySet().removeIf(key -> !key.startsWith(RESERVED_ATTR_PREFIX));
+ putData(r, data);
+ }
+
// Reads the ResourceProperties stashed at the last create/update (CR_PROPERTIES_ATTR).
// Returns null when nothing is stashed or it cannot be parsed.
private ObjectNode readStashedProperties(StackResource r) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationCustomResourceRollbackIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationCustomResourceRollbackIntegrationTest.java
new file mode 100644
index 0000000000..60d915524d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationCustomResourceRollbackIntegrationTest.java
@@ -0,0 +1,284 @@
+package io.github.hectorvent.floci.services.cloudformation;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.XmlParser;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.InjectMock;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.when;
+
+/**
+ * A stack update that changes a custom resource and then fails on a later resource rolls the custom
+ * resource back the way CloudFormation does: the handler is sent a second {@code Update} carrying
+ * the old properties, so it can undo what the first one applied.
+ *
+ * <p>The ServiceToken Lambda is mocked, as in {@code CloudFormationCustomResourceStackIdIntegrationTest},
+ * so no Docker is needed. The mock answers like a real handler, by a PUT of its result to the
+ * event's {@code ResponseURL}.
+ */
+@QuarkusTest
+class CloudFormationCustomResourceRollbackIntegrationTest {
+
+ private static final String SERVICE_TOKEN =
+ "arn:aws:lambda:us-east-1:000000000000:function:rollback-probe";
+
+ private final ObjectMapper mapper = new ObjectMapper();
+ private final List<JsonNode> events = new CopyOnWriteArrayList<>();
+ private volatile String failingRequest;
+ private volatile String failureReason;
+
+ @InjectMock
+ LambdaService lambdaService;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @BeforeEach
+ void stubHandler() {
+ when(lambdaService.invoke(any(), eq(SERVICE_TOKEN), any(), eq(InvocationType.RequestResponse)))
+ .thenAnswer(inv -> {
+ JsonNode event = mapper.readTree((byte[]) inv.getArgument(2));
+ events.add(event);
+ putResponse(event);
+ return new InvokeResult(200, null, "null".getBytes(StandardCharsets.UTF_8), null, "req-1");
+ });
+ }
+
+ @Test
+ void laterFailureSendsTheCustomResourceItsOldPropertiesBack() throws InterruptedException {
+ String stackName = "cr-rollback-" + Long.toString(System.nanoTime(), 36);
+ String stackId = createStack(stackName, template("1", false));
+ awaitStackStatus(stackId, "CREATE_COMPLETE");
+ String physicalId = probePhysicalId(stackName);
+
+ updateStack(stackName, template("2", true));
+ awaitStackStatus(stackId, "UPDATE_ROLLBACK_COMPLETE");
+
+ assertEquals(List.of("Create:1", "Update:2", "Update:1"), requests());
+ assertProperties(events.get(0), "1", null);
+ assertProperties(events.get(1), "2", "1");
+ assertProperties(events.get(2), "1", "2");
+ assertEquals(physicalId, events.get(2).get("PhysicalResourceId").asText());
+ assertEquals(physicalId, probePhysicalId(stackName));
+ assertFalse(describeStackEvents(stackId).contains("Rollback is not implemented"));
+ }
+
+ @Test
+ void handlerThatFailsTheRollbackLeavesTheStackUpdateRollbackFailed() throws InterruptedException {
+ failOn("Update:1", "cannot restore revision 1");
+ String stackName = "cr-rollback-failed-" + Long.toString(System.nanoTime(), 36);
+ String stackId = createStack(stackName, template("1", false));
+ awaitStackStatus(stackId, "CREATE_COMPLETE");
+
+ updateStack(stackName, template("2", true));
+ awaitStackStatus(stackId, "UPDATE_ROLLBACK_FAILED");
+
+ assertEquals(List.of("Create:1", "Update:2", "Update:1"), requests());
+ List<Map<String, String>> probeFailures = XmlParser.extractGroups(describeStackEvents(stackId), "member")
+ .stream()
+ .filter(e -> "Probe".equals(e.get("LogicalResourceId")))
+ .filter(e -> "UPDATE_FAILED".equals(e.get("ResourceStatus")))
+ .toList();
+ assertEquals(1, probeFailures.size(), probeFailures.toString());
+ String reason = probeFailures.get(0).get("ResourceStatusReason");
+ assertTrue(reason.contains("cannot restore revision 1"), reason);
+ }
+
+ @Test
+ void customResourceWhoseOwnUpdateFailsIsSentItsOldPropertiesBack() throws InterruptedException {
+ failOn("Update:3", "forced failure for Update v3");
+ String stackName = "cr-own-update-failed-" + Long.toString(System.nanoTime(), 36);
+ String stackId = createStack(stackName, template("1", false));
+ awaitStackStatus(stackId, "CREATE_COMPLETE");
+ String physicalId = probePhysicalId(stackName);
+
+ updateStack(stackName, template("3", false));
+ awaitStackStatus(stackId, "UPDATE_ROLLBACK_COMPLETE");
+
+ assertEquals(List.of("Create:1", "Update:3", "Update:1"), requests());
+ assertProperties(events.get(0), "1", null);
+ assertProperties(events.get(1), "3", "1");
+ assertProperties(events.get(2), "1", "3");
+ assertEquals(physicalId, events.get(2).get("PhysicalResourceId").asText());
+ assertEquals(physicalId, probePhysicalId(stackName));
+ List<String> probeStatuses = XmlParser.extractGroups(describeStackEvents(stackId), "member")
+ .reversed()
+ .stream()
+ .filter(e -> "Probe".equals(e.get("LogicalResourceId")))
+ .map(e -> e.get("ResourceStatus"))
+ .toList();
+ assertEquals(List.of("UPDATE_FAILED", "UPDATE_IN_PROGRESS", "UPDATE_COMPLETE"),
+ probeStatuses.subList(probeStatuses.size() - 3, probeStatuses.size()));
+ }
+
+ @Test
+ void deleteAfterAFailedRollbackSendsTheOldProperties() throws InterruptedException {
+ failOn("Update:1", "cannot restore revision 1");
+ String stackName = "cr-delete-after-failed-" + Long.toString(System.nanoTime(), 36);
+ String stackId = createStack(stackName, template("1", false));
+ awaitStackStatus(stackId, "CREATE_COMPLETE");
+ String physicalId = probePhysicalId(stackName);
+ updateStack(stackName, template("2", true));
+ awaitStackStatus(stackId, "UPDATE_ROLLBACK_FAILED");
+
+ deleteStack(stackName);
+ awaitStackStatus(stackId, "DELETE_COMPLETE");
+
+ assertEquals(List.of("Create:1", "Update:2", "Update:1", "Delete:1"), requests());
+ assertProperties(events.get(3), "1", null);
+ assertEquals(physicalId, events.get(3).get("PhysicalResourceId").asText());
+ }
+
+ /**
+ * Asserts the request's complete ResourceProperties, and its OldResourceProperties or, when
+ * {@code oldRevision} is null, that it carries none.
+ */
+ private void assertProperties(JsonNode event, String revision, String oldRevision) {
+ assertEquals(properties(revision), event.get("ResourceProperties"), event.toString());
+ if (oldRevision == null) {
+ assertFalse(event.has("OldResourceProperties"), event.toString());
+ } else {
+ assertEquals(properties(oldRevision), event.get("OldResourceProperties"), event.toString());
+ }
+ }
+
+ private ObjectNode properties(String revision) {
+ ObjectNode properties = mapper.createObjectNode();
+ properties.put("ServiceToken", SERVICE_TOKEN);
+ properties.put("Revision", revision);
+ return properties;
+ }
+
+ private void failOn(String request, String reason) {
+ failingRequest = request;
+ failureReason = reason;
+ }
+
+ private static String template(String revision, boolean withFailingBucket) {
+ String failingBucket = withFailingBucket
+ ? """
+ ,
+ "Bad": {
+ "Type": "AWS::S3::Bucket",
+ "DependsOn": "Probe",
+ "Properties": {"BucketName": "bad/name"}
+ }
+ """
+ : "";
+ return """
+ {
+ "Resources": {
+ "Probe": {
+ "Type": "Custom::RollbackProbe",
+ "Properties": {
+ "ServiceToken": "%s",
+ "Revision": "%s"
+ }
+ }%s
+ }
+ }
+ """.formatted(SERVICE_TOKEN, revision, failingBucket);
+ }
+
+ /** Each request the handler saw, as its RequestType and the Revision it was asked to apply. */
+ private List<String> requests() {
+ return events.stream().map(CloudFormationCustomResourceRollbackIntegrationTest::request).toList();
+ }
+
+ private static String request(JsonNode event) {
+ return event.get("RequestType").asText() + ":" + event.get("ResourceProperties").get("Revision").asText();
+ }
+
+ /**
+ * Answers the request the way a handler does: a PUT of its result document to ResponseURL. The
+ * request the test named with {@link #failOn} is answered FAILED.
+ */
+ private void putResponse(JsonNode event) {
+ ObjectNode response = mapper.createObjectNode();
+ if (request(event).equals(failingRequest)) {
+ response.put("Status", "FAILED");
+ response.put("Reason", failureReason);
+ } else {
+ response.put("Status", "SUCCESS");
+ }
+ response.put("RequestId", event.get("RequestId").asText());
+ response.put("StackId", event.get("StackId").asText());
+ response.put("LogicalResourceId", event.get("LogicalResourceId").asText());
+ response.put("PhysicalResourceId", "rollback-probe");
+ String path = URI.create(event.get("ResponseURL").asText()).getPath();
+ given().body(response.toString()).when().put(path).then().statusCode(200);
+ }
+
+ private static String createStack(String stackName, String template) {
+ String xml = cfn(Map.of("Action", "CreateStack", "StackName", stackName, "TemplateBody", template));
+ return XmlParser.extractFirst(xml, "StackId", null);
+ }
+
+ private static void updateStack(String stackName, String template) {
+ cfn(Map.of("Action", "UpdateStack", "StackName", stackName, "TemplateBody", template));
+ }
+
+ private static void deleteStack(String stackName) {
+ cfn(Map.of("Action", "DeleteStack", "StackName", stackName));
+ }
+
+ private static String probePhysicalId(String stackName) {
+ String xml = cfn(Map.of("Action", "DescribeStackResource", "StackName", stackName,
+ "LogicalResourceId", "Probe"));
+ return XmlParser.extractFirst(xml, "PhysicalResourceId", null);
+ }
+
+ private static String describeStackEvents(String stackId) {
+ return cfn(Map.of("Action", "DescribeStackEvents", "StackName", stackId));
+ }
+
+ /** Polls DescribeStacks by stack id and returns its XML once the stack reaches {@code status}. */
+ private static String awaitStackStatus(String stackId, String status) throws InterruptedException {
+ String xml = "";
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (System.currentTimeMillis() < deadline) {
+ xml = cfn(Map.of("Action", "DescribeStacks", "StackName", stackId));
+ if (xml.contains("<StackStatus>" + status + "</StackStatus>")) {
+ return xml;
+ }
+ Thread.sleep(50);
+ }
+ fail("stack " + stackId + " never reached " + status + ": " + xml);
+ return xml;
+ }
+
+ private static String cfn(Map<String, String> params) {
+ return given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParams(params)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().asString();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CustomResourceProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CustomResourceProvisionerTest.java
index 629b26a234..59bfacc8e1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CustomResourceProvisionerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CustomResourceProvisionerTest.java
@@ -4,9 +4,11 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.docker.ContainerReachableEndpoint;
import io.github.hectorvent.floci.services.cloudformation.model.StackResource;
import io.github.hectorvent.floci.services.cloudformation.provisioners.CfnResourceDispatcher;
+import io.github.hectorvent.floci.services.cloudformation.provisioners.CfnRollback;
import io.github.hectorvent.floci.services.lambda.LambdaService;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
@@ -20,6 +22,8 @@ import java.util.Map;
import java.util.function.Function;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
@@ -77,23 +81,54 @@ class CustomResourceProvisionerTest {
/** Stubs the Lambda to behave like a handler that PUTs a SUCCESS response to the ResponseURL. */
private void stubHandler(String physicalId, Map<String, String> data) {
+ respond(event -> success(physicalId, data));
+ }
+
+ /** Stubs the Lambda to PUT, for each event it is sent, the response {@code answer} builds. */
+ private void respond(Function<JsonNode, ObjectNode> answer) {
when(lambdaService.invoke(any(), eq(SERVICE_TOKEN), any(), eq(InvocationType.RequestResponse)))
.thenAnswer(inv -> {
JsonNode event = mapper.readTree((byte[]) inv.getArgument(2));
String responseUrl = event.get("ResponseURL").asText();
String token = responseUrl.substring(responseUrl.lastIndexOf('/') + 1);
-
- ObjectNode response = mapper.createObjectNode();
- response.put("Status", "SUCCESS");
- response.put("PhysicalResourceId", physicalId);
- ObjectNode dataNode = response.putObject("Data");
- data.forEach(dataNode::put);
- store.complete(token, response);
-
+ store.complete(token, answer.apply(event));
return new InvokeResult(200, null, "null".getBytes(), null, "req-1");
});
}
+ private ObjectNode success(String physicalId, Map<String, String> data) {
+ ObjectNode response = mapper.createObjectNode();
+ response.put("Status", "SUCCESS");
+ response.put("PhysicalResourceId", physicalId);
+ ObjectNode dataNode = response.putObject("Data");
+ data.forEach(dataNode::put);
+ return response;
+ }
+
+ private ObjectNode failed(String reason) {
+ ObjectNode response = mapper.createObjectNode();
+ response.put("Status", "FAILED");
+ response.put("Reason", reason);
+ return response;
+ }
+
+ private StackResource create() {
+ return provisioner.provision("MyCr", "Custom::Test", props(),
+ engine(), "us-east-1", "000000000000", "my-stack");
+ }
+
+ private StackResource update(StackResource current, String message) {
+ ObjectNode changed = props();
+ changed.put("Message", message);
+ return provisioner.provision("MyCr", "Custom::Test", changed,
+ engine(), "us-east-1", "000000000000", "my-stack",
+ current.getPhysicalId(), current.getAttributes());
+ }
+
+ private static String message(JsonNode event, String properties) {
+ return event.path(properties).path("Message").asText(null);
+ }
+
@Test
void createInvokesHandlerAndCapturesPhysicalIdAndData() {
stubHandler("phys-123", Map.of("Greeting", "hello"));
@@ -266,6 +301,229 @@ class CustomResourceProvisionerTest {
assertTrue(AwsArnUtils.isArnFor(stackId, "cloudformation"), stackId);
}
+ @Test
+ void changedUpdateRecordsTheRollbackSnapshotBeforeTheHandlerRuns() {
+ respond(event -> "Update".equals(event.get("RequestType").asText())
+ ? failed("boom") : success("phys-123", Map.of()));
+
+ StackResource updated = update(create(), "bye");
+
+ assertEquals("CREATE_FAILED", updated.getStatus());
+ assertTrue(updated.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void rollbackUpdateSendsTheOldPropertiesBackToTheSamePhysicalId() {
+ respond(event -> {
+ String message = message(event, "ResourceProperties");
+ return success("phys-123", "bye".equals(message)
+ ? Map.of("Greeting", message, "Farewell", "yes")
+ : Map.of("Greeting", message));
+ });
+ StackResource created = create();
+ String createdStash = created.getAttributes().get("__FlociResourceProperties");
+ StackResource updated = update(created, "bye");
+ List<String> progress = new ArrayList<>();
+
+ assertTrue(provisioner.rollbackUpdate(updated, event -> progress.add(event.getResourceStatus())));
+
+ JsonNode rollback = capturedEvents(3).get(2);
+ assertEquals("Update", rollback.get("RequestType").asText());
+ assertEquals("phys-123", rollback.get("PhysicalResourceId").asText());
+ assertEquals("hi", message(rollback, "ResourceProperties"));
+ assertEquals("bye", message(rollback, "OldResourceProperties"));
+ assertEquals(STACK_ID, rollback.get("StackId").asText());
+ assertEquals(List.of("UPDATE_IN_PROGRESS"), progress);
+ assertEquals("phys-123", updated.getPhysicalId());
+ assertEquals(createdStash, updated.getAttributes().get("__FlociResourceProperties"));
+ assertEquals("hi", updated.getAttributes().get("Greeting"));
+ assertFalse(updated.getAttributes().containsKey("Farewell"));
+ assertFalse(updated.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void rollbackUpdateThatTheHandlerFailsThrowsItsReasonAndKeepsTheSnapshot() {
+ respond(event -> "bye".equals(message(event, "OldResourceProperties"))
+ ? failed("cannot undo") : success("phys-123", Map.of()));
+ StackResource updated = update(create(), "bye");
+
+ AwsException failure = assertThrows(AwsException.class,
+ () -> provisioner.rollbackUpdate(updated, event -> { }));
+
+ assertTrue(failure.getMessage().contains("cannot undo"), failure.getMessage());
+ assertTrue(updated.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void retryOfAFailedRollbackSendsTheSameUpdateAgain() {
+ respond(event -> "bye".equals(message(event, "OldResourceProperties"))
+ ? failed("cannot undo") : success("phys-123", Map.of()));
+ StackResource updated = update(create(), "bye");
+ assertThrows(AwsException.class, () -> provisioner.rollbackUpdate(updated, event -> { }));
+
+ assertThrows(AwsException.class, () -> provisioner.rollbackUpdate(updated, event -> { }));
+
+ JsonNode retry = capturedEvents(4).get(3);
+ assertEquals("Update", retry.get("RequestType").asText());
+ assertEquals("hi", message(retry, "ResourceProperties"));
+ assertEquals("bye", message(retry, "OldResourceProperties"));
+ }
+
+ @Test
+ void deleteAfterAFailedRollbackCarriesTheOldProperties() {
+ // AWS treats the rollback target as the resource's properties even though the handler failed it.
+ respond(event -> "bye".equals(message(event, "OldResourceProperties"))
+ ? failed("cannot undo") : success("phys-123", Map.of()));
+ StackResource updated = update(create(), "bye");
+ assertThrows(AwsException.class, () -> provisioner.rollbackUpdate(updated, event -> { }));
+
+ provisioner.delete(updated, "us-east-1");
+
+ JsonNode delete = capturedEvents(4).get(3);
+ assertEquals("Delete", delete.get("RequestType").asText());
+ assertEquals("phys-123", delete.get("PhysicalResourceId").asText());
+ assertEquals("hi", message(delete, "ResourceProperties"));
+ }
+
+ @Test
+ void failedUpdateIsRetainedForRollbackOnlyWhenItReachedTheHandler() {
+ respond(event -> "bye".equals(message(event, "ResourceProperties"))
+ ? failed("forced failure") : success("phys-123", Map.of()));
+ StackResource created = create();
+
+ StackResource failed = update(created, "bye");
+
+ assertEquals("CREATE_FAILED", failed.getStatus());
+ assertTrue(provisioner.retainsFailedUpdateState(failed));
+ assertFalse(provisioner.retainsFailedUpdateState(created));
+ }
+
+ @Test
+ void updateThatChangesTheServiceTokenFailsBeforeAnyHandlerIsSent() {
+ stubHandler("phys-123", Map.of());
+ StackResource created = create();
+ ObjectNode otherToken = props();
+ otherToken.put("ServiceToken", "arn:aws:lambda:us-east-1:000000000000:function:OtherHandler");
+
+ StackResource failed = provisioner.provision("MyCr", "Custom::Test", otherToken,
+ engine(), "us-east-1", "000000000000", "my-stack",
+ created.getPhysicalId(), created.getAttributes());
+
+ assertEquals("CREATE_FAILED", failed.getStatus());
+ assertEquals("Modifying service token is not allowed.", failed.getStatusReason());
+ assertFalse(provisioner.retainsFailedUpdateState(failed));
+ verify(lambdaService, times(1)).invoke(any(), any(), any(), any());
+ }
+
+ @Test
+ void updateOfAResourceWithNoStashedPropertiesHasNothingToRollBack() {
+ stubHandler("phys-123", Map.of());
+ StackResource created = create();
+ created.getAttributes().remove("__FlociResourceProperties");
+
+ StackResource updated = update(created, "bye");
+
+ assertFalse(updated.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));
+ assertTrue(provisioner.rollbackUpdate(updated, event -> { }));
+ capturedEvents(2);
+ }
+
+ @Test
+ void rollbackOfAnUpdateTheHandlerFailedSendsTheAttemptedPropertiesAsOld() {
+ respond(event -> "bye".equals(message(event, "ResourceProperties"))
+ ? failed("forced failure") : success("phys-123", Map.of()));
+ StackResource created = create();
+ String createdStash = created.getAttributes().get("__FlociResourceProperties");
+ StackResource failed = update(created, "bye");
+
+ assertTrue(provisioner.rollbackUpdate(failed, event -> { }));
+
+ JsonNode rollback = capturedEvents(3).get(2);
+ assertEquals("Update", rollback.get("RequestType").asText());
+ assertEquals("phys-123", rollback.get("PhysicalResourceId").asText());
+ assertEquals("hi", message(rollback, "ResourceProperties"));
+ assertEquals("bye", message(rollback, "OldResourceProperties"));
+ assertEquals("phys-123", failed.getPhysicalId());
+ assertEquals(createdStash, failed.getAttributes().get("__FlociResourceProperties"));
+ assertFalse(failed.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void rollbackSendsTheStackIdTheFailedUpdateWasSent() {
+ respond(event -> "bye".equals(message(event, "ResourceProperties"))
+ ? failed("forced failure") : success("phys-123", Map.of()));
+ StackResource created = create();
+ created.getAttributes().remove("__FlociStackId");
+ StackResource failed = update(created, "bye");
+
+ assertTrue(provisioner.rollbackUpdate(failed, event -> { }));
+
+ List<JsonNode> events = capturedEvents(3);
+ assertEquals(STACK_ID, events.get(1).get("StackId").asText());
+ assertEquals(STACK_ID, events.get(2).get("StackId").asText());
+ }
+
+ @Test
+ void dataUnderTheReservedPrefixIsNotKept() {
+ stubHandler("phys-123", Map.of("__FlociStackId", "forged", "Greeting", "hi"));
+
+ StackResource created = create();
+
+ assertEquals(STACK_ID, created.getAttributes().get("__FlociStackId"));
+ assertEquals("hi", created.getAttributes().get("Greeting"));
+ }
+
+ @Test
+ void rollbackOfAReplacingUpdateDeletesTheReplacementAndSendsNoUpdate() {
+ respond(event -> {
+ String message = message(event, "ResourceProperties");
+ return success("bye".equals(message) ? "phys-456" : "phys-123", Map.of("Greeting", message));
+ });
+ StackResource created = create();
+ String createdStash = created.getAttributes().get("__FlociResourceProperties");
+ StackResource updated = update(created, "bye");
+ assertEquals("phys-456", updated.getPhysicalId());
+
+ assertTrue(provisioner.rollbackUpdate(updated, event -> { }));
+
+ JsonNode delete = capturedEvents(3).get(2);
+ assertEquals("Delete", delete.get("RequestType").asText());
+ assertEquals("phys-456", delete.get("PhysicalResourceId").asText());
+ assertEquals("bye", message(delete, "ResourceProperties"));
+ assertEquals("phys-123", updated.getPhysicalId());
+ assertEquals(createdStash, updated.getAttributes().get("__FlociResourceProperties"));
+ assertEquals("hi", updated.getAttributes().get("Greeting"));
+ assertFalse(updated.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void rollbackResponseNamingAnotherPhysicalIdIsTakenAndTheIdSentIsDeleted() {
+ respond(event -> success("bye".equals(message(event, "OldResourceProperties"))
+ ? "phys-789" : "phys-123", Map.of()));
+ StackResource updated = update(create(), "bye");
+
+ assertTrue(provisioner.rollbackUpdate(updated, event -> { }));
+
+ List<JsonNode> events = capturedEvents(4);
+ assertEquals("Update", events.get(2).get("RequestType").asText());
+ JsonNode delete = events.get(3);
+ assertEquals("Delete", delete.get("RequestType").asText());
+ assertEquals("phys-123", delete.get("PhysicalResourceId").asText());
+ assertEquals("bye", message(delete, "ResourceProperties"));
+ assertEquals("phys-789", updated.getPhysicalId());
+ }
+
+ @Test
+ void rollbackAfterAnUpdateThatSentNothingSendsNothing() {
+ stubHandler("phys-123", Map.of());
+ StackResource changed = update(create(), "bye");
+ StackResource unchanged = update(changed, "bye");
+
+ assertTrue(provisioner.rollbackUpdate(unchanged, event -> { }));
+
+ capturedEvents(2);
+ }
+
private List<JsonNode> capturedEvents(int count) {
ArgumentCaptor<byte[]> payloads = ArgumentCaptor.forClass(byte[].class);
verify(lambdaService, times(count)).invoke(any(), eq(SERVICE_TOKEN), payloads.capture(),