Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): roll back a changed custom resource (#4941)
Mario Toffiae8a300f4d55b
diff --git a/docs/services/cloudformation.md b/docs/services/cloudformation.mdindex 65996f1869..bdb5a55d66 100644--- a/docs/services/cloudformation.md+++ b/docs/services/cloudformation.md@@ -416,7 +416,24 @@ Lambda. floci supports two shapes:default); a resource that never completes fails the stack rather than hanging.The handler's `Data` keys resolve through `Fn::GetAtt`, except keys that begin with `__Floci`. Floci-reserves that prefix for its own resource state, which no template can read.+reserves that prefix for its own resource state and ignores such keys.++A failed stack update rolls back a custom resource whose handler was sent an `Update`, as+CloudFormation does. This holds whether the handler applied the update and a later resource failed,+or the handler answered `FAILED` itself. The handler is sent a second `Update` under the same+`PhysicalResourceId`, with the old properties as `ResourceProperties` and the attempted ones as+`OldResourceProperties`, so it can undo the change. A handler that answers `FAILED` to that rollback+leaves the resource `UPDATE_FAILED` with its `Reason`, and the stack `UPDATE_ROLLBACK_FAILED`. The+old properties still become the resource's own, as on AWS: a later `DeleteStack` sends them in its+`Delete`, and the next update sends them as `OldResourceProperties`. An update whose handler returned+a new `PhysicalResourceId` replaced the resource. Its rollback sends no `Update`: the new id gets a+`Delete` with the attempted properties, and the resource goes back to the old id. CloudFormation sends+that `Delete` in its `UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS` phase. Floci has no rollback+cleanup phase and sends it during the rollback. As in that phase, a `Delete` the handler fails is+logged and does not fail the rollback. When the rollback's own `Update` returns a new id, the id it+was sent gets a `Delete` straight away for the same reason. An update that changes `ServiceToken`+reaches no handler: it fails with `Modifying service token is not allowed.`, as on AWS, and its+rollback sends nothing.## Deleted Stacksdiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.javaindex 247ca84ba2..853dd49614 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java@@ -70,6 +70,15 @@ public final class CfnRollback {*/public static final String EVENT_INVOKE_CONFIG_SNAPSHOT_ATTR = "__FlociEventInvokeConfigSnapshot";+ /**+ * Holds what a custom resource had before an update sent its handler an {@code Update}: the+ * physical id, the old and the attempted properties, the region and the {@code Data} attributes,+ * so a failed stack update can send the handler the old properties back, whether the handler+ * applied the update or failed it. Written by {@code CustomResourceCfnProvisioner} before the+ * invoke and spent by its {@code rollbackUpdate}.+ */+ public static final String CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR = "__FlociCustomResourceUpdateSnapshot";+/*** Holds the body and tags a dashboard carried before an in-place update changed them, or the* fact that it did not exist, so a failed stack update can put it back. Written bydiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CustomResourceCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CustomResourceCfnProvisioner.javaindex a458fc77a7..ae0f55bcb7 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CustomResourceCfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CustomResourceCfnProvisioner.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.cloudformation.provisioners;+import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ArrayNode;@@ -9,6 +10,7 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.docker.ContainerReachableEndpoint;import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;import io.github.hectorvent.floci.services.cloudformation.CustomResourceResponseStore;+import io.github.hectorvent.floci.services.cloudformation.model.StackEvent;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.lambda.LambdaService;import io.github.hectorvent.floci.services.lambda.model.InvocationType;@@ -22,6 +24,7 @@ import java.time.Duration;import java.util.Set;import java.util.UUID;import java.util.concurrent.TimeoutException;+import java.util.function.Consumer;/*** Provisions {@code AWS::CloudFormation::CustomResource} and every {@code Custom::*} type, moved out@@ -46,6 +49,8 @@ public class CustomResourceCfnProvisioner implements CfnResourceProvisioner {private static final String CR_SERVICE_TOKEN_ATTR = "__FlociServiceToken";private static final String CR_PROPERTIES_ATTR = "__FlociResourceProperties";private static final String CR_STACK_ID_ATTR = "__FlociStackId";+ /** Prefix of every attribute Floci keeps for itself; the others are the handler's {@code Data}. */+ private static final String RESERVED_ATTR_PREFIX = "__Floci";/*** How long to wait for the Lambda's ResponseURL callback after the synchronous invoke returns.* The invoke already blocks until the handler finishes, so this only covers a PUT that lands@@ -75,6 +80,8 @@ public class CustomResourceCfnProvisioner implements CfnResourceProvisioner {@Overridepublic void provision(StackResource r, JsonNode props, ProvisionContext ctx) {+ // A snapshot describes the update in flight; one an earlier update left behind is stale.+ r.getAttributes().remove(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR);CloudFormationTemplateEngine engine = ctx.engine();String region = ctx.region();if (props == null || !props.has("ServiceToken")) {@@ -116,17 +123,23 @@ public class CustomResourceCfnProvisioner implements CfnResourceProvisioner {return;}+ // CloudFormation fails this update before sending anything, so no handler owes a rollback.+ String priorServiceToken = isUpdate ? r.getAttributes().get(CR_SERVICE_TOKEN_ATTR) : null;+ if (priorServiceToken != null && !priorServiceToken.equals(serviceToken)) {+ throw new AwsException("ValidationError", "Modifying service token is not allowed.", 400);+ }+String stackId = stackIdOrMinted(engine.stackId(), region, ctx.accountId(), ctx.stackName());+ // Kept before the handler runs, so the rollback of a failed Update sends the same stack id.+ r.getAttributes().put(CR_STACK_ID_ATTR, stackId);+ // A resource stashed without its properties has none to roll back to.+ if (oldResourceProperties != null) {+ snapshotBeforeUpdate(r, oldResourceProperties, resourceProperties, region);+ }JsonNode response = invokeCustomResourceHandler(serviceToken, requestType, r.getLogicalId(),r.getResourceType(), priorPhysicalId, resourceProperties, oldResourceProperties,region, stackId);-- String status = response.path("Status").asText("FAILED");- if (!"SUCCESS".equals(status)) {- throw new AwsException("CustomResourceFailed",- "Custom resource handler reported FAILED: "- + response.path("Reason").asText("(no reason given)"), 400);- }+ requireSuccess(response);String returnedPhysicalId = response.path("PhysicalResourceId").asText(null);if (returnedPhysicalId != null && !returnedPhysicalId.isBlank()) {@@ -138,27 +151,25 @@ public class CustomResourceCfnProvisioner implements CfnResourceProvisioner {}// Data.* become Fn::GetAtt attributes on the custom resource.- JsonNode data = response.path("Data");- if (data.isObject()) {- data.fields().forEachRemaining(e ->- r.getAttributes().put(e.getKey(), nodeToAttributeValue(e.getValue())));- }+ putData(r, response.path("Data"));// Stash what a later Delete invocation needs (delete() only gets the StackResource).r.getAttributes().put(CR_SERVICE_TOKEN_ATTR, serviceToken);r.getAttributes().put(CR_PROPERTIES_ATTR, resourceProperties.toString());- r.getAttributes().put(CR_STACK_ID_ATTR, stackId);}@Overridepublic void delete(StackResource r, String region) {+ ObjectNode stashed = readStashedProperties(r);+ deleteWith(r, stashed != null ? stashed : objectMapper.createObjectNode(), region);+ }++ private void deleteWith(StackResource r, ObjectNode resourceProperties, String region) {String serviceToken = r.getAttributes().get(CR_SERVICE_TOKEN_ATTR);if (serviceToken == null || serviceToken.isBlank()) {LOG.debugv("Custom resource {0} has no stored ServiceToken; skipping Delete", r.getLogicalId());return;}- ObjectNode stashed = readStashedProperties(r);- ObjectNode resourceProperties = stashed != null ? stashed : objectMapper.createObjectNode();// A resource stashed before the stack id was recorded has none, and delete() has no stack.String stackId = stackIdOrMinted(r.getAttributes().get(CR_STACK_ID_ATTR), region,accountFromArn(serviceToken), "");@@ -175,6 +186,118 @@ public class CustomResourceCfnProvisioner implements CfnResourceProvisioner {}}+ @Override+ public boolean rollbackUpdate(StackResource resource) {+ return rollbackUpdate(resource, event -> { });+ }++ /**+ * An Update that reached the handler and then failed keeps its copy for {@link #rollbackUpdate},+ * because CloudFormation sends that handler the old properties back too.+ */+ @Override+ public boolean retainsFailedUpdateState(StackResource resource) {+ return resource.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR);+ }++ /**+ * Undoes an update that reached the handler, as CloudFormation does, whether the handler applied+ * it or failed it. An update that replaced the resource gets a Delete for the replacement, and+ * the resource points back at the one it displaced. One that kept the physical id gets a second+ * Update carrying the old properties, with the attempted ones as OldResourceProperties. A+ * handler that fails that Update fails the rollback, and the snapshot stays for another attempt.+ */+ @Override+ public boolean rollbackUpdate(StackResource resource, Consumer<StackEvent> progress) {+ String raw = resource.getAttributes().get(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR);+ if (raw == null) {+ return true;+ }+ JsonNode snapshot;+ try {+ snapshot = objectMapper.readTree(raw);+ } catch (JsonProcessingException e) {+ throw new IllegalStateException("Could not read the custom resource update snapshot for "+ + resource.getLogicalId(), e);+ }+ String priorPhysicalId = snapshot.path("physicalId").asText();+ ObjectNode oldProperties = (ObjectNode) snapshot.get("properties");+ ObjectNode attemptedProperties = (ObjectNode) snapshot.get("attempted");+ String region = snapshot.path("region").asText();+ String currentPhysicalId = resource.getPhysicalId();+ // CloudFormation holds the rollback target as the resource's properties even when the handler+ // fails it: a later Delete, or the next update's OldResourceProperties, carries them.+ resource.getAttributes().put(CR_PROPERTIES_ATTR, oldProperties.toString());+ if (!priorPhysicalId.equals(currentPhysicalId)) {+ deleteWith(resource, attemptedProperties, region);+ resource.setPhysicalId(priorPhysicalId);+ replaceData(resource, snapshot.path("attributes"));+ } else {+ ProvisionContext.report(progress, currentPhysicalId, "UPDATE_IN_PROGRESS", null);+ String serviceToken = resource.getAttributes().get(CR_SERVICE_TOKEN_ATTR);+ String stackId = stackIdOrMinted(resource.getAttributes().get(CR_STACK_ID_ATTR), region,+ accountFromArn(serviceToken), "");+ JsonNode response = invokeCustomResourceHandler(serviceToken, "Update", resource.getLogicalId(),+ resource.getResourceType(), currentPhysicalId, oldProperties, attemptedProperties,+ region, stackId);+ requireSuccess(response);+ String returnedPhysicalId = response.path("PhysicalResourceId").asText(null);+ if (returnedPhysicalId != null && !returnedPhysicalId.isBlank()+ && !returnedPhysicalId.equals(currentPhysicalId)) {+ // Floci has no rollback cleanup phase, so the id the rollback displaced goes now.+ deleteWith(resource, attemptedProperties, region);+ resource.setPhysicalId(returnedPhysicalId);+ }+ replaceData(resource, response.path("Data"));+ }+ resource.getAttributes().remove(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR);+ return true;+ }++ /**+ * Keeps what the resource has before an Update reaches its handler, so {@link #rollbackUpdate}+ * can send the old properties back: the physical id, the old and the attempted properties, the+ * region and the {@code Data} attributes.+ */+ private void snapshotBeforeUpdate(StackResource r, ObjectNode oldResourceProperties,+ ObjectNode attemptedProperties, String region) {+ ObjectNode snapshot = objectMapper.createObjectNode();+ snapshot.put("physicalId", r.getPhysicalId());+ snapshot.set("properties", oldResourceProperties);+ snapshot.set("attempted", attemptedProperties);+ snapshot.put("region", region);+ ObjectNode attributes = snapshot.putObject("attributes");+ r.getAttributes().forEach((key, value) -> {+ if (!key.startsWith(RESERVED_ATTR_PREFIX)) {+ attributes.put(key, value);+ }+ });+ r.getAttributes().put(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR, snapshot.toString());+ }++ private static void requireSuccess(JsonNode response) {+ if (!"SUCCESS".equals(response.path("Status").asText("FAILED"))) {+ throw new AwsException("CustomResourceFailed",+ "Custom resource handler reported FAILED: "+ + response.path("Reason").asText("(no reason given)"), 400);+ }+ }++ private static void putData(StackResource r, JsonNode data) {+ if (data.isObject()) {+ data.fields().forEachRemaining(e -> {+ if (!e.getKey().startsWith(RESERVED_ATTR_PREFIX)) {+ r.getAttributes().put(e.getKey(), nodeToAttributeValue(e.getValue()));+ }+ });+ }+ }++ private static void replaceData(StackResource r, JsonNode data) {+ r.getAttributes().keySet().removeIf(key -> !key.startsWith(RESERVED_ATTR_PREFIX));+ putData(r, data);+ }+// Reads the ResourceProperties stashed at the last create/update (CR_PROPERTIES_ATTR).// Returns null when nothing is stashed or it cannot be parsed.private ObjectNode readStashedProperties(StackResource r) {diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationCustomResourceRollbackIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationCustomResourceRollbackIntegrationTest.javanew file mode 100644index 0000000000..60d915524d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationCustomResourceRollbackIntegrationTest.java@@ -0,0 +1,284 @@+package io.github.hectorvent.floci.services.cloudformation;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.XmlParser;+import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.InjectMock;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.net.URI;+import java.nio.charset.StandardCharsets;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CopyOnWriteArrayList;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.when;++/**+ * A stack update that changes a custom resource and then fails on a later resource rolls the custom+ * resource back the way CloudFormation does: the handler is sent a second {@code Update} carrying+ * the old properties, so it can undo what the first one applied.+ *+ * <p>The ServiceToken Lambda is mocked, as in {@code CloudFormationCustomResourceStackIdIntegrationTest},+ * so no Docker is needed. The mock answers like a real handler, by a PUT of its result to the+ * event's {@code ResponseURL}.+ */+@QuarkusTest+class CloudFormationCustomResourceRollbackIntegrationTest {++ private static final String SERVICE_TOKEN =+ "arn:aws:lambda:us-east-1:000000000000:function:rollback-probe";++ private final ObjectMapper mapper = new ObjectMapper();+ private final List<JsonNode> events = new CopyOnWriteArrayList<>();+ private volatile String failingRequest;+ private volatile String failureReason;++ @InjectMock+ LambdaService lambdaService;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @BeforeEach+ void stubHandler() {+ when(lambdaService.invoke(any(), eq(SERVICE_TOKEN), any(), eq(InvocationType.RequestResponse)))+ .thenAnswer(inv -> {+ JsonNode event = mapper.readTree((byte[]) inv.getArgument(2));+ events.add(event);+ putResponse(event);+ return new InvokeResult(200, null, "null".getBytes(StandardCharsets.UTF_8), null, "req-1");+ });+ }++ @Test+ void laterFailureSendsTheCustomResourceItsOldPropertiesBack() throws InterruptedException {+ String stackName = "cr-rollback-" + Long.toString(System.nanoTime(), 36);+ String stackId = createStack(stackName, template("1", false));+ awaitStackStatus(stackId, "CREATE_COMPLETE");+ String physicalId = probePhysicalId(stackName);++ updateStack(stackName, template("2", true));+ awaitStackStatus(stackId, "UPDATE_ROLLBACK_COMPLETE");++ assertEquals(List.of("Create:1", "Update:2", "Update:1"), requests());+ assertProperties(events.get(0), "1", null);+ assertProperties(events.get(1), "2", "1");+ assertProperties(events.get(2), "1", "2");+ assertEquals(physicalId, events.get(2).get("PhysicalResourceId").asText());+ assertEquals(physicalId, probePhysicalId(stackName));+ assertFalse(describeStackEvents(stackId).contains("Rollback is not implemented"));+ }++ @Test+ void handlerThatFailsTheRollbackLeavesTheStackUpdateRollbackFailed() throws InterruptedException {+ failOn("Update:1", "cannot restore revision 1");+ String stackName = "cr-rollback-failed-" + Long.toString(System.nanoTime(), 36);+ String stackId = createStack(stackName, template("1", false));+ awaitStackStatus(stackId, "CREATE_COMPLETE");++ updateStack(stackName, template("2", true));+ awaitStackStatus(stackId, "UPDATE_ROLLBACK_FAILED");++ assertEquals(List.of("Create:1", "Update:2", "Update:1"), requests());+ List<Map<String, String>> probeFailures = XmlParser.extractGroups(describeStackEvents(stackId), "member")+ .stream()+ .filter(e -> "Probe".equals(e.get("LogicalResourceId")))+ .filter(e -> "UPDATE_FAILED".equals(e.get("ResourceStatus")))+ .toList();+ assertEquals(1, probeFailures.size(), probeFailures.toString());+ String reason = probeFailures.get(0).get("ResourceStatusReason");+ assertTrue(reason.contains("cannot restore revision 1"), reason);+ }++ @Test+ void customResourceWhoseOwnUpdateFailsIsSentItsOldPropertiesBack() throws InterruptedException {+ failOn("Update:3", "forced failure for Update v3");+ String stackName = "cr-own-update-failed-" + Long.toString(System.nanoTime(), 36);+ String stackId = createStack(stackName, template("1", false));+ awaitStackStatus(stackId, "CREATE_COMPLETE");+ String physicalId = probePhysicalId(stackName);++ updateStack(stackName, template("3", false));+ awaitStackStatus(stackId, "UPDATE_ROLLBACK_COMPLETE");++ assertEquals(List.of("Create:1", "Update:3", "Update:1"), requests());+ assertProperties(events.get(0), "1", null);+ assertProperties(events.get(1), "3", "1");+ assertProperties(events.get(2), "1", "3");+ assertEquals(physicalId, events.get(2).get("PhysicalResourceId").asText());+ assertEquals(physicalId, probePhysicalId(stackName));+ List<String> probeStatuses = XmlParser.extractGroups(describeStackEvents(stackId), "member")+ .reversed()+ .stream()+ .filter(e -> "Probe".equals(e.get("LogicalResourceId")))+ .map(e -> e.get("ResourceStatus"))+ .toList();+ assertEquals(List.of("UPDATE_FAILED", "UPDATE_IN_PROGRESS", "UPDATE_COMPLETE"),+ probeStatuses.subList(probeStatuses.size() - 3, probeStatuses.size()));+ }++ @Test+ void deleteAfterAFailedRollbackSendsTheOldProperties() throws InterruptedException {+ failOn("Update:1", "cannot restore revision 1");+ String stackName = "cr-delete-after-failed-" + Long.toString(System.nanoTime(), 36);+ String stackId = createStack(stackName, template("1", false));+ awaitStackStatus(stackId, "CREATE_COMPLETE");+ String physicalId = probePhysicalId(stackName);+ updateStack(stackName, template("2", true));+ awaitStackStatus(stackId, "UPDATE_ROLLBACK_FAILED");++ deleteStack(stackName);+ awaitStackStatus(stackId, "DELETE_COMPLETE");++ assertEquals(List.of("Create:1", "Update:2", "Update:1", "Delete:1"), requests());+ assertProperties(events.get(3), "1", null);+ assertEquals(physicalId, events.get(3).get("PhysicalResourceId").asText());+ }++ /**+ * Asserts the request's complete ResourceProperties, and its OldResourceProperties or, when+ * {@code oldRevision} is null, that it carries none.+ */+ private void assertProperties(JsonNode event, String revision, String oldRevision) {+ assertEquals(properties(revision), event.get("ResourceProperties"), event.toString());+ if (oldRevision == null) {+ assertFalse(event.has("OldResourceProperties"), event.toString());+ } else {+ assertEquals(properties(oldRevision), event.get("OldResourceProperties"), event.toString());+ }+ }++ private ObjectNode properties(String revision) {+ ObjectNode properties = mapper.createObjectNode();+ properties.put("ServiceToken", SERVICE_TOKEN);+ properties.put("Revision", revision);+ return properties;+ }++ private void failOn(String request, String reason) {+ failingRequest = request;+ failureReason = reason;+ }++ private static String template(String revision, boolean withFailingBucket) {+ String failingBucket = withFailingBucket+ ? """+ ,+ "Bad": {+ "Type": "AWS::S3::Bucket",+ "DependsOn": "Probe",+ "Properties": {"BucketName": "bad/name"}+ }+ """+ : "";+ return """+ {+ "Resources": {+ "Probe": {+ "Type": "Custom::RollbackProbe",+ "Properties": {+ "ServiceToken": "%s",+ "Revision": "%s"+ }+ }%s+ }+ }+ """.formatted(SERVICE_TOKEN, revision, failingBucket);+ }++ /** Each request the handler saw, as its RequestType and the Revision it was asked to apply. */+ private List<String> requests() {+ return events.stream().map(CloudFormationCustomResourceRollbackIntegrationTest::request).toList();+ }++ private static String request(JsonNode event) {+ return event.get("RequestType").asText() + ":" + event.get("ResourceProperties").get("Revision").asText();+ }++ /**+ * Answers the request the way a handler does: a PUT of its result document to ResponseURL. The+ * request the test named with {@link #failOn} is answered FAILED.+ */+ private void putResponse(JsonNode event) {+ ObjectNode response = mapper.createObjectNode();+ if (request(event).equals(failingRequest)) {+ response.put("Status", "FAILED");+ response.put("Reason", failureReason);+ } else {+ response.put("Status", "SUCCESS");+ }+ response.put("RequestId", event.get("RequestId").asText());+ response.put("StackId", event.get("StackId").asText());+ response.put("LogicalResourceId", event.get("LogicalResourceId").asText());+ response.put("PhysicalResourceId", "rollback-probe");+ String path = URI.create(event.get("ResponseURL").asText()).getPath();+ given().body(response.toString()).when().put(path).then().statusCode(200);+ }++ private static String createStack(String stackName, String template) {+ String xml = cfn(Map.of("Action", "CreateStack", "StackName", stackName, "TemplateBody", template));+ return XmlParser.extractFirst(xml, "StackId", null);+ }++ private static void updateStack(String stackName, String template) {+ cfn(Map.of("Action", "UpdateStack", "StackName", stackName, "TemplateBody", template));+ }++ private static void deleteStack(String stackName) {+ cfn(Map.of("Action", "DeleteStack", "StackName", stackName));+ }++ private static String probePhysicalId(String stackName) {+ String xml = cfn(Map.of("Action", "DescribeStackResource", "StackName", stackName,+ "LogicalResourceId", "Probe"));+ return XmlParser.extractFirst(xml, "PhysicalResourceId", null);+ }++ private static String describeStackEvents(String stackId) {+ return cfn(Map.of("Action", "DescribeStackEvents", "StackName", stackId));+ }++ /** Polls DescribeStacks by stack id and returns its XML once the stack reaches {@code status}. */+ private static String awaitStackStatus(String stackId, String status) throws InterruptedException {+ String xml = "";+ long deadline = System.currentTimeMillis() + 10_000;+ while (System.currentTimeMillis() < deadline) {+ xml = cfn(Map.of("Action", "DescribeStacks", "StackName", stackId));+ if (xml.contains("<StackStatus>" + status + "</StackStatus>")) {+ return xml;+ }+ Thread.sleep(50);+ }+ fail("stack " + stackId + " never reached " + status + ": " + xml);+ return xml;+ }++ private static String cfn(Map<String, String> params) {+ return given()+ .contentType("application/x-www-form-urlencoded")+ .formParams(params)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().asString();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CustomResourceProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CustomResourceProvisionerTest.javaindex 629b26a234..59bfacc8e1 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CustomResourceProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CustomResourceProvisionerTest.java@@ -4,9 +4,11 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.docker.ContainerReachableEndpoint;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.cloudformation.provisioners.CfnResourceDispatcher;+import io.github.hectorvent.floci.services.cloudformation.provisioners.CfnRollback;import io.github.hectorvent.floci.services.lambda.LambdaService;import io.github.hectorvent.floci.services.lambda.model.InvocationType;import io.github.hectorvent.floci.services.lambda.model.InvokeResult;@@ -20,6 +22,8 @@ import java.util.Map;import java.util.function.Function;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.eq;@@ -77,23 +81,54 @@ class CustomResourceProvisionerTest {/** Stubs the Lambda to behave like a handler that PUTs a SUCCESS response to the ResponseURL. */private void stubHandler(String physicalId, Map<String, String> data) {+ respond(event -> success(physicalId, data));+ }++ /** Stubs the Lambda to PUT, for each event it is sent, the response {@code answer} builds. */+ private void respond(Function<JsonNode, ObjectNode> answer) {when(lambdaService.invoke(any(), eq(SERVICE_TOKEN), any(), eq(InvocationType.RequestResponse))).thenAnswer(inv -> {JsonNode event = mapper.readTree((byte[]) inv.getArgument(2));String responseUrl = event.get("ResponseURL").asText();String token = responseUrl.substring(responseUrl.lastIndexOf('/') + 1);-- ObjectNode response = mapper.createObjectNode();- response.put("Status", "SUCCESS");- response.put("PhysicalResourceId", physicalId);- ObjectNode dataNode = response.putObject("Data");- data.forEach(dataNode::put);- store.complete(token, response);-+ store.complete(token, answer.apply(event));return new InvokeResult(200, null, "null".getBytes(), null, "req-1");});}+ private ObjectNode success(String physicalId, Map<String, String> data) {+ ObjectNode response = mapper.createObjectNode();+ response.put("Status", "SUCCESS");+ response.put("PhysicalResourceId", physicalId);+ ObjectNode dataNode = response.putObject("Data");+ data.forEach(dataNode::put);+ return response;+ }++ private ObjectNode failed(String reason) {+ ObjectNode response = mapper.createObjectNode();+ response.put("Status", "FAILED");+ response.put("Reason", reason);+ return response;+ }++ private StackResource create() {+ return provisioner.provision("MyCr", "Custom::Test", props(),+ engine(), "us-east-1", "000000000000", "my-stack");+ }++ private StackResource update(StackResource current, String message) {+ ObjectNode changed = props();+ changed.put("Message", message);+ return provisioner.provision("MyCr", "Custom::Test", changed,+ engine(), "us-east-1", "000000000000", "my-stack",+ current.getPhysicalId(), current.getAttributes());+ }++ private static String message(JsonNode event, String properties) {+ return event.path(properties).path("Message").asText(null);+ }+@Testvoid createInvokesHandlerAndCapturesPhysicalIdAndData() {stubHandler("phys-123", Map.of("Greeting", "hello"));@@ -266,6 +301,229 @@ class CustomResourceProvisionerTest {assertTrue(AwsArnUtils.isArnFor(stackId, "cloudformation"), stackId);}+ @Test+ void changedUpdateRecordsTheRollbackSnapshotBeforeTheHandlerRuns() {+ respond(event -> "Update".equals(event.get("RequestType").asText())+ ? failed("boom") : success("phys-123", Map.of()));++ StackResource updated = update(create(), "bye");++ assertEquals("CREATE_FAILED", updated.getStatus());+ assertTrue(updated.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));+ }++ @Test+ void rollbackUpdateSendsTheOldPropertiesBackToTheSamePhysicalId() {+ respond(event -> {+ String message = message(event, "ResourceProperties");+ return success("phys-123", "bye".equals(message)+ ? Map.of("Greeting", message, "Farewell", "yes")+ : Map.of("Greeting", message));+ });+ StackResource created = create();+ String createdStash = created.getAttributes().get("__FlociResourceProperties");+ StackResource updated = update(created, "bye");+ List<String> progress = new ArrayList<>();++ assertTrue(provisioner.rollbackUpdate(updated, event -> progress.add(event.getResourceStatus())));++ JsonNode rollback = capturedEvents(3).get(2);+ assertEquals("Update", rollback.get("RequestType").asText());+ assertEquals("phys-123", rollback.get("PhysicalResourceId").asText());+ assertEquals("hi", message(rollback, "ResourceProperties"));+ assertEquals("bye", message(rollback, "OldResourceProperties"));+ assertEquals(STACK_ID, rollback.get("StackId").asText());+ assertEquals(List.of("UPDATE_IN_PROGRESS"), progress);+ assertEquals("phys-123", updated.getPhysicalId());+ assertEquals(createdStash, updated.getAttributes().get("__FlociResourceProperties"));+ assertEquals("hi", updated.getAttributes().get("Greeting"));+ assertFalse(updated.getAttributes().containsKey("Farewell"));+ assertFalse(updated.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));+ }++ @Test+ void rollbackUpdateThatTheHandlerFailsThrowsItsReasonAndKeepsTheSnapshot() {+ respond(event -> "bye".equals(message(event, "OldResourceProperties"))+ ? failed("cannot undo") : success("phys-123", Map.of()));+ StackResource updated = update(create(), "bye");++ AwsException failure = assertThrows(AwsException.class,+ () -> provisioner.rollbackUpdate(updated, event -> { }));++ assertTrue(failure.getMessage().contains("cannot undo"), failure.getMessage());+ assertTrue(updated.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));+ }++ @Test+ void retryOfAFailedRollbackSendsTheSameUpdateAgain() {+ respond(event -> "bye".equals(message(event, "OldResourceProperties"))+ ? failed("cannot undo") : success("phys-123", Map.of()));+ StackResource updated = update(create(), "bye");+ assertThrows(AwsException.class, () -> provisioner.rollbackUpdate(updated, event -> { }));++ assertThrows(AwsException.class, () -> provisioner.rollbackUpdate(updated, event -> { }));++ JsonNode retry = capturedEvents(4).get(3);+ assertEquals("Update", retry.get("RequestType").asText());+ assertEquals("hi", message(retry, "ResourceProperties"));+ assertEquals("bye", message(retry, "OldResourceProperties"));+ }++ @Test+ void deleteAfterAFailedRollbackCarriesTheOldProperties() {+ // AWS treats the rollback target as the resource's properties even though the handler failed it.+ respond(event -> "bye".equals(message(event, "OldResourceProperties"))+ ? failed("cannot undo") : success("phys-123", Map.of()));+ StackResource updated = update(create(), "bye");+ assertThrows(AwsException.class, () -> provisioner.rollbackUpdate(updated, event -> { }));++ provisioner.delete(updated, "us-east-1");++ JsonNode delete = capturedEvents(4).get(3);+ assertEquals("Delete", delete.get("RequestType").asText());+ assertEquals("phys-123", delete.get("PhysicalResourceId").asText());+ assertEquals("hi", message(delete, "ResourceProperties"));+ }++ @Test+ void failedUpdateIsRetainedForRollbackOnlyWhenItReachedTheHandler() {+ respond(event -> "bye".equals(message(event, "ResourceProperties"))+ ? failed("forced failure") : success("phys-123", Map.of()));+ StackResource created = create();++ StackResource failed = update(created, "bye");++ assertEquals("CREATE_FAILED", failed.getStatus());+ assertTrue(provisioner.retainsFailedUpdateState(failed));+ assertFalse(provisioner.retainsFailedUpdateState(created));+ }++ @Test+ void updateThatChangesTheServiceTokenFailsBeforeAnyHandlerIsSent() {+ stubHandler("phys-123", Map.of());+ StackResource created = create();+ ObjectNode otherToken = props();+ otherToken.put("ServiceToken", "arn:aws:lambda:us-east-1:000000000000:function:OtherHandler");++ StackResource failed = provisioner.provision("MyCr", "Custom::Test", otherToken,+ engine(), "us-east-1", "000000000000", "my-stack",+ created.getPhysicalId(), created.getAttributes());++ assertEquals("CREATE_FAILED", failed.getStatus());+ assertEquals("Modifying service token is not allowed.", failed.getStatusReason());+ assertFalse(provisioner.retainsFailedUpdateState(failed));+ verify(lambdaService, times(1)).invoke(any(), any(), any(), any());+ }++ @Test+ void updateOfAResourceWithNoStashedPropertiesHasNothingToRollBack() {+ stubHandler("phys-123", Map.of());+ StackResource created = create();+ created.getAttributes().remove("__FlociResourceProperties");++ StackResource updated = update(created, "bye");++ assertFalse(updated.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));+ assertTrue(provisioner.rollbackUpdate(updated, event -> { }));+ capturedEvents(2);+ }++ @Test+ void rollbackOfAnUpdateTheHandlerFailedSendsTheAttemptedPropertiesAsOld() {+ respond(event -> "bye".equals(message(event, "ResourceProperties"))+ ? failed("forced failure") : success("phys-123", Map.of()));+ StackResource created = create();+ String createdStash = created.getAttributes().get("__FlociResourceProperties");+ StackResource failed = update(created, "bye");++ assertTrue(provisioner.rollbackUpdate(failed, event -> { }));++ JsonNode rollback = capturedEvents(3).get(2);+ assertEquals("Update", rollback.get("RequestType").asText());+ assertEquals("phys-123", rollback.get("PhysicalResourceId").asText());+ assertEquals("hi", message(rollback, "ResourceProperties"));+ assertEquals("bye", message(rollback, "OldResourceProperties"));+ assertEquals("phys-123", failed.getPhysicalId());+ assertEquals(createdStash, failed.getAttributes().get("__FlociResourceProperties"));+ assertFalse(failed.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));+ }++ @Test+ void rollbackSendsTheStackIdTheFailedUpdateWasSent() {+ respond(event -> "bye".equals(message(event, "ResourceProperties"))+ ? failed("forced failure") : success("phys-123", Map.of()));+ StackResource created = create();+ created.getAttributes().remove("__FlociStackId");+ StackResource failed = update(created, "bye");++ assertTrue(provisioner.rollbackUpdate(failed, event -> { }));++ List<JsonNode> events = capturedEvents(3);+ assertEquals(STACK_ID, events.get(1).get("StackId").asText());+ assertEquals(STACK_ID, events.get(2).get("StackId").asText());+ }++ @Test+ void dataUnderTheReservedPrefixIsNotKept() {+ stubHandler("phys-123", Map.of("__FlociStackId", "forged", "Greeting", "hi"));++ StackResource created = create();++ assertEquals(STACK_ID, created.getAttributes().get("__FlociStackId"));+ assertEquals("hi", created.getAttributes().get("Greeting"));+ }++ @Test+ void rollbackOfAReplacingUpdateDeletesTheReplacementAndSendsNoUpdate() {+ respond(event -> {+ String message = message(event, "ResourceProperties");+ return success("bye".equals(message) ? "phys-456" : "phys-123", Map.of("Greeting", message));+ });+ StackResource created = create();+ String createdStash = created.getAttributes().get("__FlociResourceProperties");+ StackResource updated = update(created, "bye");+ assertEquals("phys-456", updated.getPhysicalId());++ assertTrue(provisioner.rollbackUpdate(updated, event -> { }));++ JsonNode delete = capturedEvents(3).get(2);+ assertEquals("Delete", delete.get("RequestType").asText());+ assertEquals("phys-456", delete.get("PhysicalResourceId").asText());+ assertEquals("bye", message(delete, "ResourceProperties"));+ assertEquals("phys-123", updated.getPhysicalId());+ assertEquals(createdStash, updated.getAttributes().get("__FlociResourceProperties"));+ assertEquals("hi", updated.getAttributes().get("Greeting"));+ assertFalse(updated.getAttributes().containsKey(CfnRollback.CUSTOM_RESOURCE_UPDATE_SNAPSHOT_ATTR));+ }++ @Test+ void rollbackResponseNamingAnotherPhysicalIdIsTakenAndTheIdSentIsDeleted() {+ respond(event -> success("bye".equals(message(event, "OldResourceProperties"))+ ? "phys-789" : "phys-123", Map.of()));+ StackResource updated = update(create(), "bye");++ assertTrue(provisioner.rollbackUpdate(updated, event -> { }));++ List<JsonNode> events = capturedEvents(4);+ assertEquals("Update", events.get(2).get("RequestType").asText());+ JsonNode delete = events.get(3);+ assertEquals("Delete", delete.get("RequestType").asText());+ assertEquals("phys-123", delete.get("PhysicalResourceId").asText());+ assertEquals("bye", message(delete, "ResourceProperties"));+ assertEquals("phys-789", updated.getPhysicalId());+ }++ @Test+ void rollbackAfterAnUpdateThatSentNothingSendsNothing() {+ stubHandler("phys-123", Map.of());+ StackResource changed = update(create(), "bye");+ StackResource unchanged = update(changed, "bye");++ assertTrue(provisioner.rollbackUpdate(unchanged, event -> { }));++ capturedEvents(2);+ }+private List<JsonNode> capturedEvents(int count) {ArgumentCaptor<byte[]> payloads = ArgumentCaptor.forClass(byte[].class);verify(lambdaService, times(count)).invoke(any(), eq(SERVICE_TOKEN), payloads.capture(),