Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): report and continue a failed REST API update rollback (#6568)
Jackson Pricee1726e839421
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ApiGatewayRestApiCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ApiGatewayRestApiCfnProvisioner.javaindex 3f442ebb95..7b45b40b34 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ApiGatewayRestApiCfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ApiGatewayRestApiCfnProvisioner.java@@ -179,6 +179,21 @@ public class ApiGatewayRestApiCfnProvisioner implements CfnResourceProvisioner {ReplacementCleanup.clear(resource);}+ /**+ * Keeps the snapshot of a kept API whose own undo did not complete. The engine restores the+ * previous resource and reports the rollback failure once; with the snapshot, a+ * ContinueUpdateRollback tries the restore again, or fails again when the update's document had+ * replaced the definition, instead of finding nothing to undo.+ */+ @Override+ public void mergeFailedUpdateResourceTracking(StackResource previous, StackResource attempted) {+ String snapshot = attempted.getAttributes().get(CfnRollback.REST_API_UPDATE_SNAPSHOT_ATTR);+ if (REST_API.equals(previous.getResourceType()) && snapshot != null+ && attempted.getAttributes().containsKey(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR)) {+ previous.getAttributes().put(CfnRollback.REST_API_UPDATE_SNAPSHOT_ATTR, snapshot);+ }+ }+/*** Puts a replaced RestApi, Resource or Method back when a later resource fails the update, and a* RestApi patched in place back to its snapshot. Without a replacement, a Resource update@@ -198,8 +213,8 @@ public class ApiGatewayRestApiCfnProvisioner implements CfnResourceProvisioner {}/**- * Patches the API back from the snapshot taken before its in-place update. An update that also- * re-applied an OpenAPI document replaced the API's resources, methods and models, which the+ * Patches the API back from the snapshot taken before its in-place update. An update whose+ * OpenAPI document was applied replaced the API's resources, methods and models, which the* snapshot does not hold, so its rollback is not implemented. With no snapshot, nothing changed.*/private boolean rollbackRestApiUpdate(StackResource resource) {@@ -286,12 +301,14 @@ public class ApiGatewayRestApiCfnProvisioner implements CfnResourceProvisioner {// applyOpenApiSpec is the only place that turns it into resources and methods. Resolved// before anything changes, so a document that cannot be read leaves the API as it was.JsonNode openApiDocument = OpenApiDocuments.resolve(props, engine, s3Service, objectMapper);+ // The declared Name and Description; null clears an undeclared Description.+ List<Map<String, String>> nameAndDescription = List.of(+ replacePatchOp("/name", name), replacePatchOp("/description", description));RestApi api;if (existing != null) {- snapshotBeforeUpdate(r, existing, region, openApiDocument != null);- List<Map<String, String>> operations = new ArrayList<>(List.of(- replacePatchOp("/name", name), replacePatchOp("/description", description)));+ snapshotBeforeUpdate(r, existing, region);+ List<Map<String, String>> operations = new ArrayList<>(nameAndDescription);if (endpoint != null) {operations.addAll(endpointPatchOps(existing.getEndpointConfiguration(),endpointTypes.isEmpty() ? null : endpointTypes.getFirst(),@@ -312,17 +329,22 @@ public class ApiGatewayRestApiCfnProvisioner implements CfnResourceProvisioner {}r.setPhysicalId(api.getId());boolean created = existing == null;- // putRestApi overwrites Name and Description from the document's info, so the declared Name- // and Description (null clearing an undeclared Description) are re-applied immediately after.- List<Map<String, String>> declaredNameAndDescription = List.of(- replacePatchOp("/name", name), replacePatchOp("/description", description));- // A step that fails once the API exists undoes this provision: an API it created is deleted- // again (or left marked for the stack's rollback when that delete fails too), and a kept API- // gets back what the update patched. putRestApi leaves the API as it was when it rejects the- // document.+ // A step that fails once the API exists undoes this provision. An API it created is deleted+ // again; if that delete fails too, a create leaves the API marked as created by the stack, and+ // an update lists it for the stack's next cleanup. A kept API gets back what the update+ // patched (putRestApi leaves the API as it was when it rejects the document); when its+ // document was already applied, the definition cannot be put back, which is reported as a+ // rollback failure.CfnRollback.guarded(r, ctx, created,() -> delete(REST_API, api.getId(), region),- () -> restore(api.getId(), readSnapshot(r)),+ () -> {+ JsonNode snapshot = readSnapshot(r);+ restore(api.getId(), snapshot);+ if (snapshot.path("body").asBoolean(false)) {+ throw new IllegalStateException("its OpenAPI document was already applied, and the"+ + " update keeps no copy of the definition it replaced");+ }+ },() -> {r.getAttributes().put("RestApiId", api.getId());// Not the first of getResources, which is the root only on a freshly created API.@@ -330,17 +352,12 @@ public class ApiGatewayRestApiCfnProvisioner implements CfnResourceProvisioner {apiGatewayService.findRootResourceId(region, api.getId()).orElseThrow());if (openApiDocument != null) {apiGatewayService.putRestApi(region, api.getId(), "overwrite", openApiDocument.toString());- if (created) {- apiGatewayService.updateRestApi(region, api.getId(), declaredNameAndDescription);- }+ markDocumentApplied(r);+ // putRestApi overwrites Name and Description from the document's info, so the+ // declared ones are applied again.+ apiGatewayService.updateRestApi(region, api.getId(), nameAndDescription);}});- // On a kept API the re-apply follows the guard: the document putRestApi accepted replaced the- // API's definition, which the snapshot does not hold, so putting back only the patch would- // not restore the API.- if (!created && openApiDocument != null) {- apiGatewayService.updateRestApi(region, api.getId(), declaredNameAndDescription);- }// An API created in place of one removed out of band is a replacement: a rollback deletes it.ReplacementCleanup.record(r, ctx, attributesBefore);}@@ -381,10 +398,11 @@ public class ApiGatewayRestApiCfnProvisioner implements CfnResourceProvisioner {/*** Keeps what an in-place update is about to patch, the name, description and endpoint- * configuration the API has now, so {@link #rollbackUpdate} can put them back, and whether the- * update also re-applies an OpenAPI document.+ * configuration the API has now, so {@link #rollbackUpdate} can put them back. Its {@code body}+ * flag starts false and is set by {@link #markDocumentApplied} once the update's OpenAPI+ * document has replaced the API's definition.*/- private void snapshotBeforeUpdate(StackResource r, RestApi existing, String region, boolean body) {+ private void snapshotBeforeUpdate(StackResource r, RestApi existing, String region) {ObjectNode snapshot = objectMapper.createObjectNode();snapshot.put("region", region);snapshot.put("name", existing.getName());@@ -394,7 +412,21 @@ public class ApiGatewayRestApiCfnProvisioner implements CfnResourceProvisioner {if (existing.getEndpointConfiguration() != null) {existing.getEndpointConfiguration().getVpcEndpointIds().forEach(vpcEndpointIds::add);}- snapshot.put("body", body);+ snapshot.put("body", false);+ r.getAttributes().put(CfnRollback.REST_API_UPDATE_SNAPSHOT_ATTR, snapshot.toString());+ }++ /**+ * Records on an in-place update's snapshot that putRestApi accepted its OpenAPI document, which+ * replaced the API's resources, methods and models: the snapshot cannot put those back. An API+ * this provision created has no snapshot.+ */+ private void markDocumentApplied(StackResource r) {+ if (!r.getAttributes().containsKey(CfnRollback.REST_API_UPDATE_SNAPSHOT_ATTR)) {+ return;+ }+ ObjectNode snapshot = (ObjectNode) readSnapshot(r);+ snapshot.put("body", true);r.getAttributes().put(CfnRollback.REST_API_UPDATE_SNAPSHOT_ATTR, snapshot.toString());}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnResourceDispatcher.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnResourceDispatcher.javaindex 369e514c1e..af122a69f1 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnResourceDispatcher.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnResourceDispatcher.java@@ -372,9 +372,10 @@ public class CfnResourceDispatcher {}/**- * Carries ownership discovered by a failed update onto the last known-good resource metadata- * that CloudFormation restores. Only additive cleanup tracking belongs here; normal attempted- * attributes must not overwrite the committed resource state.+ * Carries ownership discovered by a failed update, and the snapshot of an undo it could not+ * complete, onto the last known-good resource metadata that CloudFormation restores. Only+ * additive tracking belongs here; normal attempted attributes must not overwrite the committed+ * resource state.*/public void mergeFailedUpdateResourceTracking(StackResource previous, StackResource attempted) {// Any provisioner using ReplacementCleanup: an entity the failed attempt created and coulddiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnResourceProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnResourceProvisioner.javaindex 87ca06c1fa..187a5c73b2 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnResourceProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnResourceProvisioner.java@@ -150,12 +150,15 @@ public interface CfnResourceProvisioner {}/**- * Carries additive ownership tracking a failed update discovered onto the last known-good- * resource metadata CloudFormation restores, so entities the failed attempt created but could- * not remove are not orphaned. Only additive tracking belongs here; committed resource state- * must not be overwritten. The default has none to carry.+ * Carries additive tracking a failed update discovered onto the last known-good resource+ * metadata CloudFormation restores: entities the failed attempt created but could not remove,+ * so they are not orphaned, and the snapshot of an undo the attempt could not complete, so a+ * ContinueUpdateRollback can try it again, or report again that it cannot. Only additive+ * tracking belongs here; committed resource state must not be overwritten. The default has+ * none to carry.*/default void mergeFailedUpdateResourceTracking(StackResource previous, StackResource attempted) {- // no-op by default: only a provisioner that tracks generated sub-resources needs this+ // no-op by default: only a provisioner with tracking of its own to carry (generated+ // sub-resources, cleanup entries, an undo snapshot) needs this}}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.javaindex 429502eec6..e18fbea845 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java@@ -128,9 +128,11 @@ public final class CfnRollback {/*** Holds the name, description and endpoint configuration a REST API had before an in-place- * update patched them, and whether the update also re-applied an OpenAPI document, so a failed- * stack update can put them back. Written by {@code ApiGatewayRestApiCfnProvisioner} before its- * update call and spent by its {@code rollbackUpdate}.+ * update patched them, so a failed stack update can put them back, and a {@code body} flag set+ * once the update's OpenAPI document was applied, which the snapshot cannot undo. Written by+ * {@code ApiGatewayRestApiCfnProvisioner} before its update call and spent by its+ * {@code rollbackUpdate}; when its own undo did not complete, the provisioner carries it onto+ * the restored resource through {@code mergeFailedUpdateResourceTracking}.*/public static final String REST_API_UPDATE_SNAPSHOT_ATTR = "__FlociRestApiUpdateSnapshot";@@ -324,7 +326,9 @@ public final class CfnRollback {* deleted again by {@code undoCreate}, a reused one is put back by {@code undoInPlace}, which* restores the provisioner's snapshot and is a no-op when there is none. The engine only removes* a failed create it can see, and on a failed update it restores the previous resource metadata- * and discards the attempted one, snapshot included, so the undo has to happen here.+ * and discards the attempted one, snapshot included, so the undo has to happen here. A+ * provisioner whose snapshot a ContinueUpdateRollback still needs after an undo that failed+ * carries it in {@link CfnResourceProvisioner#mergeFailedUpdateResourceTracking}.** <p>A compensation that fails is reported, never hidden. A created entity carries* {@link #ROLLBACK_OWNED_ATTR} while the steps run, so the engine's create rollback deletes itdiff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/ApiGatewayRestApiCfnImportRollbackIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/ApiGatewayRestApiCfnImportRollbackIntegrationTest.javaindex 427992fc35..75086ceb98 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/ApiGatewayRestApiCfnImportRollbackIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/ApiGatewayRestApiCfnImportRollbackIntegrationTest.java@@ -9,16 +9,18 @@ import org.junit.jupiter.api.Test;import static io.restassured.RestAssured.given;import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.contains;import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.equalTo;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;/*** A stack whose AWS::ApiGateway::RestApi declares a Body that PutRestApi rejects fails to create.- * The REST API was created before the import ran, and the rollback removes it, so neither- * ROLLBACK_COMPLETE nor DeleteStack leaves it behind. A REST API of the same name that the stack- * did not create is left alone.+ * The provisioner created the REST API before the import ran and deletes it when the import fails,+ * so neither ROLLBACK_COMPLETE nor DeleteStack leaves it behind. A REST API of the same name that+ * the stack did not create is left alone.*/@QuarkusTestclass ApiGatewayRestApiCfnImportRollbackIntegrationTest {@@ -43,7 +45,7 @@ class ApiGatewayRestApiCfnImportRollbackIntegrationTest {}@Test- void rollbackRemovesTheRestApiWhoseBodyWasRejected() {+ void aCreateWhoseBodyIsRejectedLeavesNoRestApiBehind() {String stack = "apigw-cfn-restapi-import-rollback-it";String outsideId = given().contentType("application/json").body("{\"name\":\"" + API_NAME + "\",\"description\":\"outside the stack\"}")@@ -56,15 +58,16 @@ class ApiGatewayRestApiCfnImportRollbackIntegrationTest {assertEquals("ROLLBACK_COMPLETE", state.status(), state.reason());assertThat(CfnStackWaits.failedResourceReason(stack), containsString("authorizerResultTtlInSeconds"));createdId = physicalId(stack, "Api");+ assertNotNull(createdId, "the failed RestApi resource carries the id of the API it created");assertNotEquals(outsideId, createdId);given().when().get("/restapis/" + createdId).then().statusCode(404);- assertOutsideApiIntact(outsideId);+ assertOnlyTheOutsideApiIsLeft(outsideId);deleteStack(stack);stackDeleted = true;given().when().get("/restapis/" + createdId).then().statusCode(404);- assertOutsideApiIntact(outsideId);+ assertOnlyTheOutsideApiIsLeft(outsideId);} finally {try {if (!stackDeleted) {@@ -79,9 +82,11 @@ class ApiGatewayRestApiCfnImportRollbackIntegrationTest {}}- private static void assertOutsideApiIntact(String apiId) {- given().when().get("/restapis/" + apiId).then().statusCode(200)- .body("name", equalTo(API_NAME))+ /** Of the REST APIs named {@link #API_NAME}, only the one created outside the stack is left, unchanged. */+ private static void assertOnlyTheOutsideApiIsLeft(String outsideId) {+ given().when().get("/restapis").then().statusCode(200)+ .body("item.findAll { it.name == '" + API_NAME + "' }.id", contains(outsideId));+ given().when().get("/restapis/" + outsideId).then().statusCode(200).body("description", equalTo("outside the stack"));}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/ApiGatewayRestApiCfnUpdateRollbackRetryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/ApiGatewayRestApiCfnUpdateRollbackRetryIntegrationTest.javanew file mode 100644index 0000000000..cf9cb111b1--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/ApiGatewayRestApiCfnUpdateRollbackRetryIntegrationTest.java@@ -0,0 +1,177 @@+package io.github.hectorvent.floci.services.cloudformation;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.XmlParser;+import io.github.hectorvent.floci.services.apigateway.ApiGatewayService;+import io.github.hectorvent.floci.testing.CfnStackWaits;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.mockito.InjectSpy;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;+import org.mockito.ArgumentMatcher;+import org.mockito.Mockito;++import java.util.List;+import java.util.Map;++import static io.github.hectorvent.floci.testing.CfnRequests.cfn;+import static io.restassured.RestAssured.given;+import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.contains;+import static org.hamcrest.Matchers.containsInAnyOrder;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.argThat;+import static org.mockito.ArgumentMatchers.eq;++/**+ * An in-place update of an AWS::ApiGateway::RestApi whose own undo cannot complete leaves the stack+ * in UPDATE_ROLLBACK_FAILED, and ContinueUpdateRollback takes that undo up again: a restore that+ * failed is tried again, and an update whose OpenAPI document replaced the API's definition fails+ * again until the API is skipped. Each test injects one failure into ApiGatewayService, at the call+ * it is about.+ */+@QuarkusTest+class ApiGatewayRestApiCfnUpdateRollbackRetryIntegrationTest {++ /** The stack's template: the API's name, its description and endpoint type, then its Body. */+ private static final String TEMPLATE = """+ {"Resources":{"Api":{"Type":"AWS::ApiGateway::RestApi","Properties":{+ "Name":"%s","Description":"%s","EndpointConfiguration":{"Types":["%s"]}%s}}}}+ """;+ private static final String ACCEPTED_BODY = """+ ,"Body":{"openapi":"3.0.1","info":{"title":"replacing-document","version":"1"},+ "paths":{"/items":{"get":{"responses":{"200":{"description":"ok"}}}}}}""";+ /** A token authorizer whose cache TTL is above the 3600-second maximum, which the import rejects. */+ private static final String REJECTED_BODY = """+ ,"Body":{"openapi":"3.0.1","info":{"title":"rejected-document","version":"1"},+ "components":{"securitySchemes":{"token":{"type":"apiKey","name":"Authorization","in":"header",+ "x-amazon-apigateway-authtype":"custom",+ "x-amazon-apigateway-authorizer":{"type":"token","authorizerResultTtlInSeconds":3601,+ "authorizerUri":{"Fn::Sub":"arn:${AWS::Partition}:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:authorizer/invocations"}}}}},+ "paths":{"/items":{"get":{"security":[{"token":[]}],"responses":{"200":{"description":"ok"}}}}}}""";++ @InjectSpy+ ApiGatewayService apiGatewayService;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void aRetryFailsAgainOnceTheUpdatesDocumentReplacedTheApi() {+ String stack = "apigw-cfn-restapi-retry-document-it";+ String name = "cfn-restapi-retry-document";+ try {+ stackAction(stack, "CreateStack", TEMPLATE.formatted(name, "first", "REGIONAL", ""));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ String apiId = physicalId(stack);+ // After putRestApi accepted the document, applying the declared name and description+ // again (just those two operations) fails once.+ Mockito.doThrow(new AwsException("TooManyRequestsException", "Too Many Requests", 429))+ .doCallRealMethod()+ .when(apiGatewayService).updateRestApi(anyString(), eq(apiId),+ argThat(onlyNameAndDescription(name, "second")));++ stackAction(stack, "UpdateStack", TEMPLATE.formatted(name, "second", "EDGE", ACCEPTED_BODY));+ CfnStackWaits.StackState state = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_FAILED", state.status(), state.reason());+ assertThat(apiResourceReason(stack), containsString("OpenAPI document was already applied"));+ // The patch is put back; the definition the document replaced is not.+ given().when().get("/restapis/" + apiId).then().statusCode(200)+ .body("name", equalTo(name))+ .body("description", equalTo("first"))+ .body("endpointConfiguration.types", contains("REGIONAL"));+ assertPaths(apiId, "/", "/items");++ cfn("ContinueUpdateRollback", stack).when().post("/").then().statusCode(200);+ state = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_FAILED", state.status(), state.reason());+ assertEquals("Rollback is not implemented for AWS::ApiGateway::RestApi", apiResourceReason(stack));++ cfn("ContinueUpdateRollback", stack).formParam("ResourcesToSkip.member.1", "Api")+ .when().post("/").then().statusCode(200);+ state = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", state.status(), state.reason());+ } finally {+ deleteStack(stack);+ }+ }++ @Test+ void aRetryPutsBackThePatchWhoseFirstRestoreFailed() {+ String stack = "apigw-cfn-restapi-retry-restore-it";+ String name = "cfn-restapi-retry-restore";+ try {+ stackAction(stack, "CreateStack", TEMPLATE.formatted(name, "first", "REGIONAL", ""));+ assertEquals("CREATE_COMPLETE", CfnStackWaits.awaitTerminal(stack).status());+ String apiId = physicalId(stack);+ // The import rejects the document, and the first restore of the patch fails.+ ArgumentMatcher<List<Map<String, String>>> restore =+ operations -> operations != null && operations.contains(replace("/description", "first"));+ Mockito.doThrow(new AwsException("TooManyRequestsException", "Too Many Requests", 429))+ .doCallRealMethod()+ .when(apiGatewayService).updateRestApi(anyString(), eq(apiId), argThat(restore));++ stackAction(stack, "UpdateStack", TEMPLATE.formatted(name, "second", "EDGE", REJECTED_BODY));+ CfnStackWaits.StackState state = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_FAILED", state.status(), state.reason());+ given().when().get("/restapis/" + apiId).then().statusCode(200)+ .body("description", equalTo("second"))+ .body("endpointConfiguration.types", contains("EDGE"));++ cfn("ContinueUpdateRollback", stack).when().post("/").then().statusCode(200);+ state = CfnStackWaits.awaitTerminal(stack);+ assertEquals("UPDATE_ROLLBACK_COMPLETE", state.status(), state.reason());+ given().when().get("/restapis/" + apiId).then().statusCode(200)+ .body("name", equalTo(name))+ .body("description", equalTo("first"))+ .body("endpointConfiguration.types", contains("REGIONAL"));+ assertPaths(apiId, "/");+ } finally {+ deleteStack(stack);+ }+ }++ /** The re-apply of the declared name and description: exactly those two operations. */+ private static ArgumentMatcher<List<Map<String, String>>> onlyNameAndDescription(String name, String description) {+ return operations -> operations != null && operations.size() == 2+ && operations.contains(replace("/name", name))+ && operations.contains(replace("/description", description));+ }++ private static Map<String, String> replace(String path, String value) {+ return Map.of("op", "replace", "path", path, "value", value);+ }++ private static void assertPaths(String apiId, String... paths) {+ given().when().get("/restapis/" + apiId + "/resources").then().statusCode(200)+ .body("item.path", containsInAnyOrder(paths));+ }++ private static String physicalId(String stack) {+ return XmlParser.extractFirst(apiResource(stack), "PhysicalResourceId", null);+ }++ private static String apiResourceReason(String stack) {+ return XmlParser.extractFirst(apiResource(stack), "ResourceStatusReason", null);+ }++ private static String apiResource(String stack) {+ return cfn("DescribeStackResource", stack).formParam("LogicalResourceId", "Api")+ .when().post("/").then().statusCode(200).extract().asString();+ }++ private static void stackAction(String stack, String action, String template) {+ cfn(action, stack).formParam("TemplateBody", template).when().post("/").then().statusCode(200);+ }++ private static void deleteStack(String stack) {+ cfn("DeleteStack", stack).when().post("/").then().statusCode(200);+ CfnStackWaits.awaitStackDeleted(stack);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ApiGatewayRestApiCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ApiGatewayRestApiCfnProvisionerTest.javaindex ebe4a91f3a..a65fdfe23e 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ApiGatewayRestApiCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/ApiGatewayRestApiCfnProvisionerTest.java@@ -229,6 +229,107 @@ class ApiGatewayRestApiCfnProvisionerTest {verify(api, never()).deleteRestApi(anyString(), anyString());}+ @Test+ void restApiUpdateReportsARollbackFailureWhenItsPatchCannotBePutBack() throws Exception {+ RestApi existing = restApi("api-1", "shop", "v1", "REGIONAL");+ existingApi(existing);+ when(api.putRestApi(eq("us-east-1"), eq("api-1"), eq("overwrite"), anyString()))+ .thenThrow(new AwsException("BadRequestException", "Invalid OpenAPI input", 400));+ AwsException restoreFailure = new AwsException("TooManyRequestsException", "Too Many Requests", 429);+ when(api.updateRestApi(eq("us-east-1"), eq("api-1"), any())).thenReturn(existing).thenThrow(restoreFailure);+ StackResource r = resource("AWS::ApiGateway::RestApi", "Api");++ AwsException failure = assertThrows(AwsException.class, () -> provisioner.provision(r, props("""+ {"Name": "shop", "Description": "v2", "Body": {"openapi": "3.0.1", "paths": {}}}+ """), ctx("api-1")));++ assertEquals("BadRequestException", failure.getErrorCode());+ assertSame(restoreFailure, failure.getSuppressed()[0]);+ assertTrue(r.getAttributes().getOrDefault(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR, "").contains("api-1"));+ verify(api, never()).deleteRestApi(anyString(), anyString());+ }++ @Test+ void restApiUpdateReportsARollbackFailureOnceItsDocumentWasApplied() throws Exception {+ StackResource r = updateWhoseReapplyFailsAfterItsDocumentWasApplied();++ // The patch is put back, but the definition the document replaced cannot be, which is+ // reported rather than claimed restored.+ verify(api).updateRestApi("us-east-1", "api-1", List.of(+ op("replace", "/name", "shop"), op("replace", "/description", "v1")));+ assertTrue(r.getAttributes().getOrDefault(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR, "")+ .contains("OpenAPI document was already applied"));+ verify(api, never()).deleteRestApi(anyString(), anyString());+ }++ @Test+ void restApiRetriedRollbackFailsAgainOnceTheDocumentWasApplied() throws Exception {+ StackResource attempted = updateWhoseReapplyFailsAfterItsDocumentWasApplied();+ StackResource previous = committedRestApi("api-1");++ provisioner.mergeFailedUpdateResourceTracking(previous, attempted);++ // ContinueUpdateRollback asks again, and is told again that this rollback cannot be done.+ assertTrue(previous.getAttributes().containsKey(CfnRollback.REST_API_UPDATE_SNAPSHOT_ATTR));+ assertFalse(provisioner.rollbackUpdate(previous));+ assertFalse(provisioner.rollbackUpdate(previous));+ }++ @Test+ void restApiRetriedRollbackPutsBackThePatchWhoseFirstRestoreFailed() throws Exception {+ RestApi existing = restApi("api-1", "shop", "v1", "REGIONAL");+ existingApi(existing);+ when(api.putRestApi(eq("us-east-1"), eq("api-1"), eq("overwrite"), anyString()))+ .thenThrow(new AwsException("BadRequestException", "Invalid OpenAPI input", 400));+ when(api.updateRestApi(eq("us-east-1"), eq("api-1"), any())).thenReturn(existing)+ .thenThrow(new AwsException("TooManyRequestsException", "Too Many Requests", 429))+ .thenReturn(existing);+ StackResource attempted = resource("AWS::ApiGateway::RestApi", "Api");+ assertThrows(AwsException.class, () -> provisioner.provision(attempted, props("""+ {"Name": "shop", "Description": "v2", "Body": {"openapi": "3.0.1", "paths": {}}}+ """), ctx("api-1")));+ StackResource previous = committedRestApi("api-1");++ provisioner.mergeFailedUpdateResourceTracking(previous, attempted);++ assertTrue(provisioner.rollbackUpdate(previous));+ verify(api, times(2)).updateRestApi("us-east-1", "api-1", List.of(+ op("replace", "/name", "shop"), op("replace", "/description", "v1")));+ assertFalse(previous.getAttributes().containsKey(CfnRollback.REST_API_UPDATE_SNAPSHOT_ATTR));+ }++ @Test+ void restApiFailedUpdateWhoseOwnUndoCompletedLeavesNoSnapshotForARetry() throws Exception {+ existingApi(restApi("api-1", "shop", "v1", "REGIONAL"));+ when(api.putRestApi(eq("us-east-1"), eq("api-1"), eq("overwrite"), anyString()))+ .thenThrow(new AwsException("BadRequestException", "Invalid OpenAPI input", 400));+ StackResource attempted = resource("AWS::ApiGateway::RestApi", "Api");+ assertThrows(AwsException.class, () -> provisioner.provision(attempted, props("""+ {"Name": "shop", "Description": "v2", "Body": {"openapi": "3.0.1", "paths": {}}}+ """), ctx("api-1")));+ StackResource previous = committedRestApi("api-1");++ provisioner.mergeFailedUpdateResourceTracking(previous, attempted);++ // The patch was already put back, so the restored resource owes no rollback.+ assertFalse(previous.getAttributes().containsKey(CfnRollback.REST_API_UPDATE_SNAPSHOT_ATTR));+ }++ @Test+ void restApiUpdatePutsThePatchBackWhenTheRootResourceIsMissing() throws Exception {+ existingApi(restApi("api-1", "shop", "v1", "REGIONAL"));+ when(api.findRootResourceId("us-east-1", "api-1")).thenReturn(Optional.empty());+ StackResource r = resource("AWS::ApiGateway::RestApi", "Api");++ assertThrows(RuntimeException.class, () -> provisioner.provision(r, props("""+ {"Name": "shop", "Description": "v2"}+ """), ctx("api-1")));++ verify(api).updateRestApi("us-east-1", "api-1", List.of(+ op("replace", "/name", "shop"), op("replace", "/description", "v1")));+ verify(api, never()).deleteRestApi(anyString(), anyString());+ }+@Testvoid restApiUpdateRecreatesAnApiRemovedOutOfBandAndItsRollbackDeletesIt() throws Exception {when(api.getRestApi("us-east-1", "api-1"))@@ -264,6 +365,25 @@ class ApiGatewayRestApiCfnProvisionerTest {verify(api).deleteRestApi("us-east-1", "api-2");}+ @Test+ void restApiUpdateDeletesTheRecreatedApiWhenReapplyingItsNameFails() throws Exception {+ when(api.getRestApi("us-east-1", "api-1"))+ .thenThrow(new AwsException("NotFoundException", "Invalid API id specified", 404));+ when(api.createRestApi(eq("us-east-1"), anyMap())).thenReturn(restApi("api-2", "shop", null, "REGIONAL"));+ when(api.findRootResourceId("us-east-1", "api-2")).thenReturn(Optional.of("root-2"));+ AwsException reapplyFailure = new AwsException("TooManyRequestsException", "Too Many Requests", 429);+ when(api.updateRestApi(eq("us-east-1"), eq("api-2"), any())).thenThrow(reapplyFailure);+ StackResource r = resource("AWS::ApiGateway::RestApi", "Api");++ AwsException failure = assertThrows(AwsException.class, () -> provisioner.provision(r, props("""+ {"Name": "shop", "Body": {"openapi": "3.0.1", "paths": {}}}+ """), ctx("api-1")));++ assertSame(reapplyFailure, failure);+ verify(api).deleteRestApi("us-east-1", "api-2");+ assertFalse(r.getAttributes().containsKey(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR));+ }+@Testvoid restApiUpdateListsARecreatedApiItCannotDeleteAndReportsTheRollbackFailure() throws Exception {when(api.getRestApi("us-east-1", "api-1"))@@ -598,6 +718,37 @@ class ApiGatewayRestApiCfnProvisionerTest {return new ProvisionContext(engine, "us-east-1", "000000000000", "my-stack", priorPhysicalId);}+ /**+ * Runs an in-place update of api-1 that putRestApi accepts and whose re-applied name and+ * description then fail, and returns the attempted resource it leaves.+ */+ private StackResource updateWhoseReapplyFailsAfterItsDocumentWasApplied() throws Exception {+ RestApi existing = restApi("api-1", "shop", "v1", "REGIONAL");+ existingApi(existing);+ AwsException reapplyFailure = new AwsException("NotFoundException", "Invalid API id specified", 404);+ // The update's patch, the re-apply that fails, then the restore of the patch.+ when(api.updateRestApi(eq("us-east-1"), eq("api-1"), any()))+ .thenReturn(existing).thenThrow(reapplyFailure).thenReturn(existing);+ StackResource r = resource("AWS::ApiGateway::RestApi", "Api");++ AwsException failure = assertThrows(AwsException.class, () -> provisioner.provision(r, props("""+ {"Name": "shop", "Description": "v2", "Body": {"openapi": "3.0.1", "info": {"title": "other"}, "paths": {}}}+ """), ctx("api-1")));++ assertSame(reapplyFailure, failure);+ verify(api).putRestApi(eq("us-east-1"), eq("api-1"), eq("overwrite"), anyString());+ return r;+ }++ /** The resource a stack held for api-1 before the update, as the engine restores it. */+ private static StackResource committedRestApi(String apiId) {+ StackResource r = resource("AWS::ApiGateway::RestApi", "Api");+ r.setPhysicalId(apiId);+ r.getAttributes().put("RestApiId", apiId);+ r.getAttributes().put("RootResourceId", "root-1");+ return r;+ }+/** Stubs what an in-place update of {@code existing} looks up and patches. */private void existingApi(RestApi existing) {when(api.getRestApi("us-east-1", existing.getId())).thenReturn(existing);