Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): improve EC2 Instance update fidelity (#4130)
Hector Venturac847a9982839
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.javaindex e00b7c4af8..74e1067451 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.java@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.services.ec2.model.GroupIdentifier;import io.github.hectorvent.floci.services.ec2.model.Instance;import io.github.hectorvent.floci.services.ec2.model.InstanceState;import io.github.hectorvent.floci.services.ec2.model.LaunchTemplateData;@@ -15,6 +16,7 @@ import jakarta.inject.Inject;import java.util.ArrayList;import java.util.HashMap;+import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Set;@@ -29,6 +31,12 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {private static final String INSTANCE = "AWS::EC2::Instance";+ // Declared values of createOnly properties the launch does not apply to the instance, recorded+ // so a later update can tell a genuine change from the instance's auto-assigned value. The+ // __Floci prefix keeps them off the published Fn::GetAtt attributes.+ private static final String DECLARED_PRIVATE_IP_ATTR = "__FlociDeclaredPrivateIpAddress";+ private static final String DECLARED_AZ_ATTR = "__FlociDeclaredAvailabilityZone";+private final Ec2Service ec2Service;@Inject@@ -77,15 +85,28 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {instanceType = "t3.micro";}String subnetId = ctx.resolveOptional(props, "SubnetId");+ List<String> securityGroupIds = resolveSecurityGroupIds(props, engine);+ String privateIpAddress = ctx.resolveOptional(props, "PrivateIpAddress");+ String availabilityZone = ctx.resolveOptional(props, "AvailabilityZone");- // On update, keep the existing instance unless a createOnly property changed: only a- // changed ImageId, SubnetId or KeyName replaces it, a mutable-only change (tags, and the- // like) reuses it. Launching unconditionally would leak the prior instance every update.+ // On update, keep the existing instance unless a createOnly property changed: a changed+ // ImageId, SubnetId or KeyName replaces it (compared against the instance, which carries+ // them), as does a changed PrivateIpAddress or AvailabilityZone (compared against the value+ // declared last time, since the launch does not apply those to the instance). A mutable-only+ // change (tags, and the like) reuses it. Launching unconditionally would leak the prior+ // instance every update.Instance prior = ctx.isUpdate() ? findInstance(region, ctx.priorPhysicalId()) : null;- if (prior != null && !createOnlyChanged(prior, imageId, subnetId, keyName)) {- r.setPhysicalId(prior.getInstanceId());- publishInstanceAttributes(r, prior);- Ec2Tags.reconcile(ec2Service, region, prior.getInstanceId(), ctx.resolveTags(props, "Tags"));+ if (prior != null && !createOnlyChanged(prior, imageId, subnetId, keyName)+ && !declaredCreateOnlyChanged(attributesBefore, privateIpAddress, availabilityZone)) {+ // A reused instance still has its mutable properties reconciled to the template rather+ // than left at their prior values; re-read it so the published attributes reflect them.+ reconcileMutableProperties(region, prior, instanceType, securityGroupIds);+ Instance reconciled = findInstance(region, prior.getInstanceId());+ Instance current = reconciled != null ? reconciled : prior;+ r.setPhysicalId(current.getInstanceId());+ publishInstanceAttributes(r, current);+ storeDeclaredCreateOnly(r, privateIpAddress, availabilityZone);+ Ec2Tags.reconcile(ec2Service, region, current.getInstanceId(), ctx.resolveTags(props, "Tags"));ReplacementCleanup.record(r, ctx, attributesBefore);return;}@@ -93,13 +114,6 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {String userData = ctx.resolveOptional(props, "UserData");String iamInstanceProfile = ctx.resolveOptional(props, "IamInstanceProfile");- List<String> securityGroupIds = new ArrayList<>();- if (props != null && props.has("SecurityGroupIds") && props.get("SecurityGroupIds").isArray()) {- for (JsonNode sg : props.get("SecurityGroupIds")) {- securityGroupIds.add(engine.resolve(sg));- }- }-List<Tag> tags = new ArrayList<>();JsonNode tagsNode = props != null ? engine.resolveNode(props.get("Tags")) : null;if (tagsNode != null && tagsNode.isArray()) {@@ -134,6 +148,7 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {ec2Service.awaitContainerLaunch(instance);r.getAttributes().remove(CfnRollback.ROLLBACK_OWNED_ATTR);publishInstanceAttributes(r, instance);+ storeDeclaredCreateOnly(r, privateIpAddress, availabilityZone);// A createOnly change that landed a new instance id replaced the prior one: record it so// the stack cleans the displaced instance up after the update commits.ReplacementCleanup.record(r, ctx, attributesBefore);@@ -179,6 +194,80 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {return declared != null && !declared.equals(actual);}+ private List<String> resolveSecurityGroupIds(JsonNode props, CloudFormationTemplateEngine engine) {+ List<String> securityGroupIds = new ArrayList<>();+ if (props != null && props.has("SecurityGroupIds") && props.get("SecurityGroupIds").isArray()) {+ for (JsonNode sg : props.get("SecurityGroupIds")) {+ securityGroupIds.add(engine.resolve(sg));+ }+ }+ return securityGroupIds;+ }++ /**+ * Applies the mutable properties CloudFormation allows to change without replacing the instance:+ * a changed InstanceType is resized in place and a changed SecurityGroupIds set is reattached,+ * through the same Ec2Service calls the ModifyInstanceAttribute API uses. Only a declared+ * SecurityGroupIds is reconciled, so a template that omits it does not strip the groups the+ * instance already has. UserData and IamInstanceProfile have no in-place Ec2Service path yet, so+ * they stay a follow-up: an instance kept on update still carries their prior values.+ */+ private void reconcileMutableProperties(String region, Instance prior, String instanceType,+ List<String> securityGroupIds) {+ if (instanceType != null && !instanceType.isBlank()+ && !instanceType.equals(prior.getInstanceType())) {+ ec2Service.modifyInstanceAttribute(region, prior.getInstanceId(), "instanceType", instanceType);+ }+ if (!securityGroupIds.isEmpty() && !sameSecurityGroups(prior, securityGroupIds)) {+ ec2Service.modifyInstanceGroups(region, prior.getInstanceId(), securityGroupIds);+ }+ }++ private static boolean sameSecurityGroups(Instance prior, List<String> desired) {+ List<GroupIdentifier> current = prior.getSecurityGroups();+ if (current == null) {+ return desired.isEmpty();+ }+ Set<String> currentIds = new HashSet<>();+ for (GroupIdentifier group : current) {+ currentIds.add(group.getGroupId());+ }+ return currentIds.equals(new HashSet<>(desired));+ }++ /**+ * Whether a createOnly property the launch does not apply to the instance (PrivateIpAddress,+ * AvailabilityZone) changed since the last provision. It is compared against the value declared+ * then and recorded on the resource, not the instance's auto-assigned value, which never matches+ * a declared one. A property with no recorded prior value is not treated as changed, so adopting+ * this behavior does not replace instances whose template did not actually change; the value is+ * recorded from this provision on, and a later change to it then forces replacement.+ */+ private static boolean declaredCreateOnlyChanged(Map<String, String> priorAttributes,+ String privateIpAddress, String availabilityZone) {+ return recordedValueChanged(priorAttributes.get(DECLARED_PRIVATE_IP_ATTR), privateIpAddress)+ || recordedValueChanged(priorAttributes.get(DECLARED_AZ_ATTR), availabilityZone);+ }++ private static boolean recordedValueChanged(String priorDeclared, String nowDeclared) {+ String now = (nowDeclared == null || nowDeclared.isBlank()) ? null : nowDeclared;+ return priorDeclared != null && !priorDeclared.equals(now);+ }++ private static void storeDeclaredCreateOnly(StackResource r, String privateIpAddress,+ String availabilityZone) {+ putOrRemove(r, DECLARED_PRIVATE_IP_ATTR, privateIpAddress);+ putOrRemove(r, DECLARED_AZ_ATTR, availabilityZone);+ }++ private static void putOrRemove(StackResource r, String key, String value) {+ if (value != null && !value.isBlank()) {+ r.getAttributes().put(key, value);+ } else {+ r.getAttributes().remove(key);+ }+ }+private void publishInstanceAttributes(StackResource r, Instance instance) {// Each attribute is written when present and removed when absent, so a replacement never// inherits the prior instance's value: on update the resource still carries the olddiff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEc2InstanceIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEc2InstanceIntegrationTest.javaindex 08cb32bec1..2bc8471548 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEc2InstanceIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEc2InstanceIntegrationTest.java@@ -8,6 +8,7 @@ import java.util.regex.Pattern;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.containsString;+import static org.junit.jupiter.api.Assertions.assertNotEquals;import static org.junit.jupiter.api.Assertions.assertTrue;/**@@ -83,4 +84,114 @@ class CloudFormationEc2InstanceIntegrationTest {.statusCode(200).body(containsString(instanceId));}++ @Test+ void updateStackResizesInstanceTypeInPlaceWithoutReplacingTheInstance() {+ String suffix = Long.toString(System.nanoTime(), 36);+ String stackName = "cfn-ec2-resize-" + suffix;+ String template = """+ {+ "Resources": {+ "Server": {+ "Type": "AWS::EC2::Instance",+ "Properties": {"ImageId": "ami-12345678", "InstanceType": "%s"}+ }+ },+ "Outputs": {+ "InstanceId": {"Value": {"Ref": "Server"}}+ }+ }+ """;++ cfn("CreateStack", stackName, template.formatted("t3.micro"));+ String createXml = describeStacks(stackName, "CREATE_COMPLETE");+ Matcher m = Pattern.compile("<OutputValue>(i-[0-9a-fA-F]+)</OutputValue>").matcher(createXml);+ assertTrue(m.find(), "expected an instance id in the stack outputs");+ String instanceId = m.group(1);++ cfn("UpdateStack", stackName, template.formatted("t3.small"));+ String updateXml = describeStacks(stackName, "UPDATE_COMPLETE");++ // Ref is unchanged: InstanceType is mutable, so the instance was resized in place rather+ // than replaced with a new one.+ assertTrue(updateXml.contains("<OutputValue>" + instanceId + "</OutputValue>"),+ "instance id changed on update, so it was replaced not resized: " + updateXml);++ // DescribeInstances reports the new type on the same instance.+ given()+ .contentType("application/x-www-form-urlencoded")+ .header("Authorization", EC2_AUTH)+ .formParam("Action", "DescribeInstances")+ .formParam("InstanceId.1", instanceId)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<instanceType>t3.small</instanceType>"));+ }++ @Test+ void changingPrivateIpAddressReplacesTheInstance() {+ String suffix = Long.toString(System.nanoTime(), 36);+ String stackName = "cfn-ec2-replace-" + suffix;+ String template = """+ {+ "Resources": {+ "Server": {+ "Type": "AWS::EC2::Instance",+ "Properties": {"ImageId": "ami-12345678", "InstanceType": "t3.micro",+ "PrivateIpAddress": "%s"}+ }+ },+ "Outputs": {+ "InstanceId": {"Value": {"Ref": "Server"}}+ }+ }+ """;++ cfn("CreateStack", stackName, template.formatted("10.0.0.5"));+ String firstId = instanceIdFrom(describeStacks(stackName, "CREATE_COMPLETE"));++ // PrivateIpAddress is createOnly, so changing it replaces the instance rather than reusing it.+ cfn("UpdateStack", stackName, template.formatted("10.0.0.6"));+ String secondId = instanceIdFrom(describeStacks(stackName, "UPDATE_COMPLETE"));++ assertNotEquals(firstId, secondId, "PrivateIpAddress changed, so the instance must be replaced");++ // The replacement instance really exists in EC2.+ given()+ .contentType("application/x-www-form-urlencoded")+ .header("Authorization", EC2_AUTH)+ .formParam("Action", "DescribeInstances")+ .formParam("InstanceId.1", secondId)+ .when().post("/").then().statusCode(200)+ .body(containsString(secondId));+ }++ private static void cfn(String action, String stackName, String template) {+ given()+ .contentType("application/x-www-form-urlencoded")+ .header("Authorization", CFN_AUTH)+ .formParam("Action", action)+ .formParam("StackName", stackName)+ .formParam("TemplateBody", template)+ .when().post("/").then().statusCode(200);+ }++ private static String describeStacks(String stackName, String expectedStatus) {+ return given()+ .contentType("application/x-www-form-urlencoded")+ .header("Authorization", CFN_AUTH)+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", stackName)+ .when().post("/").then().statusCode(200)+ .body(containsString("<StackStatus>" + expectedStatus + "</StackStatus>"))+ .extract().asString();+ }++ private static String instanceIdFrom(String stackXml) {+ Matcher m = Pattern.compile("<OutputValue>(i-[0-9a-fA-F]+)</OutputValue>").matcher(stackXml);+ assertTrue(m.find(), "expected an instance id in the stack outputs: " + stackXml);+ return m.group(1);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.javaindex 652ebaf0c9..fe94f95ff0 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.java@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.services.ec2.model.GroupIdentifier;import io.github.hectorvent.floci.services.ec2.model.Instance;import io.github.hectorvent.floci.services.ec2.model.InstanceState;import io.github.hectorvent.floci.services.ec2.model.LaunchTemplateData;@@ -15,6 +16,7 @@ import io.github.hectorvent.floci.services.ec2.model.Tag;import org.junit.jupiter.api.Test;import org.mockito.ArgumentCaptor;+import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;@@ -32,6 +34,7 @@ import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@@ -206,6 +209,92 @@ class Ec2InstanceCfnProvisionerTest {assertEquals("i-1", provisioner.updateCleanupPhysicalId(r));}+ @Test+ void changingPrivateIpAddressReplacesTheInstanceAndRecordsTheCleanup() throws Exception {+ Instance i1 = new Instance();+ i1.setInstanceId("i-1");+ i1.setImageId("ami-1");+ Instance i2 = new Instance();+ i2.setInstanceId("i-2");+ i2.setImageId("ami-1");+ when(ec2.runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any()))+ .thenReturn(reservationOf(i1), reservationOf(i2));+ StackResource r = resource("Server");+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\", \"PrivateIpAddress\": \"10.0.0.5\"}"), ctx(null));+ assertEquals("i-1", r.getPhysicalId());++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(i1)));+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\", \"PrivateIpAddress\": \"10.0.0.6\"}"), ctx("i-1"));++ assertEquals("i-2", r.getPhysicalId());+ assertTrue(provisioner.hasReplacementUpdate(r));+ assertEquals("i-1", provisioner.updateCleanupPhysicalId(r));+ }++ @Test+ void changingAvailabilityZoneReplacesTheInstance() throws Exception {+ Instance i1 = new Instance();+ i1.setInstanceId("i-1");+ i1.setImageId("ami-1");+ Instance i2 = new Instance();+ i2.setInstanceId("i-2");+ i2.setImageId("ami-1");+ when(ec2.runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any()))+ .thenReturn(reservationOf(i1), reservationOf(i2));+ StackResource r = resource("Server");+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\", \"AvailabilityZone\": \"us-east-1a\"}"), ctx(null));++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(i1)));+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\", \"AvailabilityZone\": \"us-east-1b\"}"), ctx("i-1"));++ assertEquals("i-2", r.getPhysicalId());+ assertTrue(provisioner.hasReplacementUpdate(r));+ }++ @Test+ void unchangedDeclaredCreateOnlyPropertiesReuseTheInstance() throws Exception {+ Instance i1 = new Instance();+ i1.setInstanceId("i-1");+ i1.setImageId("ami-1");+ stubLaunch(i1);+ StackResource r = resource("Server");+ String template = "{\"ImageId\": \"ami-1\", \"PrivateIpAddress\": \"10.0.0.5\", "+ + "\"AvailabilityZone\": \"us-east-1a\"}";+ provisioner.provision(r, props(template), ctx(null));++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(i1)));+ provisioner.provision(r, props(template), ctx("i-1"));++ assertEquals("i-1", r.getPhysicalId());+ verify(ec2, times(1)).runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(),+ anyList(), any(), any(), anyList(), any(), any(), any());+ }++ @Test+ void aNewlyDeclaredCreateOnlyValueWithNoPriorRecordDoesNotReplaceTheInstance() throws Exception {+ Instance i1 = new Instance();+ i1.setInstanceId("i-1");+ i1.setImageId("ami-1");+ stubLaunch(i1);+ StackResource r = resource("Server");+ // The prior provision declared no PrivateIpAddress, so none was recorded; a first declared+ // value is not treated as a change, only recorded for next time.+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx(null));++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(i1)));+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\", \"PrivateIpAddress\": \"10.0.0.5\"}"), ctx("i-1"));++ assertEquals("i-1", r.getPhysicalId());+ verify(ec2, times(1)).runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(),+ anyList(), any(), any(), anyList(), any(), any(), any());+ }+@Testvoid reusedInstanceReconcilesTagsInPlace() throws Exception {Instance i1 = new Instance();@@ -232,6 +321,62 @@ class Ec2InstanceCfnProvisionerTest {assertEquals("old", removed.getValue().get(0).getKey());}+ @Test+ void reusedInstanceResizesInstanceTypeInPlace() throws Exception {+ Instance i1 = new Instance();+ i1.setInstanceId("i-1");+ i1.setImageId("ami-1");+ i1.setInstanceType("t3.micro");+ stubLaunch(i1);+ StackResource r = resource("Server");+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\", \"InstanceType\": \"t3.micro\"}"), ctx(null));++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(i1)));+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\", \"InstanceType\": \"t3.small\"}"), ctx("i-1"));++ assertEquals("i-1", r.getPhysicalId());+ verify(ec2).modifyInstanceAttribute("us-east-1", "i-1", "instanceType", "t3.small");+ }++ @Test+ void reusedInstanceReattachesChangedSecurityGroups() throws Exception {+ Instance i1 = new Instance();+ i1.setInstanceId("i-1");+ i1.setImageId("ami-1");+ i1.setSecurityGroups(new ArrayList<>(List.of(new GroupIdentifier("sg-1", "old"))));+ stubLaunch(i1);+ StackResource r = resource("Server");+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx(null));++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(i1)));+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\", \"SecurityGroupIds\": [\"sg-2\"]}"), ctx("i-1"));++ verify(ec2).modifyInstanceGroups("us-east-1", "i-1", List.of("sg-2"));+ }++ @Test+ void reusedInstanceLeavesUnchangedMutablePropertiesAlone() throws Exception {+ Instance i1 = new Instance();+ i1.setInstanceId("i-1");+ i1.setImageId("ami-1");+ i1.setInstanceType("t3.small");+ i1.setSecurityGroups(new ArrayList<>(List.of(new GroupIdentifier("sg-1", "keep"))));+ stubLaunch(i1);+ StackResource r = resource("Server");+ String template = "{\"ImageId\": \"ami-1\", \"InstanceType\": \"t3.small\", "+ + "\"SecurityGroupIds\": [\"sg-1\"]}";+ provisioner.provision(r, props(template), ctx(null));++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(i1)));+ provisioner.provision(r, props(template), ctx("i-1"));++ verify(ec2, never()).modifyInstanceAttribute(anyString(), anyString(), anyString(), anyString());+ verify(ec2, never()).modifyInstanceGroups(anyString(), anyString(), anyList());+ }+@Testvoid updateCreatesAFreshInstanceWhenThePriorOneWasTerminatedOutOfBand() throws Exception {Instance created = new Instance();