Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): reconcile or replace EC2 instances on stack update (#3884)
Hector Ventura21beccceb182
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.javaindex 8b72f0b30e..8045bddb74 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.java@@ -1,10 +1,12 @@package io.github.hectorvent.floci.services.cloudformation.provisioners;import com.fasterxml.jackson.databind.JsonNode;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.ec2.Ec2Service;import io.github.hectorvent.floci.services.ec2.model.Instance;+import io.github.hectorvent.floci.services.ec2.model.InstanceState;import io.github.hectorvent.floci.services.ec2.model.LaunchTemplateData;import io.github.hectorvent.floci.services.ec2.model.Reservation;import io.github.hectorvent.floci.services.ec2.model.Tag;@@ -13,7 +15,10 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.util.ArrayList;+import java.util.HashMap;+import java.util.LinkedHashMap;import java.util.List;+import java.util.Map;import java.util.Set;/**@@ -42,6 +47,7 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {@Overridepublic void provision(StackResource r, JsonNode props, ProvisionContext ctx) {+ Map<String, String> attributesBefore = new HashMap<>(r.getAttributes());CloudFormationTemplateEngine engine = ctx.engine();String region = ctx.region();String imageId = ctx.resolveOptional(props, "ImageId");@@ -75,6 +81,19 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {instanceType = "t3.micro";}String subnetId = ctx.resolveOptional(props, "SubnetId");++ // On update, keep the existing instance unless a createOnly property changed: only a+ // changed ImageId, SubnetId or KeyName replaces it, a mutable-only change (tags, and the+ // like) reuses it. Launching unconditionally would leak the prior instance every update.+ Instance prior = ctx.isUpdate() ? findInstance(region, ctx.priorPhysicalId()) : null;+ if (prior != null && !createOnlyChanged(prior, imageId, subnetId, keyName)) {+ r.setPhysicalId(prior.getInstanceId());+ publishInstanceAttributes(r, prior);+ reconcileTags(prior.getInstanceId(), region, props, engine);+ ReplacementCleanup.record(r, ctx, attributesBefore);+ return;+ }+String userData = ctx.resolveOptional(props, "UserData");String iamInstanceProfile = ctx.resolveOptional(props, "IamInstanceProfile");@@ -118,23 +137,147 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {r.getAttributes().put(CfnRollback.ROLLBACK_OWNED_ATTR, "true");ec2Service.awaitContainerLaunch(instance);r.getAttributes().remove(CfnRollback.ROLLBACK_OWNED_ATTR);+ publishInstanceAttributes(r, instance);+ // A createOnly change that landed a new instance id replaced the prior one: record it so+ // the stack cleans the displaced instance up after the update commits.+ ReplacementCleanup.record(r, ctx, attributesBefore);+ }++ private Instance findInstance(String region, String instanceId) {+ List<Reservation> reservations;+ try {+ reservations = ec2Service.describeInstances(region, List.of(instanceId), null);+ } catch (AwsException e) {+ if ("InvalidInstanceID.NotFound".equals(e.getErrorCode())) {+ // The prior instance was terminated out of band; treat it as gone so the update+ // creates a fresh one instead of failing outright, the same way delete() tolerates it.+ return null;+ }+ throw e;+ }+ for (Reservation reservation : reservations) {+ for (Instance instance : reservation.getInstances()) {+ if (instanceId.equals(instance.getInstanceId()) && !isTerminated(instance)) {+ return instance;+ }+ }+ }+ return null;+ }++ // A terminated instance lingers in the store, so reusing it would point Ref at a dead instance.+ private static boolean isTerminated(Instance instance) {+ InstanceState state = instance.getState();+ String name = state == null ? null : state.getName();+ return "terminated".equals(name) || "shutting-down".equals(name);+ }++ private static boolean createOnlyChanged(Instance prior, String imageId, String subnetId, String keyName) {+ return changed(imageId, prior.getImageId())+ || changed(subnetId, prior.getSubnetId())+ || changed(keyName, prior.getKeyName());+ }++ /** A property forces replacement only when the template declares a value that differs. */+ private static boolean changed(String declared, String actual) {+ return declared != null && !declared.equals(actual);+ }++ private void publishInstanceAttributes(StackResource r, Instance instance) {+ // Each attribute is written when present and removed when absent, so a replacement never+ // inherits the prior instance's value: on update the resource still carries the old+ // attributes, so a new instance that lacks a PublicIp must not keep the old one. The put+ // calls stay literal so CfnSchemaCoverageTest still sees these attributes as published.+ r.getAttributes().put("InstanceId", instance.getInstanceId());+ String availabilityZone = instance.getPlacement() != null+ ? instance.getPlacement().getAvailabilityZone() : null;if (instance.getPrivateIpAddress() != null) {r.getAttributes().put("PrivateIp", instance.getPrivateIpAddress());+ } else {+ r.getAttributes().remove("PrivateIp");}if (instance.getPublicIpAddress() != null) {r.getAttributes().put("PublicIp", instance.getPublicIpAddress());+ } else {+ r.getAttributes().remove("PublicIp");}if (instance.getPrivateDnsName() != null) {r.getAttributes().put("PrivateDnsName", instance.getPrivateDnsName());+ } else {+ r.getAttributes().remove("PrivateDnsName");}if (instance.getPublicDnsName() != null) {r.getAttributes().put("PublicDnsName", instance.getPublicDnsName());+ } else {+ r.getAttributes().remove("PublicDnsName");+ }+ if (availabilityZone != null) {+ r.getAttributes().put("AvailabilityZone", availabilityZone);+ } else {+ r.getAttributes().remove("AvailabilityZone");+ }+ }++ /** Reconciles the instance tags to the template when the instance is kept in place. */+ private void reconcileTags(String instanceId, String region, JsonNode props,+ CloudFormationTemplateEngine engine) {+ Map<String, String> desired = new LinkedHashMap<>();+ JsonNode tagsNode = props != null ? engine.resolveNode(props.get("Tags")) : null;+ if (tagsNode != null && tagsNode.isArray()) {+ for (JsonNode tag : tagsNode) {+ String key = engine.resolve(tag.path("Key"));+ if (key != null && !key.isEmpty()) {+ String value = engine.resolve(tag.path("Value"));+ desired.put(key, value == null ? "" : value);+ }+ }+ }+ Map<String, String> current = new LinkedHashMap<>();+ for (Map<String, String> entry : ec2Service.describeTags(region, Map.of("resource-id", List.of(instanceId)))) {+ current.put(entry.get("key"), entry.get("value"));}- if (instance.getPlacement() != null && instance.getPlacement().getAvailabilityZone() != null) {- r.getAttributes().put("AvailabilityZone", instance.getPlacement().getAvailabilityZone());+ List<String> stale = ProvisionContext.staleTagKeys(current, desired);+ if (!stale.isEmpty()) {+ List<Tag> remove = new ArrayList<>();+ for (String key : stale) {+ remove.add(new Tag(key, current.get(key)));+ }+ ec2Service.deleteTags(region, List.of(instanceId), remove);+ }+ if (!desired.isEmpty()) {+ List<Tag> add = new ArrayList<>();+ for (Map.Entry<String, String> entry : desired.entrySet()) {+ add.add(new Tag(entry.getKey(), entry.getValue()));+ }+ ec2Service.createTags(region, List.of(instanceId), add);}}+ @Override+ public boolean hasReplacementUpdate(StackResource resource) {+ return ReplacementCleanup.hasReplacement(resource);+ }++ @Override+ public String updateCleanupPhysicalId(StackResource resource) {+ return ReplacementCleanup.cleanupPhysicalId(resource);+ }++ @Override+ public UpdateCleanupResult completeUpdate(StackResource resource) {+ return ReplacementCleanup.complete(resource, this::delete);+ }++ @Override+ public void clearUpdate(StackResource resource) {+ ReplacementCleanup.clear(resource);+ }++ @Override+ public boolean rollbackUpdate(StackResource resource) {+ return ReplacementCleanup.rollback(resource, this::delete);+ }+@Overridepublic void delete(String resourceType, String physicalId, String region) {// Tolerate an instance already terminated out of band, so DeleteStack does not fail on it.diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.javaindex e64f92dcf8..66a04cca71 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.java@@ -2,32 +2,39 @@ package io.github.hectorvent.floci.services.cloudformation.provisioners;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.ec2.Ec2Service;import io.github.hectorvent.floci.services.ec2.model.Instance;+import io.github.hectorvent.floci.services.ec2.model.InstanceState;import io.github.hectorvent.floci.services.ec2.model.LaunchTemplateData;import io.github.hectorvent.floci.services.ec2.model.Placement;import io.github.hectorvent.floci.services.ec2.model.Reservation;+import io.github.hectorvent.floci.services.ec2.model.Tag;import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;import java.util.HashMap;import java.util.List;+import java.util.Map;import java.util.Set;+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyInt;import static org.mockito.ArgumentMatchers.anyList;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;-import io.github.hectorvent.floci.core.common.AwsException;-import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;-import static org.junit.jupiter.api.Assertions.assertThrows;-import static org.mockito.Mockito.doThrow;/*** {@code AWS::EC2::Instance} in isolation: the attributes Ref and Fn::GetAtt read once the launch@@ -124,6 +131,152 @@ class Ec2InstanceCfnProvisionerTest {assertEquals("DependencyViolation", failure.getErrorCode());}+ @Test+ void updateWithoutACreateOnlyChangeReusesTheInstance() throws Exception {+ Instance i1 = new Instance();+ i1.setInstanceId("i-1");+ i1.setImageId("ami-1");+ stubLaunch(i1);+ StackResource r = resource("Server");+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx(null));+ assertEquals("i-1", r.getPhysicalId());++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(i1)));+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\", \"InstanceType\": \"t3.small\"}"),+ ctx("i-1"));++ assertEquals("i-1", r.getPhysicalId());+ verify(ec2, times(1)).runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(),+ anyList(), any(), any(), anyList(), any(), any(), any());+ }++ @Test+ void changingImageIdReplacesTheInstanceAndRecordsTheCleanup() throws Exception {+ Instance i1 = new Instance();+ i1.setInstanceId("i-1");+ i1.setImageId("ami-1");+ Instance i2 = new Instance();+ i2.setInstanceId("i-2");+ i2.setImageId("ami-2");+ when(ec2.runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any()))+ .thenReturn(reservationOf(i1), reservationOf(i2));+ StackResource r = resource("Server");+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx(null));+ assertEquals("i-1", r.getPhysicalId());++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(i1)));+ provisioner.provision(r, props("{\"ImageId\": \"ami-2\"}"), ctx("i-1"));++ assertEquals("i-2", r.getPhysicalId());+ assertTrue(provisioner.hasReplacementUpdate(r));+ assertEquals("i-1", provisioner.updateCleanupPhysicalId(r));+ }++ @Test+ void reusedInstanceReconcilesTagsInPlace() throws Exception {+ Instance i1 = new Instance();+ i1.setInstanceId("i-1");+ i1.setImageId("ami-1");+ stubLaunch(i1);+ StackResource r = resource("Server");+ provisioner.provision(r, props("""+ {"ImageId": "ami-1", "Tags": [{"Key": "env", "Value": "prod"}]}+ """), ctx(null));++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(i1)));+ when(ec2.describeTags(eq("us-east-1"), any())).thenReturn(List.of(+ tagEntry("i-1", "env", "prod"), tagEntry("i-1", "old", "stale")));+ provisioner.provision(r, props("""+ {"ImageId": "ami-1", "Tags": [{"Key": "env", "Value": "prod"}]}+ """), ctx("i-1"));++ @SuppressWarnings("unchecked")+ ArgumentCaptor<List<Tag>> removed = ArgumentCaptor.forClass(List.class);+ verify(ec2).deleteTags(eq("us-east-1"), eq(List.of("i-1")), removed.capture());+ assertEquals(1, removed.getValue().size());+ assertEquals("old", removed.getValue().get(0).getKey());+ }++ @Test+ void updateCreatesAFreshInstanceWhenThePriorOneWasTerminatedOutOfBand() throws Exception {+ Instance created = new Instance();+ created.setInstanceId("i-9");+ created.setImageId("ami-1");+ stubLaunch(created);+ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenThrow(new AwsException("InvalidInstanceID.NotFound", "gone", 400));+ StackResource r = resource("Server");++ assertDoesNotThrow(() -> provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx("i-1")));++ assertEquals("i-9", r.getPhysicalId());+ verify(ec2).runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(),+ anyList(), any(), any(), anyList(), any(), any(), any());+ }++ @Test+ void updateDoesNotReuseATerminatedPriorInstance() throws Exception {+ Instance terminated = new Instance();+ terminated.setInstanceId("i-1");+ terminated.setImageId("ami-1");+ terminated.setState(InstanceState.terminated());+ Instance created = new Instance();+ created.setInstanceId("i-9");+ created.setImageId("ami-1");+ stubLaunch(created);+ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(terminated)));+ StackResource r = resource("Server");++ provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx("i-1"));++ assertEquals("i-9", r.getPhysicalId());+ verify(ec2).runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(),+ anyList(), any(), any(), anyList(), any(), any(), any());+ }++ @Test+ void replacementDoesNotInheritStaleAttributesFromThePriorInstance() throws Exception {+ Instance withPublicIp = new Instance();+ withPublicIp.setInstanceId("i-1");+ withPublicIp.setImageId("ami-1");+ withPublicIp.setPublicIpAddress("54.0.0.5");+ Instance withoutPublicIp = new Instance();+ withoutPublicIp.setInstanceId("i-2");+ withoutPublicIp.setImageId("ami-2");+ when(ec2.runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any()))+ .thenReturn(reservationOf(withPublicIp), reservationOf(withoutPublicIp));+ StackResource r = resource("Server");+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx(null));+ assertEquals("54.0.0.5", r.getAttributes().get("PublicIp"));++ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))+ .thenReturn(List.of(reservationOf(withPublicIp)));+ provisioner.provision(r, props("{\"ImageId\": \"ami-2\"}"), ctx("i-1"));++ assertEquals("i-2", r.getPhysicalId());+ assertFalse(r.getAttributes().containsKey("PublicIp"));+ }++ private static Reservation reservationOf(Instance instance) {+ Reservation reservation = new Reservation();+ reservation.getInstances().add(instance);+ return reservation;+ }++ private static Map<String, String> tagEntry(String resourceId, String key, String value) {+ Map<String, String> entry = new HashMap<>();+ entry.put("resourceId", resourceId);+ entry.put("key", key);+ entry.put("value", value);+ return entry;+ }+private void stubLaunch(Instance instance) {Reservation reservation = new Reservation();reservation.getInstances().add(instance);@@ -132,13 +285,17 @@ class Ec2InstanceCfnProvisionerTest {}private ProvisionContext ctx() {+ return ctx(null);+ }++ private ProvisionContext ctx(String priorPhysicalId) {CloudFormationTemplateEngine engine = mock(CloudFormationTemplateEngine.class);when(engine.resolve(any())).thenAnswer(inv -> {JsonNode node = inv.getArgument(0);return node == null ? null : node.asText();});when(engine.resolveNode(any())).thenAnswer(inv -> inv.getArgument(0));- return new ProvisionContext(engine, "us-east-1", "000000000000", "my-stack");+ return new ProvisionContext(engine, "us-east-1", "000000000000", "my-stack", priorPhysicalId);}private JsonNode props(String json) throws Exception {