Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudformation): reconcile or replace EC2 instances on stack update (#3884)

Hector Ventura21beccceb182
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.java
index 8b72f0b30e..8045bddb74 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisioner.java
@@ -1,10 +1,12 @@
package io.github.hectorvent.floci.services.cloudformation.provisioners;
import com.fasterxml.jackson.databind.JsonNode;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;
import io.github.hectorvent.floci.services.cloudformation.model.StackResource;
import io.github.hectorvent.floci.services.ec2.Ec2Service;
import io.github.hectorvent.floci.services.ec2.model.Instance;
+import io.github.hectorvent.floci.services.ec2.model.InstanceState;
import io.github.hectorvent.floci.services.ec2.model.LaunchTemplateData;
import io.github.hectorvent.floci.services.ec2.model.Reservation;
import io.github.hectorvent.floci.services.ec2.model.Tag;
@@ -13,7 +15,10 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -42,6 +47,7 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {
@Override
public void provision(StackResource r, JsonNode props, ProvisionContext ctx) {
+ Map<String, String> attributesBefore = new HashMap<>(r.getAttributes());
CloudFormationTemplateEngine engine = ctx.engine();
String region = ctx.region();
String imageId = ctx.resolveOptional(props, "ImageId");
@@ -75,6 +81,19 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {
instanceType = "t3.micro";
}
String subnetId = ctx.resolveOptional(props, "SubnetId");
+
+ // On update, keep the existing instance unless a createOnly property changed: only a
+ // changed ImageId, SubnetId or KeyName replaces it, a mutable-only change (tags, and the
+ // like) reuses it. Launching unconditionally would leak the prior instance every update.
+ Instance prior = ctx.isUpdate() ? findInstance(region, ctx.priorPhysicalId()) : null;
+ if (prior != null && !createOnlyChanged(prior, imageId, subnetId, keyName)) {
+ r.setPhysicalId(prior.getInstanceId());
+ publishInstanceAttributes(r, prior);
+ reconcileTags(prior.getInstanceId(), region, props, engine);
+ ReplacementCleanup.record(r, ctx, attributesBefore);
+ return;
+ }
+
String userData = ctx.resolveOptional(props, "UserData");
String iamInstanceProfile = ctx.resolveOptional(props, "IamInstanceProfile");
@@ -118,23 +137,147 @@ public class Ec2InstanceCfnProvisioner implements CfnResourceProvisioner {
r.getAttributes().put(CfnRollback.ROLLBACK_OWNED_ATTR, "true");
ec2Service.awaitContainerLaunch(instance);
r.getAttributes().remove(CfnRollback.ROLLBACK_OWNED_ATTR);
+ publishInstanceAttributes(r, instance);
+ // A createOnly change that landed a new instance id replaced the prior one: record it so
+ // the stack cleans the displaced instance up after the update commits.
+ ReplacementCleanup.record(r, ctx, attributesBefore);
+ }
+
+ private Instance findInstance(String region, String instanceId) {
+ List<Reservation> reservations;
+ try {
+ reservations = ec2Service.describeInstances(region, List.of(instanceId), null);
+ } catch (AwsException e) {
+ if ("InvalidInstanceID.NotFound".equals(e.getErrorCode())) {
+ // The prior instance was terminated out of band; treat it as gone so the update
+ // creates a fresh one instead of failing outright, the same way delete() tolerates it.
+ return null;
+ }
+ throw e;
+ }
+ for (Reservation reservation : reservations) {
+ for (Instance instance : reservation.getInstances()) {
+ if (instanceId.equals(instance.getInstanceId()) && !isTerminated(instance)) {
+ return instance;
+ }
+ }
+ }
+ return null;
+ }
+
+ // A terminated instance lingers in the store, so reusing it would point Ref at a dead instance.
+ private static boolean isTerminated(Instance instance) {
+ InstanceState state = instance.getState();
+ String name = state == null ? null : state.getName();
+ return "terminated".equals(name) || "shutting-down".equals(name);
+ }
+
+ private static boolean createOnlyChanged(Instance prior, String imageId, String subnetId, String keyName) {
+ return changed(imageId, prior.getImageId())
+ || changed(subnetId, prior.getSubnetId())
+ || changed(keyName, prior.getKeyName());
+ }
+
+ /** A property forces replacement only when the template declares a value that differs. */
+ private static boolean changed(String declared, String actual) {
+ return declared != null && !declared.equals(actual);
+ }
+
+ private void publishInstanceAttributes(StackResource r, Instance instance) {
+ // Each attribute is written when present and removed when absent, so a replacement never
+ // inherits the prior instance's value: on update the resource still carries the old
+ // attributes, so a new instance that lacks a PublicIp must not keep the old one. The put
+ // calls stay literal so CfnSchemaCoverageTest still sees these attributes as published.
+ r.getAttributes().put("InstanceId", instance.getInstanceId());
+ String availabilityZone = instance.getPlacement() != null
+ ? instance.getPlacement().getAvailabilityZone() : null;
if (instance.getPrivateIpAddress() != null) {
r.getAttributes().put("PrivateIp", instance.getPrivateIpAddress());
+ } else {
+ r.getAttributes().remove("PrivateIp");
}
if (instance.getPublicIpAddress() != null) {
r.getAttributes().put("PublicIp", instance.getPublicIpAddress());
+ } else {
+ r.getAttributes().remove("PublicIp");
}
if (instance.getPrivateDnsName() != null) {
r.getAttributes().put("PrivateDnsName", instance.getPrivateDnsName());
+ } else {
+ r.getAttributes().remove("PrivateDnsName");
}
if (instance.getPublicDnsName() != null) {
r.getAttributes().put("PublicDnsName", instance.getPublicDnsName());
+ } else {
+ r.getAttributes().remove("PublicDnsName");
+ }
+ if (availabilityZone != null) {
+ r.getAttributes().put("AvailabilityZone", availabilityZone);
+ } else {
+ r.getAttributes().remove("AvailabilityZone");
+ }
+ }
+
+ /** Reconciles the instance tags to the template when the instance is kept in place. */
+ private void reconcileTags(String instanceId, String region, JsonNode props,
+ CloudFormationTemplateEngine engine) {
+ Map<String, String> desired = new LinkedHashMap<>();
+ JsonNode tagsNode = props != null ? engine.resolveNode(props.get("Tags")) : null;
+ if (tagsNode != null && tagsNode.isArray()) {
+ for (JsonNode tag : tagsNode) {
+ String key = engine.resolve(tag.path("Key"));
+ if (key != null && !key.isEmpty()) {
+ String value = engine.resolve(tag.path("Value"));
+ desired.put(key, value == null ? "" : value);
+ }
+ }
+ }
+ Map<String, String> current = new LinkedHashMap<>();
+ for (Map<String, String> entry : ec2Service.describeTags(region, Map.of("resource-id", List.of(instanceId)))) {
+ current.put(entry.get("key"), entry.get("value"));
}
- if (instance.getPlacement() != null && instance.getPlacement().getAvailabilityZone() != null) {
- r.getAttributes().put("AvailabilityZone", instance.getPlacement().getAvailabilityZone());
+ List<String> stale = ProvisionContext.staleTagKeys(current, desired);
+ if (!stale.isEmpty()) {
+ List<Tag> remove = new ArrayList<>();
+ for (String key : stale) {
+ remove.add(new Tag(key, current.get(key)));
+ }
+ ec2Service.deleteTags(region, List.of(instanceId), remove);
+ }
+ if (!desired.isEmpty()) {
+ List<Tag> add = new ArrayList<>();
+ for (Map.Entry<String, String> entry : desired.entrySet()) {
+ add.add(new Tag(entry.getKey(), entry.getValue()));
+ }
+ ec2Service.createTags(region, List.of(instanceId), add);
}
}
+ @Override
+ public boolean hasReplacementUpdate(StackResource resource) {
+ return ReplacementCleanup.hasReplacement(resource);
+ }
+
+ @Override
+ public String updateCleanupPhysicalId(StackResource resource) {
+ return ReplacementCleanup.cleanupPhysicalId(resource);
+ }
+
+ @Override
+ public UpdateCleanupResult completeUpdate(StackResource resource) {
+ return ReplacementCleanup.complete(resource, this::delete);
+ }
+
+ @Override
+ public void clearUpdate(StackResource resource) {
+ ReplacementCleanup.clear(resource);
+ }
+
+ @Override
+ public boolean rollbackUpdate(StackResource resource) {
+ return ReplacementCleanup.rollback(resource, this::delete);
+ }
+
@Override
public void delete(String resourceType, String physicalId, String region) {
// Tolerate an instance already terminated out of band, so DeleteStack does not fail on it.
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.java
index e64f92dcf8..66a04cca71 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/Ec2InstanceCfnProvisionerTest.java
@@ -2,32 +2,39 @@ package io.github.hectorvent.floci.services.cloudformation.provisioners;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;
import io.github.hectorvent.floci.services.cloudformation.model.StackResource;
import io.github.hectorvent.floci.services.ec2.Ec2Service;
import io.github.hectorvent.floci.services.ec2.model.Instance;
+import io.github.hectorvent.floci.services.ec2.model.InstanceState;
import io.github.hectorvent.floci.services.ec2.model.LaunchTemplateData;
import io.github.hectorvent.floci.services.ec2.model.Placement;
import io.github.hectorvent.floci.services.ec2.model.Reservation;
+import io.github.hectorvent.floci.services.ec2.model.Tag;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import io.github.hectorvent.floci.core.common.AwsException;
-import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.doThrow;
/**
* {@code AWS::EC2::Instance} in isolation: the attributes Ref and Fn::GetAtt read once the launch
@@ -124,6 +131,152 @@ class Ec2InstanceCfnProvisionerTest {
assertEquals("DependencyViolation", failure.getErrorCode());
}
+ @Test
+ void updateWithoutACreateOnlyChangeReusesTheInstance() throws Exception {
+ Instance i1 = new Instance();
+ i1.setInstanceId("i-1");
+ i1.setImageId("ami-1");
+ stubLaunch(i1);
+ StackResource r = resource("Server");
+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx(null));
+ assertEquals("i-1", r.getPhysicalId());
+
+ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))
+ .thenReturn(List.of(reservationOf(i1)));
+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\", \"InstanceType\": \"t3.small\"}"),
+ ctx("i-1"));
+
+ assertEquals("i-1", r.getPhysicalId());
+ verify(ec2, times(1)).runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(),
+ anyList(), any(), any(), anyList(), any(), any(), any());
+ }
+
+ @Test
+ void changingImageIdReplacesTheInstanceAndRecordsTheCleanup() throws Exception {
+ Instance i1 = new Instance();
+ i1.setInstanceId("i-1");
+ i1.setImageId("ami-1");
+ Instance i2 = new Instance();
+ i2.setInstanceId("i-2");
+ i2.setImageId("ami-2");
+ when(ec2.runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(), anyList(),
+ any(), any(), anyList(), any(), any(), any()))
+ .thenReturn(reservationOf(i1), reservationOf(i2));
+ StackResource r = resource("Server");
+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx(null));
+ assertEquals("i-1", r.getPhysicalId());
+
+ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))
+ .thenReturn(List.of(reservationOf(i1)));
+ provisioner.provision(r, props("{\"ImageId\": \"ami-2\"}"), ctx("i-1"));
+
+ assertEquals("i-2", r.getPhysicalId());
+ assertTrue(provisioner.hasReplacementUpdate(r));
+ assertEquals("i-1", provisioner.updateCleanupPhysicalId(r));
+ }
+
+ @Test
+ void reusedInstanceReconcilesTagsInPlace() throws Exception {
+ Instance i1 = new Instance();
+ i1.setInstanceId("i-1");
+ i1.setImageId("ami-1");
+ stubLaunch(i1);
+ StackResource r = resource("Server");
+ provisioner.provision(r, props("""
+ {"ImageId": "ami-1", "Tags": [{"Key": "env", "Value": "prod"}]}
+ """), ctx(null));
+
+ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))
+ .thenReturn(List.of(reservationOf(i1)));
+ when(ec2.describeTags(eq("us-east-1"), any())).thenReturn(List.of(
+ tagEntry("i-1", "env", "prod"), tagEntry("i-1", "old", "stale")));
+ provisioner.provision(r, props("""
+ {"ImageId": "ami-1", "Tags": [{"Key": "env", "Value": "prod"}]}
+ """), ctx("i-1"));
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<List<Tag>> removed = ArgumentCaptor.forClass(List.class);
+ verify(ec2).deleteTags(eq("us-east-1"), eq(List.of("i-1")), removed.capture());
+ assertEquals(1, removed.getValue().size());
+ assertEquals("old", removed.getValue().get(0).getKey());
+ }
+
+ @Test
+ void updateCreatesAFreshInstanceWhenThePriorOneWasTerminatedOutOfBand() throws Exception {
+ Instance created = new Instance();
+ created.setInstanceId("i-9");
+ created.setImageId("ami-1");
+ stubLaunch(created);
+ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))
+ .thenThrow(new AwsException("InvalidInstanceID.NotFound", "gone", 400));
+ StackResource r = resource("Server");
+
+ assertDoesNotThrow(() -> provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx("i-1")));
+
+ assertEquals("i-9", r.getPhysicalId());
+ verify(ec2).runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(),
+ anyList(), any(), any(), anyList(), any(), any(), any());
+ }
+
+ @Test
+ void updateDoesNotReuseATerminatedPriorInstance() throws Exception {
+ Instance terminated = new Instance();
+ terminated.setInstanceId("i-1");
+ terminated.setImageId("ami-1");
+ terminated.setState(InstanceState.terminated());
+ Instance created = new Instance();
+ created.setInstanceId("i-9");
+ created.setImageId("ami-1");
+ stubLaunch(created);
+ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))
+ .thenReturn(List.of(reservationOf(terminated)));
+ StackResource r = resource("Server");
+
+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx("i-1"));
+
+ assertEquals("i-9", r.getPhysicalId());
+ verify(ec2).runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(),
+ anyList(), any(), any(), anyList(), any(), any(), any());
+ }
+
+ @Test
+ void replacementDoesNotInheritStaleAttributesFromThePriorInstance() throws Exception {
+ Instance withPublicIp = new Instance();
+ withPublicIp.setInstanceId("i-1");
+ withPublicIp.setImageId("ami-1");
+ withPublicIp.setPublicIpAddress("54.0.0.5");
+ Instance withoutPublicIp = new Instance();
+ withoutPublicIp.setInstanceId("i-2");
+ withoutPublicIp.setImageId("ami-2");
+ when(ec2.runInstances(anyString(), any(), anyString(), anyInt(), anyInt(), any(), anyList(),
+ any(), any(), anyList(), any(), any(), any()))
+ .thenReturn(reservationOf(withPublicIp), reservationOf(withoutPublicIp));
+ StackResource r = resource("Server");
+ provisioner.provision(r, props("{\"ImageId\": \"ami-1\"}"), ctx(null));
+ assertEquals("54.0.0.5", r.getAttributes().get("PublicIp"));
+
+ when(ec2.describeInstances("us-east-1", List.of("i-1"), null))
+ .thenReturn(List.of(reservationOf(withPublicIp)));
+ provisioner.provision(r, props("{\"ImageId\": \"ami-2\"}"), ctx("i-1"));
+
+ assertEquals("i-2", r.getPhysicalId());
+ assertFalse(r.getAttributes().containsKey("PublicIp"));
+ }
+
+ private static Reservation reservationOf(Instance instance) {
+ Reservation reservation = new Reservation();
+ reservation.getInstances().add(instance);
+ return reservation;
+ }
+
+ private static Map<String, String> tagEntry(String resourceId, String key, String value) {
+ Map<String, String> entry = new HashMap<>();
+ entry.put("resourceId", resourceId);
+ entry.put("key", key);
+ entry.put("value", value);
+ return entry;
+ }
+
private void stubLaunch(Instance instance) {
Reservation reservation = new Reservation();
reservation.getInstances().add(instance);
@@ -132,13 +285,17 @@ class Ec2InstanceCfnProvisionerTest {
}
private ProvisionContext ctx() {
+ return ctx(null);
+ }
+
+ private ProvisionContext ctx(String priorPhysicalId) {
CloudFormationTemplateEngine engine = mock(CloudFormationTemplateEngine.class);
when(engine.resolve(any())).thenAnswer(inv -> {
JsonNode node = inv.getArgument(0);
return node == null ? null : node.asText();
});
when(engine.resolveNode(any())).thenAnswer(inv -> inv.getArgument(0));
- return new ProvisionContext(engine, "us-east-1", "000000000000", "my-stack");
+ return new ProvisionContext(engine, "us-east-1", "000000000000", "my-stack", priorPhysicalId);
}
private JsonNode props(String json) throws Exception {