Light, fluffy, and always free - The AWS Local Emulator alternative
fix(autoscaling): prevent reconciler resurrecting deleted groups (#3818)
Vũ Thế Huybd9700db0d1a
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.javaindex bce1108164..b5792dc0ce 100644--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java@@ -22,6 +22,7 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import io.quarkus.runtime.StartupEvent;+import java.util.ArrayList;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;@@ -102,8 +103,9 @@ public class AutoScalingReconciler {} else if (activeCapacity > desired) {scaleIn(asg, (int) (activeCapacity - desired));}- asgService.saveAutoScalingGroup(asg);- asgService.completeInstanceRefreshIfSettled(asg.getRegion(), asg.getAutoScalingGroupName());+ if (asgService.saveAutoScalingGroupIfPresent(asg)) {+ asgService.completeInstanceRefreshIfSettled(asg.getRegion(), asg.getAutoScalingGroupName());+ }}static long activeCapacity(AutoScalingGroup asg) {@@ -148,7 +150,7 @@ public class AutoScalingReconciler {}}if (changed) {- asgService.saveAutoScalingGroup(asg);+ asgService.saveAutoScalingGroupIfPresent(asg);}}@@ -167,7 +169,7 @@ public class AutoScalingReconciler {deregisterFromTargetGroups(asg, instanceIds);deregisterFromClassicLoadBalancers(asg, instanceIds);asg.getInstances().removeIf(instance -> instanceIds.contains(instance.getInstanceId()));- asgService.saveAutoScalingGroup(asg);+ asgService.saveAutoScalingGroupIfPresent(asg);asgService.recordActivity(asg.getRegion(), asg.getAutoScalingGroupName(),"Removing stale EC2 instance reference(s): " + instanceIds,"Persisted Auto Scaling state referenced instance containers that are no longer running.",@@ -279,7 +281,7 @@ public class AutoScalingReconciler {}asg.getInstances().removeIf(instance -> instanceIds.contains(instance.getInstanceId()));- asgService.saveAutoScalingGroup(asg);+ asgService.saveAutoScalingGroupIfPresent(asg);asgService.recordActivity(asg.getRegion(), asg.getAutoScalingGroupName(),"Terminating EC2 instance(s) for refresh: " + instanceIds,"An instance refresh requested replacement of active instances.",@@ -318,6 +320,7 @@ public class AutoScalingReconciler {launchSource.iamInstanceProfile(),launchSource.associatePublicIpAddress());+ List<String> launchedInstanceIds = new ArrayList<>();for (Instance ec2Inst : reservation.getInstances()) {AsgInstance asgInst = new AsgInstance();asgInst.setInstanceId(ec2Inst.getInstanceId());@@ -330,10 +333,18 @@ public class AutoScalingReconciler {asgInst.setLaunchTemplateVersion(launchSource.launchTemplateVersion());asgInst.setInstanceType(launchSource.instanceType());asg.getInstances().add(asgInst);+ launchedInstanceIds.add(ec2Inst.getInstanceId());LOG.infov("ASG {0}: launched instance {1} (Pending)",asg.getAutoScalingGroupName(), ec2Inst.getInstanceId());}- asgService.saveAutoScalingGroup(asg);+ if (!asgService.saveAutoScalingGroupIfPresent(asg) && !launchedInstanceIds.isEmpty()) {+ try {+ ec2Service.terminateInstances(asg.getRegion(), launchedInstanceIds);+ } catch (Exception e) {+ LOG.warnv("ASG {0}: failed to clean up instances {1} after group deletion: {2}",+ asg.getAutoScalingGroupName(), launchedInstanceIds, e.getMessage());+ }+ }} catch (Exception e) {LOG.warnv("ASG {0}: failed to launch instances: {1}",asg.getAutoScalingGroupName(), e.getMessage());@@ -384,7 +395,7 @@ public class AutoScalingReconciler {}asg.getInstances().removeIf(i -> instanceIds.contains(i.getInstanceId()));- asgService.saveAutoScalingGroup(asg);+ asgService.saveAutoScalingGroupIfPresent(asg);asgService.recordActivity(asg.getRegion(), asg.getAutoScalingGroupName(),"Terminating EC2 instance(s): " + instanceIds,"An instance was terminated in response to a desired capacity change.",diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.javaindex 4d99d32948..1c7d978db1 100644--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java@@ -63,6 +63,7 @@ public class AutoScalingService {// region :: name → resourceprivate Map<String, LaunchConfiguration> launchConfigs = new ConcurrentHashMap<>();private Map<String, AutoScalingGroup> groups = new ConcurrentHashMap<>();+ private final Map<String, Object> groupLocks = new ConcurrentHashMap<>();private Map<String, LifecycleHook> hooks = new ConcurrentHashMap<>();private Map<String, ScalingPolicy> policies = new ConcurrentHashMap<>();private Map<String, ScalingActivity> activities = new ConcurrentHashMap<>();@@ -328,29 +329,32 @@ public class AutoScalingService {}public void deleteAutoScalingGroup(String region, String name, boolean forceDelete) {- AutoScalingGroup asg = requireGroup(region, name);- List<AsgInstance> active = asg.getInstances().stream()- .filter(i -> !"Terminated".equals(i.getLifecycleState()))- .collect(Collectors.toList());- if (!active.isEmpty() && !forceDelete) {- throw new AwsException("ResourceInUse",- "Auto Scaling group '" + name + "' has " + active.size()- + " instance(s). Set ForceDelete=true to delete anyway.", 400);- }- if (forceDelete && ec2Service != null && !active.isEmpty()) {- active.stream()- .map(AsgInstance::getInstanceId)- .filter(Objects::nonNull)- .forEach(instanceId -> {- try {- ec2Service.terminateInstances(region, List.of(instanceId));- }- catch (AwsException ignored) {- // ForceDelete should remove stale ASG membership even if EC2 no longer has the instance.- }- });- }- groups.remove(asgKey(region, name));+ String key = asgKey(region, name);+ synchronized (lockFor(key)) {+ AutoScalingGroup asg = requireGroup(region, name);+ List<AsgInstance> active = asg.getInstances().stream()+ .filter(i -> !"Terminated".equals(i.getLifecycleState()))+ .collect(Collectors.toList());+ if (!active.isEmpty() && !forceDelete) {+ throw new AwsException("ResourceInUse",+ "Auto Scaling group '" + name + "' has " + active.size()+ + " instance(s). Set ForceDelete=true to delete anyway.", 400);+ }+ if (forceDelete && ec2Service != null && !active.isEmpty()) {+ active.stream()+ .map(AsgInstance::getInstanceId)+ .filter(Objects::nonNull)+ .forEach(instanceId -> {+ try {+ ec2Service.terminateInstances(region, List.of(instanceId));+ }+ catch (AwsException ignored) {+ // ForceDelete should remove stale ASG membership even if EC2 no longer has the instance.+ }+ });+ }+ groups.remove(key);+ }// clean up associated hooks and policieshooks.entrySet().removeIf(e -> e.getValue().getAutoScalingGroupName().equals(name));policies.entrySet().removeIf(e -> e.getValue().getAutoScalingGroupName().equals(name));@@ -372,8 +376,20 @@ public class AutoScalingService {.collect(Collectors.toList());}- public void saveAutoScalingGroup(AutoScalingGroup asg) {- groups.put(asgKey(asg.getRegion(), asg.getAutoScalingGroupName()), asg);+ public boolean saveAutoScalingGroupIfPresent(AutoScalingGroup asg) {+ String key = asgKey(asg.getRegion(), asg.getAutoScalingGroupName());+ synchronized (lockFor(key)) {+ AutoScalingGroup current = groups.get(key);+ if (current == asg) {+ groups.put(key, asg);+ return true;+ }+ return false;+ }+ }++ private Object lockFor(String key) {+ return groupLocks.computeIfAbsent(key, ignored -> new Object());}public void setDesiredCapacity(String region, String name, int desiredCapacity) {diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.javaindex edec6f7ae9..cbce25505d 100644--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java@@ -78,6 +78,7 @@ class AutoScalingReconcilerTest {asg.setRegion("us-east-1");asg.setAutoScalingGroupName("app-asg");asg.setDesiredCapacity(0);+ when(asgService.saveAutoScalingGroupIfPresent(asg)).thenReturn(true);reconciler.reconcile(asg);@@ -144,6 +145,54 @@ class AutoScalingReconcilerTest {assertEquals("2001", tags.getValue().get(1).getValue());}+ @Test+ void scaleOutTerminatesInstancesWhenGroupDisappears() {+ AutoScalingService asgService = mock(AutoScalingService.class);+ Ec2Service ec2Service = mock(Ec2Service.class);+ ElbV2Service elbV2Service = mock(ElbV2Service.class);+ AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);+ AutoScalingGroup asg = launchTemplateAsg();+ stubLaunchTemplate(ec2Service, "ami-version-1", "t3.micro");+ when(asgService.saveAutoScalingGroupIfPresent(asg)).thenReturn(false);++ Instance ec2Instance = new Instance();+ ec2Instance.setInstanceId("i-stale");+ Reservation reservation = new Reservation();+ reservation.setInstances(List.of(ec2Instance));+ when(ec2Service.runInstances(eq("us-east-1"), eq("ami-version-1"), eq("t3.micro"),+ eq(1), eq(1), eq(null), eq(List.of()), eq(null), eq(null), eq(List.of()),+ eq(null), eq(null), eq(null))).thenReturn(reservation);++ reconciler.reconcile(asg);++ verify(ec2Service).terminateInstances("us-east-1", List.of("i-stale"));+ verify(asgService, never()).completeInstanceRefreshIfSettled("us-east-1", "app-asg");+ }++ @Test+ void scaleOutPersistsInstancesWhenGroupStillExists() {+ AutoScalingService asgService = mock(AutoScalingService.class);+ Ec2Service ec2Service = mock(Ec2Service.class);+ ElbV2Service elbV2Service = mock(ElbV2Service.class);+ AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);+ AutoScalingGroup asg = launchTemplateAsg();+ stubLaunchTemplate(ec2Service, "ami-version-1", "t3.micro");+ when(asgService.saveAutoScalingGroupIfPresent(asg)).thenReturn(true);++ Instance ec2Instance = new Instance();+ ec2Instance.setInstanceId("i-owned");+ Reservation reservation = new Reservation();+ reservation.setInstances(List.of(ec2Instance));+ when(ec2Service.runInstances(eq("us-east-1"), eq("ami-version-1"), eq("t3.micro"),+ eq(1), eq(1), eq(null), eq(List.of()), eq(null), eq(null), eq(List.of()),+ eq(null), eq(null), eq(null))).thenReturn(reservation);++ reconciler.reconcile(asg);++ verify(asgService, times(2)).saveAutoScalingGroupIfPresent(asg);+ verify(ec2Service, never()).terminateInstances("us-east-1", List.of("i-owned"));+ }+@Testvoid scaleOutForwardsLaunchConfigurationAssociatePublicIpAddressBothDirections() {// An LC that explicitly sets false must suppress the public IP even in a@@ -306,7 +355,7 @@ class AutoScalingReconcilerTest {targets.capture());assertEquals(1, targets.getValue().size());assertEquals("i-stale", targets.getValue().getFirst().getId());- verify(asgService).saveAutoScalingGroup(asg);+ verify(asgService).saveAutoScalingGroupIfPresent(asg);verify(ec2Service, never()).terminateInstances(asg.getRegion(), List.of("i-active"));}@@ -466,7 +515,7 @@ class AutoScalingReconcilerTest {eq("Launching a new EC2 instance: i-pending"),eq("An instance was started in response to a desired capacity change."),eq("Successful"));- verify(asgService, times(2)).saveAutoScalingGroup(asg);+ verify(asgService, times(2)).saveAutoScalingGroupIfPresent(asg);}@Test@@ -478,6 +527,7 @@ class AutoScalingReconcilerTest {"The launch template 'app-lt' does not exist.", 400));AutoScalingReconciler reconciler =new AutoScalingReconciler(asgService, ec2Service, mock(ElbV2Service.class));+ when(asgService.saveAutoScalingGroupIfPresent(asg)).thenReturn(true);reconciler.reconcile(asg);@@ -511,6 +561,29 @@ class AutoScalingReconcilerTest {return asg;}+ private static AutoScalingGroup launchTemplateAsg() {+ AutoScalingGroup asg = new AutoScalingGroup();+ asg.setRegion("us-east-1");+ asg.setAutoScalingGroupName("app-asg");+ asg.setDesiredCapacity(1);+ asg.setLaunchTemplateId("lt-123");+ asg.setLaunchTemplateVersion("1");+ return asg;+ }++ private static void stubLaunchTemplate(Ec2Service ec2Service, String imageId, String instanceType) {+ LaunchTemplate launchTemplate = new LaunchTemplate();+ launchTemplate.setLaunchTemplateId("lt-123");+ LaunchTemplate version = new LaunchTemplate();+ version.setLatestVersionNumber("1");+ version.getData().setImageId(imageId);+ version.getData().setInstanceType(instanceType);+ when(ec2Service.describeLaunchTemplates("us-east-1", List.of("lt-123"), List.of(), Map.of()))+ .thenReturn(List.of(launchTemplate));+ when(ec2Service.describeLaunchTemplateVersions("us-east-1", "lt-123", null, List.of("1")))+ .thenReturn(List.of(version));+ }+private static AsgInstance instance(String lifecycleState) {AsgInstance instance = new AsgInstance();instance.setLifecycleState(lifecycleState);diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.javaindex 06513dd9c4..d32b4afe24 100644--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java@@ -588,6 +588,16 @@ class AutoScalingServiceTest {assertEquals("ResourceInUse", error.getErrorCode());}+ @Test+ void conditionalSaveDoesNotReinsertDeletedGroup() {+ AutoScalingGroup group = service.describeAutoScalingGroups(REGION, List.of("test-asg")).getFirst();++ service.deleteAutoScalingGroup(REGION, "test-asg", false);++ assertFalse(service.saveAutoScalingGroupIfPresent(group));+ assertTrue(service.describeAutoScalingGroups(REGION, List.of("test-asg")).isEmpty());+ }+@Testvoid forceDeleteAutoScalingGroupTerminatesActiveEc2Instances() {Ec2Service ec2Service = mock(Ec2Service.class);