Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(autoscaling): prevent reconciler resurrecting deleted groups (#3818)

Vũ Thế Huybd9700db0d1a
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
index bce1108164..b5792dc0ce 100644
--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
@@ -22,6 +22,7 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import io.quarkus.runtime.StartupEvent;
+import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -102,8 +103,9 @@ public class AutoScalingReconciler {
} else if (activeCapacity > desired) {
scaleIn(asg, (int) (activeCapacity - desired));
}
- asgService.saveAutoScalingGroup(asg);
- asgService.completeInstanceRefreshIfSettled(asg.getRegion(), asg.getAutoScalingGroupName());
+ if (asgService.saveAutoScalingGroupIfPresent(asg)) {
+ asgService.completeInstanceRefreshIfSettled(asg.getRegion(), asg.getAutoScalingGroupName());
+ }
}
static long activeCapacity(AutoScalingGroup asg) {
@@ -148,7 +150,7 @@ public class AutoScalingReconciler {
}
}
if (changed) {
- asgService.saveAutoScalingGroup(asg);
+ asgService.saveAutoScalingGroupIfPresent(asg);
}
}
@@ -167,7 +169,7 @@ public class AutoScalingReconciler {
deregisterFromTargetGroups(asg, instanceIds);
deregisterFromClassicLoadBalancers(asg, instanceIds);
asg.getInstances().removeIf(instance -> instanceIds.contains(instance.getInstanceId()));
- asgService.saveAutoScalingGroup(asg);
+ asgService.saveAutoScalingGroupIfPresent(asg);
asgService.recordActivity(asg.getRegion(), asg.getAutoScalingGroupName(),
"Removing stale EC2 instance reference(s): " + instanceIds,
"Persisted Auto Scaling state referenced instance containers that are no longer running.",
@@ -279,7 +281,7 @@ public class AutoScalingReconciler {
}
asg.getInstances().removeIf(instance -> instanceIds.contains(instance.getInstanceId()));
- asgService.saveAutoScalingGroup(asg);
+ asgService.saveAutoScalingGroupIfPresent(asg);
asgService.recordActivity(asg.getRegion(), asg.getAutoScalingGroupName(),
"Terminating EC2 instance(s) for refresh: " + instanceIds,
"An instance refresh requested replacement of active instances.",
@@ -318,6 +320,7 @@ public class AutoScalingReconciler {
launchSource.iamInstanceProfile(),
launchSource.associatePublicIpAddress());
+ List<String> launchedInstanceIds = new ArrayList<>();
for (Instance ec2Inst : reservation.getInstances()) {
AsgInstance asgInst = new AsgInstance();
asgInst.setInstanceId(ec2Inst.getInstanceId());
@@ -330,10 +333,18 @@ public class AutoScalingReconciler {
asgInst.setLaunchTemplateVersion(launchSource.launchTemplateVersion());
asgInst.setInstanceType(launchSource.instanceType());
asg.getInstances().add(asgInst);
+ launchedInstanceIds.add(ec2Inst.getInstanceId());
LOG.infov("ASG {0}: launched instance {1} (Pending)",
asg.getAutoScalingGroupName(), ec2Inst.getInstanceId());
}
- asgService.saveAutoScalingGroup(asg);
+ if (!asgService.saveAutoScalingGroupIfPresent(asg) && !launchedInstanceIds.isEmpty()) {
+ try {
+ ec2Service.terminateInstances(asg.getRegion(), launchedInstanceIds);
+ } catch (Exception e) {
+ LOG.warnv("ASG {0}: failed to clean up instances {1} after group deletion: {2}",
+ asg.getAutoScalingGroupName(), launchedInstanceIds, e.getMessage());
+ }
+ }
} catch (Exception e) {
LOG.warnv("ASG {0}: failed to launch instances: {1}",
asg.getAutoScalingGroupName(), e.getMessage());
@@ -384,7 +395,7 @@ public class AutoScalingReconciler {
}
asg.getInstances().removeIf(i -> instanceIds.contains(i.getInstanceId()));
- asgService.saveAutoScalingGroup(asg);
+ asgService.saveAutoScalingGroupIfPresent(asg);
asgService.recordActivity(asg.getRegion(), asg.getAutoScalingGroupName(),
"Terminating EC2 instance(s): " + instanceIds,
"An instance was terminated in response to a desired capacity change.",
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java
index 4d99d32948..1c7d978db1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java
@@ -63,6 +63,7 @@ public class AutoScalingService {
// region :: name → resource
private Map<String, LaunchConfiguration> launchConfigs = new ConcurrentHashMap<>();
private Map<String, AutoScalingGroup> groups = new ConcurrentHashMap<>();
+ private final Map<String, Object> groupLocks = new ConcurrentHashMap<>();
private Map<String, LifecycleHook> hooks = new ConcurrentHashMap<>();
private Map<String, ScalingPolicy> policies = new ConcurrentHashMap<>();
private Map<String, ScalingActivity> activities = new ConcurrentHashMap<>();
@@ -328,29 +329,32 @@ public class AutoScalingService {
}
public void deleteAutoScalingGroup(String region, String name, boolean forceDelete) {
- AutoScalingGroup asg = requireGroup(region, name);
- List<AsgInstance> active = asg.getInstances().stream()
- .filter(i -> !"Terminated".equals(i.getLifecycleState()))
- .collect(Collectors.toList());
- if (!active.isEmpty() && !forceDelete) {
- throw new AwsException("ResourceInUse",
- "Auto Scaling group '" + name + "' has " + active.size()
- + " instance(s). Set ForceDelete=true to delete anyway.", 400);
- }
- if (forceDelete && ec2Service != null && !active.isEmpty()) {
- active.stream()
- .map(AsgInstance::getInstanceId)
- .filter(Objects::nonNull)
- .forEach(instanceId -> {
- try {
- ec2Service.terminateInstances(region, List.of(instanceId));
- }
- catch (AwsException ignored) {
- // ForceDelete should remove stale ASG membership even if EC2 no longer has the instance.
- }
- });
- }
- groups.remove(asgKey(region, name));
+ String key = asgKey(region, name);
+ synchronized (lockFor(key)) {
+ AutoScalingGroup asg = requireGroup(region, name);
+ List<AsgInstance> active = asg.getInstances().stream()
+ .filter(i -> !"Terminated".equals(i.getLifecycleState()))
+ .collect(Collectors.toList());
+ if (!active.isEmpty() && !forceDelete) {
+ throw new AwsException("ResourceInUse",
+ "Auto Scaling group '" + name + "' has " + active.size()
+ + " instance(s). Set ForceDelete=true to delete anyway.", 400);
+ }
+ if (forceDelete && ec2Service != null && !active.isEmpty()) {
+ active.stream()
+ .map(AsgInstance::getInstanceId)
+ .filter(Objects::nonNull)
+ .forEach(instanceId -> {
+ try {
+ ec2Service.terminateInstances(region, List.of(instanceId));
+ }
+ catch (AwsException ignored) {
+ // ForceDelete should remove stale ASG membership even if EC2 no longer has the instance.
+ }
+ });
+ }
+ groups.remove(key);
+ }
// clean up associated hooks and policies
hooks.entrySet().removeIf(e -> e.getValue().getAutoScalingGroupName().equals(name));
policies.entrySet().removeIf(e -> e.getValue().getAutoScalingGroupName().equals(name));
@@ -372,8 +376,20 @@ public class AutoScalingService {
.collect(Collectors.toList());
}
- public void saveAutoScalingGroup(AutoScalingGroup asg) {
- groups.put(asgKey(asg.getRegion(), asg.getAutoScalingGroupName()), asg);
+ public boolean saveAutoScalingGroupIfPresent(AutoScalingGroup asg) {
+ String key = asgKey(asg.getRegion(), asg.getAutoScalingGroupName());
+ synchronized (lockFor(key)) {
+ AutoScalingGroup current = groups.get(key);
+ if (current == asg) {
+ groups.put(key, asg);
+ return true;
+ }
+ return false;
+ }
+ }
+
+ private Object lockFor(String key) {
+ return groupLocks.computeIfAbsent(key, ignored -> new Object());
}
public void setDesiredCapacity(String region, String name, int desiredCapacity) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
index edec6f7ae9..cbce25505d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
@@ -78,6 +78,7 @@ class AutoScalingReconcilerTest {
asg.setRegion("us-east-1");
asg.setAutoScalingGroupName("app-asg");
asg.setDesiredCapacity(0);
+ when(asgService.saveAutoScalingGroupIfPresent(asg)).thenReturn(true);
reconciler.reconcile(asg);
@@ -144,6 +145,54 @@ class AutoScalingReconcilerTest {
assertEquals("2001", tags.getValue().get(1).getValue());
}
+ @Test
+ void scaleOutTerminatesInstancesWhenGroupDisappears() {
+ AutoScalingService asgService = mock(AutoScalingService.class);
+ Ec2Service ec2Service = mock(Ec2Service.class);
+ ElbV2Service elbV2Service = mock(ElbV2Service.class);
+ AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
+ AutoScalingGroup asg = launchTemplateAsg();
+ stubLaunchTemplate(ec2Service, "ami-version-1", "t3.micro");
+ when(asgService.saveAutoScalingGroupIfPresent(asg)).thenReturn(false);
+
+ Instance ec2Instance = new Instance();
+ ec2Instance.setInstanceId("i-stale");
+ Reservation reservation = new Reservation();
+ reservation.setInstances(List.of(ec2Instance));
+ when(ec2Service.runInstances(eq("us-east-1"), eq("ami-version-1"), eq("t3.micro"),
+ eq(1), eq(1), eq(null), eq(List.of()), eq(null), eq(null), eq(List.of()),
+ eq(null), eq(null), eq(null))).thenReturn(reservation);
+
+ reconciler.reconcile(asg);
+
+ verify(ec2Service).terminateInstances("us-east-1", List.of("i-stale"));
+ verify(asgService, never()).completeInstanceRefreshIfSettled("us-east-1", "app-asg");
+ }
+
+ @Test
+ void scaleOutPersistsInstancesWhenGroupStillExists() {
+ AutoScalingService asgService = mock(AutoScalingService.class);
+ Ec2Service ec2Service = mock(Ec2Service.class);
+ ElbV2Service elbV2Service = mock(ElbV2Service.class);
+ AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
+ AutoScalingGroup asg = launchTemplateAsg();
+ stubLaunchTemplate(ec2Service, "ami-version-1", "t3.micro");
+ when(asgService.saveAutoScalingGroupIfPresent(asg)).thenReturn(true);
+
+ Instance ec2Instance = new Instance();
+ ec2Instance.setInstanceId("i-owned");
+ Reservation reservation = new Reservation();
+ reservation.setInstances(List.of(ec2Instance));
+ when(ec2Service.runInstances(eq("us-east-1"), eq("ami-version-1"), eq("t3.micro"),
+ eq(1), eq(1), eq(null), eq(List.of()), eq(null), eq(null), eq(List.of()),
+ eq(null), eq(null), eq(null))).thenReturn(reservation);
+
+ reconciler.reconcile(asg);
+
+ verify(asgService, times(2)).saveAutoScalingGroupIfPresent(asg);
+ verify(ec2Service, never()).terminateInstances("us-east-1", List.of("i-owned"));
+ }
+
@Test
void scaleOutForwardsLaunchConfigurationAssociatePublicIpAddressBothDirections() {
// An LC that explicitly sets false must suppress the public IP even in a
@@ -306,7 +355,7 @@ class AutoScalingReconcilerTest {
targets.capture());
assertEquals(1, targets.getValue().size());
assertEquals("i-stale", targets.getValue().getFirst().getId());
- verify(asgService).saveAutoScalingGroup(asg);
+ verify(asgService).saveAutoScalingGroupIfPresent(asg);
verify(ec2Service, never()).terminateInstances(asg.getRegion(), List.of("i-active"));
}
@@ -466,7 +515,7 @@ class AutoScalingReconcilerTest {
eq("Launching a new EC2 instance: i-pending"),
eq("An instance was started in response to a desired capacity change."),
eq("Successful"));
- verify(asgService, times(2)).saveAutoScalingGroup(asg);
+ verify(asgService, times(2)).saveAutoScalingGroupIfPresent(asg);
}
@Test
@@ -478,6 +527,7 @@ class AutoScalingReconcilerTest {
"The launch template 'app-lt' does not exist.", 400));
AutoScalingReconciler reconciler =
new AutoScalingReconciler(asgService, ec2Service, mock(ElbV2Service.class));
+ when(asgService.saveAutoScalingGroupIfPresent(asg)).thenReturn(true);
reconciler.reconcile(asg);
@@ -511,6 +561,29 @@ class AutoScalingReconcilerTest {
return asg;
}
+ private static AutoScalingGroup launchTemplateAsg() {
+ AutoScalingGroup asg = new AutoScalingGroup();
+ asg.setRegion("us-east-1");
+ asg.setAutoScalingGroupName("app-asg");
+ asg.setDesiredCapacity(1);
+ asg.setLaunchTemplateId("lt-123");
+ asg.setLaunchTemplateVersion("1");
+ return asg;
+ }
+
+ private static void stubLaunchTemplate(Ec2Service ec2Service, String imageId, String instanceType) {
+ LaunchTemplate launchTemplate = new LaunchTemplate();
+ launchTemplate.setLaunchTemplateId("lt-123");
+ LaunchTemplate version = new LaunchTemplate();
+ version.setLatestVersionNumber("1");
+ version.getData().setImageId(imageId);
+ version.getData().setInstanceType(instanceType);
+ when(ec2Service.describeLaunchTemplates("us-east-1", List.of("lt-123"), List.of(), Map.of()))
+ .thenReturn(List.of(launchTemplate));
+ when(ec2Service.describeLaunchTemplateVersions("us-east-1", "lt-123", null, List.of("1")))
+ .thenReturn(List.of(version));
+ }
+
private static AsgInstance instance(String lifecycleState) {
AsgInstance instance = new AsgInstance();
instance.setLifecycleState(lifecycleState);
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java
index 06513dd9c4..d32b4afe24 100644
--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java
@@ -588,6 +588,16 @@ class AutoScalingServiceTest {
assertEquals("ResourceInUse", error.getErrorCode());
}
+ @Test
+ void conditionalSaveDoesNotReinsertDeletedGroup() {
+ AutoScalingGroup group = service.describeAutoScalingGroups(REGION, List.of("test-asg")).getFirst();
+
+ service.deleteAutoScalingGroup(REGION, "test-asg", false);
+
+ assertFalse(service.saveAutoScalingGroupIfPresent(group));
+ assertTrue(service.describeAutoScalingGroups(REGION, List.of("test-asg")).isEmpty());
+ }
+
@Test
void forceDeleteAutoScalingGroupTerminatesActiveEc2Instances() {
Ec2Service ec2Service = mock(Ec2Service.class);