Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(autoscaling): pause suspended capacity adjustments (#5545)

David Phillips41517f3c2ffb
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingCapacitySuspensionTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingCapacitySuspensionTest.java
new file mode 100644
index 0000000000..99829d7303
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingCapacitySuspensionTest.java
@@ -0,0 +1,49 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.autoscaling.AutoScalingClient;
+import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup;
+import software.amazon.awssdk.services.autoscaling.model.SuspendedProcess;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@DisplayName("Auto Scaling capacity process suspension")
+class AutoScalingCapacitySuspensionTest {
+ @Test
+ @DisplayName("Launch and Terminate suspension round trips retain desired capacity")
+ void suspendAndResumeRoundTrip() {
+ String name = TestFixtures.uniqueName("capacity-pause");
+ try (AutoScalingClient asg = TestFixtures.autoScalingClient()) {
+ asg.createLaunchConfiguration(r -> r.launchConfigurationName(name)
+ .imageId("ami-amazonlinux2023").instanceType("t3.micro"));
+ try {
+ asg.createAutoScalingGroup(r -> r.autoScalingGroupName(name).launchConfigurationName(name)
+ .minSize(0).maxSize(1).desiredCapacity(0).availabilityZones("us-east-1a"));
+ try {
+ for (String process : List.of("Launch", "Terminate")) {
+ asg.suspendProcesses(r -> r.autoScalingGroupName(name).scalingProcesses(process));
+ AutoScalingGroup paused = group(asg, name);
+ assertThat(paused.suspendedProcesses()).extracting(SuspendedProcess::processName)
+ .containsExactly(process);
+ assertThat(paused.desiredCapacity()).isZero();
+ asg.resumeProcesses(r -> r.autoScalingGroupName(name).scalingProcesses(process));
+ AutoScalingGroup resumed = group(asg, name);
+ assertThat(resumed.suspendedProcesses()).isEmpty();
+ assertThat(resumed.desiredCapacity()).isZero();
+ }
+ } finally {
+ asg.deleteAutoScalingGroup(r -> r.autoScalingGroupName(name).forceDelete(true));
+ }
+ } finally {
+ asg.deleteLaunchConfiguration(r -> r.launchConfigurationName(name));
+ }
+ }
+ }
+
+ private static AutoScalingGroup group(AutoScalingClient asg, String name) {
+ return asg.describeAutoScalingGroups(r -> r.autoScalingGroupNames(name)).autoScalingGroups().get(0);
+ }
+}
diff --git a/docs/services/autoscaling.md b/docs/services/autoscaling.md
index da544e6ba3..e869b63914 100644
--- a/docs/services/autoscaling.md
+++ b/docs/services/autoscaling.md
@@ -129,6 +129,19 @@ Floci runs a background reconciler (10 s fixed rate) that keeps each group's InS
- **Scale-in**: selects InService instances not protected from scale-in, deregisters them from target groups, then calls `TerminateInstances`.
- Activity records are written on each scale-out and scale-in event.
+`SuspendProcesses` pauses desired-capacity scale-out for `Launch` and scale-in for
+`Terminate`. The desired count remains stored, and `ResumeProcesses` allows the
+next reconciliation pass to continue toward it. Omitting process names suspends
+or resumes all processes. Suspending another process does not pause these capacity
+adjustments. Suspension and resumption wait for any desired-capacity adjustment already
+in progress for that group, so a completed suspension cannot be bypassed by an earlier
+reconciliation check.
+
+Suspension does not yet pause instance-refresh selection or replacement cleanup.
+Explicitly requested termination and stale-instance cleanup continue independently.
+Other suspended process names are stored and returned but do not pause their
+associated activities.
+
## Launch Source Compatibility
Auto Scaling groups preserve either a launch configuration, a top-level launch template, or a `MixedInstancesPolicy`. These launch sources are mutually exclusive in create/update requests. When a mixed instances policy is supplied, Floci stores and returns:
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
index 869418f3a7..691ffd2065 100644
--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
@@ -129,14 +129,16 @@ public class AutoScalingReconciler {
removeOrphanedTargetRegistrations(asg);
promoteReadyInstances(asg);
- long activeCapacity = activeCapacity(asg);
- int desired = asg.getDesiredCapacity();
-
- if (activeCapacity < desired) {
- scaleOut(asg, (int) (desired - activeCapacity));
- } else if (activeCapacity > desired) {
- scaleIn(asg, (int) (activeCapacity - desired));
- }
+ asgService.adjustCapacityIfPresent(asg, () -> {
+ long activeCapacity = activeCapacity(asg);
+ int desired = asg.getDesiredCapacity();
+
+ if (activeCapacity < desired && !asg.getSuspendedProcesses().contains("Launch")) {
+ scaleOut(asg, (int) (desired - activeCapacity));
+ } else if (activeCapacity > desired && !asg.getSuspendedProcesses().contains("Terminate")) {
+ scaleIn(asg, (int) (activeCapacity - desired));
+ }
+ });
if (asgService.saveAutoScalingGroupIfPresent(asg)) {
asgService.completeInstanceRefreshIfSettled(asg.getRegion(), asg.getAutoScalingGroupName());
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java
index 2b9bd9387c..dca198c488 100644
--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java
@@ -493,8 +493,17 @@ public class AutoScalingService {
}
}
+ void adjustCapacityIfPresent(AutoScalingGroup asg, Runnable adjustment) {
+ String key = asgKey(asg.getRegion(), asg.getAutoScalingGroupName());
+ synchronized (lockFor(key)) {
+ if (groups.get(key) == asg) {
+ adjustment.run();
+ }
+ }
+ }
+
private Object lockFor(String key) {
- return groupLocks.computeIfAbsent(key, ignored -> new Object());
+ return groupLocks.computeIfAbsent(regionResolver.getAccountId() + "::" + key, ignored -> new Object());
}
Set<String> autoScalingGroupAccountIds() {
@@ -544,36 +553,38 @@ public class AutoScalingService {
return requireGroup(region, resourceId);
}
- // Real AWS suspends the named scaling processes ("Launch", "Terminate", "HealthCheck", ...)
- // so they stop running until resumed; an empty list suspends all of them. Floci has no
- // scaling loop for these processes to pause, so this only needs to record which processes
- // are suspended for DescribeAutoScalingGroups to report back - the same "accept and
- // remember" shape as terminationPolicies. The AWS provider calls this unconditionally
- // around ASG creation whenever wait_for_capacity_timeout is non-zero (the default), so an
- // unimplemented SuspendProcesses fails ASG creation outright, not just a feature gap.
+ // An empty list suspends all processes. The reconciler honors Launch and Terminate
+ // for desired-capacity adjustments; other names remain stored state. Refresh selection
+ // and cleanup are separate paths and do not yet honor suspended processes.
private static final List<String> ALL_SCALING_PROCESSES = List.of(
"Launch", "Terminate", "AddToLoadBalancer", "AlarmNotification", "AZRebalance",
"HealthCheck", "InstanceRefresh", "ReplaceUnhealthy", "ScheduledActions");
public void suspendProcesses(String region, String name, List<String> processes) {
- AutoScalingGroup asg = requireGroup(region, name);
- List<String> toSuspend = processes.isEmpty() ? ALL_SCALING_PROCESSES : processes;
- Set<String> suspended = new LinkedHashSet<>(asg.getSuspendedProcesses());
- suspended.addAll(toSuspend);
- asg.setSuspendedProcesses(new ArrayList<>(suspended));
- groups.put(asgKey(region, name), asg);
+ String key = asgKey(region, name);
+ synchronized (lockFor(key)) {
+ AutoScalingGroup asg = requireGroup(region, name);
+ List<String> toSuspend = processes.isEmpty() ? ALL_SCALING_PROCESSES : processes;
+ Set<String> suspended = new LinkedHashSet<>(asg.getSuspendedProcesses());
+ suspended.addAll(toSuspend);
+ asg.setSuspendedProcesses(new ArrayList<>(suspended));
+ groups.put(key, asg);
+ }
}
public void resumeProcesses(String region, String name, List<String> processes) {
- AutoScalingGroup asg = requireGroup(region, name);
- if (processes.isEmpty()) {
- asg.setSuspendedProcesses(new ArrayList<>());
- } else {
- List<String> remaining = new ArrayList<>(asg.getSuspendedProcesses());
- remaining.removeAll(processes);
- asg.setSuspendedProcesses(remaining);
+ String key = asgKey(region, name);
+ synchronized (lockFor(key)) {
+ AutoScalingGroup asg = requireGroup(region, name);
+ if (processes.isEmpty()) {
+ asg.setSuspendedProcesses(new ArrayList<>());
+ } else {
+ List<String> remaining = new ArrayList<>(asg.getSuspendedProcesses());
+ remaining.removeAll(processes);
+ asg.setSuspendedProcesses(remaining);
+ }
+ groups.put(key, asg);
}
- groups.put(asgKey(region, name), asg);
}
// ── Instance management ────────────────────────────────────────────────────
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java
new file mode 100644
index 0000000000..48faa51c1a
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java
@@ -0,0 +1,79 @@
+package io.github.hectorvent.floci.services.autoscaling;
+
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.services.autoscaling.model.AsgOptionalFields;
+import io.github.hectorvent.floci.services.autoscaling.model.AutoScalingGroup;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@QuarkusTest
+class AutoScalingCapacityAccountLockIntegrationTest {
+ private static final String ACCOUNT = "111111111111";
+ private static final String REGION = "us-east-1";
+
+ @Inject
+ AutoScalingService service;
+
+ @ParameterizedTest
+ @CsvSource({"222222222222,us-east-1", "111111111111,us-west-2"})
+ void unrelatedAccountOrRegionDoesNotWaitForCapacityOperation(String otherAccount, String otherRegion)
+ throws Exception {
+ String name = "capacity-lock-" + UUID.randomUUID();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ ExecutorService workers = Executors.newFixedThreadPool(2);
+ try {
+ AutoScalingGroup group = RequestScopes.callAs(ACCOUNT, REGION, () -> create(REGION, name));
+ RequestScopes.runAs(otherAccount, otherRegion, () -> create(otherRegion, name));
+ Future<?> adjustment = workers.submit(() -> RequestScopes.runAs(ACCOUNT, REGION,
+ () -> service.adjustCapacityIfPresent(group, () -> {
+ entered.countDown();
+ try {
+ assertTrue(release.await(10, TimeUnit.SECONDS));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ })));
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ Future<?> suspension = workers.submit(() -> RequestScopes.runAs(otherAccount, otherRegion,
+ () -> service.suspendProcesses(otherRegion, name, List.of("Launch"))));
+ suspension.get(2, TimeUnit.SECONDS);
+ RequestScopes.runAs(otherAccount, otherRegion, () -> assertTrue(service
+ .describeAutoScalingGroups(otherRegion, List.of(name)).getFirst()
+ .getSuspendedProcesses().contains("Launch")));
+ release.countDown();
+ adjustment.get(5, TimeUnit.SECONDS);
+ } finally {
+ release.countDown();
+ workers.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ try {
+ RequestScopes.runAs(ACCOUNT, REGION, () -> service.deleteAutoScalingGroup(REGION, name, true));
+ } finally {
+ RequestScopes.runAs(otherAccount, otherRegion,
+ () -> service.deleteAutoScalingGroup(otherRegion, name, true));
+ }
+ }
+ }
+
+ private AutoScalingGroup create(String region, String name) {
+ service.createAutoScalingGroup(region, name, "unused-capacity-lock-source", null, null, null, null,
+ 0, 1, 0, 300, List.of(region + "a"), List.of(), List.of(), List.of(), "EC2", 0,
+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());
+ return service.describeAutoScalingGroups(region, List.of(name)).getFirst();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionIntegrationTest.java
new file mode 100644
index 0000000000..a3eee18abb
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionIntegrationTest.java
@@ -0,0 +1,111 @@
+package io.github.hectorvent.floci.services.autoscaling;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import io.restassured.specification.RequestSpecification;
+import jakarta.inject.Inject;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@QuarkusTest
+class AutoScalingCapacitySuspensionIntegrationTest {
+ private static final String GROUP_PATH = "DescribeAutoScalingGroupsResponse."
+ + "DescribeAutoScalingGroupsResult.AutoScalingGroups.member";
+
+ @Inject
+ AutoScalingReconciler reconciler;
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void suspendAndResumePreserveDesiredCapacityAndControlEc2Instances(boolean allProcesses) {
+ String name = "capacity-pause-" + UUID.randomUUID().toString().substring(0, 8);
+ request("CreateLaunchConfiguration")
+ .formParam("LaunchConfigurationName", name)
+ .formParam("ImageId", "ami-amazonlinux2023").formParam("InstanceType", "t3.micro")
+ .post("/").then().statusCode(200);
+ try {
+ request("CreateAutoScalingGroup").formParam("AutoScalingGroupName", name)
+ .formParam("LaunchConfigurationName", name)
+ .formParam("MinSize", 0).formParam("MaxSize", 1).formParam("DesiredCapacity", 0)
+ .formParam("AvailabilityZones.member.1", "us-east-1a")
+ .post("/").then().statusCode(200);
+ try {
+ process("SuspendProcesses", name, allProcesses, "Launch");
+ setDesired(name, 1);
+ assertPaused(name, 1, 0);
+ process("ResumeProcesses", name, allProcesses, "Launch");
+ reconciler.reconcileAll();
+ reconciler.reconcileAll();
+ assertCapacity(name, 1, 1);
+ String instanceId = describe(name).xmlPath().getString(GROUP_PATH + ".Instances.member.InstanceId");
+
+ process("SuspendProcesses", name, allProcesses, "Terminate");
+ setDesired(name, 0);
+ assertPaused(name, 0, 1);
+ assertEquals("running", instanceState(instanceId));
+ process("ResumeProcesses", name, allProcesses, "Terminate");
+ reconciler.reconcileAll();
+ reconciler.reconcileAll();
+ assertCapacity(name, 0, 0);
+ assertEquals("terminated", instanceState(instanceId));
+ } finally {
+ request("DeleteAutoScalingGroup").formParam("AutoScalingGroupName", name)
+ .formParam("ForceDelete", true).post("/").then().statusCode(200);
+ }
+ } finally {
+ request("DeleteLaunchConfiguration").formParam("LaunchConfigurationName", name)
+ .post("/").then().statusCode(200);
+ }
+ }
+
+ private void assertPaused(String name, int desired, int instances) {
+ reconciler.reconcileAll();
+ reconciler.reconcileAll();
+ assertCapacity(name, desired, instances);
+ }
+
+ private static void assertCapacity(String name, int desired, int instances) {
+ Response response = describe(name);
+ assertEquals(desired, response.xmlPath().getInt(GROUP_PATH + ".DesiredCapacity"));
+ assertEquals(instances, response.xmlPath().getList(GROUP_PATH + ".Instances.member").size());
+ }
+
+ private static Response describe(String name) {
+ return request("DescribeAutoScalingGroups").formParam("AutoScalingGroupNames.member.1", name)
+ .post("/").then().statusCode(200).extract().response();
+ }
+
+ private static String instanceState(String id) {
+ return given().header("Authorization", auth("ec2"))
+ .formParam("Action", "DescribeInstances").formParam("InstanceId.1", id)
+ .post("/").then().statusCode(200).extract().xmlPath()
+ .getString("DescribeInstancesResponse.reservationSet.item.instancesSet.item.instanceState.name");
+ }
+
+ private static void setDesired(String name, int desired) {
+ request("SetDesiredCapacity").formParam("AutoScalingGroupName", name)
+ .formParam("DesiredCapacity", desired).post("/").then().statusCode(200);
+ }
+
+ private static void process(String action, String name, boolean allProcesses, String process) {
+ RequestSpecification request = request(action).formParam("AutoScalingGroupName", name);
+ if (!allProcesses) {
+ request.formParam("ScalingProcesses.member.1", process);
+ }
+ request.post("/").then().statusCode(200);
+ }
+
+ private static RequestSpecification request(String action) {
+ return given().header("Authorization", auth("autoscaling")).formParam("Action", action);
+ }
+
+ private static String auth(String service) {
+ return "AWS4-HMAC-SHA256 Credential=AKID/20260215/us-east-1/" + service
+ + "/aws4_request, SignedHeaders=host, Signature=abc";
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionTest.java
new file mode 100644
index 0000000000..20ce95a78b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionTest.java
@@ -0,0 +1,202 @@
+package io.github.hectorvent.floci.services.autoscaling;
+
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.autoscaling.model.AsgInstance;
+import io.github.hectorvent.floci.services.autoscaling.model.AsgOptionalFields;
+import io.github.hectorvent.floci.services.autoscaling.model.AutoScalingGroup;
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.ec2.model.Instance;
+import io.github.hectorvent.floci.services.ec2.model.InstanceState;
+import io.github.hectorvent.floci.services.ec2.model.LaunchTemplate;
+import io.github.hectorvent.floci.services.ec2.model.Reservation;
+import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+class AutoScalingCapacitySuspensionTest {
+ private static final String REGION = "us-east-1";
+ private AutoScalingService service;
+ private Ec2Service ec2;
+ private AutoScalingReconciler reconciler;
+ private AutoScalingGroup group;
+
+ @BeforeEach
+ void setUp() {
+ service = new AutoScalingService();
+ service.regionResolver = new RegionResolver(REGION, "000000000000");
+ service.createAutoScalingGroup(REGION, "suspended-group", null, "lt-original", null, "1", null,
+ 0, 3, 1, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,
+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());
+ group = service.describeAutoScalingGroups(REGION, List.of("suspended-group")).getFirst();
+ AsgInstance existing = new AsgInstance();
+ existing.setInstanceId("i-original");
+ existing.setLifecycleState("InService");
+ existing.setHealthStatus("Healthy");
+ existing.setLaunchTemplateId("lt-original");
+ existing.setLaunchTemplateVersion("1");
+ group.getInstances().add(existing);
+ service.saveAutoScalingGroupIfPresent(group);
+ ec2 = mock(Ec2Service.class);
+ service.ec2Service = ec2;
+ reconciler = new AutoScalingReconciler(service, ec2, mock(ElbV2Service.class));
+ when(ec2.isInstanceContainerRunning(anyString())).thenReturn(true);
+ when(ec2.describeInstances(eq(REGION), anyList(), isNull()))
+ .thenReturn(List.of(reservation("i-original")));
+ LaunchTemplate template = new LaunchTemplate();
+ template.setLaunchTemplateId("lt-original");
+ template.setLatestVersionNumber("1");
+ template.getData().setImageId("ami-original");
+ template.getData().setInstanceType("t3.micro");
+ when(ec2.describeLaunchTemplates(eq(REGION), anyList(), anyList(), anyMap())).thenReturn(List.of(template));
+ when(ec2.describeLaunchTemplateVersions(eq(REGION), eq("lt-original"), isNull(), anyList()))
+ .thenReturn(List.of(template));
+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),
+ anyList(), any(), any(), any())).thenReturn(reservation("i-replacement"));
+ }
+
+ @ParameterizedTest
+ @CsvSource({"Launch,2,true", "Terminate,0,true", "ALL,2,true", "ALL,0,true",
+ "Launch,0,false", "Terminate,2,false", "HealthCheck,2,false", "HealthCheck,0,false"})
+ void suspensionPausesOnlyItsCapacityDirectionAndResumeContinues(String process, int desired, boolean paused) {
+ group.setDesiredCapacity(desired);
+ List<String> processes = "ALL".equals(process) ? List.of() : List.of(process);
+ service.suspendProcesses(REGION, "suspended-group", processes);
+ reconciler.reconcile(group);
+ assertEquals(desired, group.getDesiredCapacity());
+ if (paused) {
+ verify(ec2, never()).terminateInstances(anyString(), anyList());
+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),
+ any(), any(), anyList(), any(), any(), any());
+ assertEquals(List.of("i-original"), group.getInstances().stream()
+ .map(AsgInstance::getInstanceId).toList());
+ service.resumeProcesses(REGION, "suspended-group", processes);
+ reconciler.reconcile(group);
+ }
+ assertEquals(desired, group.getInstances().size());
+ if (desired == 0) {
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ } else {
+ verify(ec2).runInstances(eq(REGION), any(), any(), eq(1), eq(1), any(), anyList(),
+ any(), any(), anyList(), any(), any(), any());
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"Terminate", "InstanceRefresh"})
+ void suspensionDoesNotBlockExplicitManualTermination(String process) {
+ service.suspendProcesses(REGION, "suspended-group", List.of(process));
+ service.terminateInstanceInAutoScalingGroup(REGION, "i-original", true);
+ reconciler.reconcile(group);
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ assertTrue(group.getInstances().isEmpty());
+ }
+
+ @ParameterizedTest
+ @CsvSource({"Launch,false", "Launch,true", "Terminate,false", "Terminate,true"})
+ void suspensionWaitsForCapacityOperationAlreadyPastItsGuard(String process, boolean allProcesses)
+ throws Exception {
+ CountDownLatch operationEntered = new CountDownLatch(1);
+ CountDownLatch releaseOperation = new CountDownLatch(1);
+ AtomicReference<Thread> suspensionThread = new AtomicReference<>();
+ if ("Launch".equals(process)) {
+ group.setDesiredCapacity(2);
+ doAnswer(invocation -> {
+ operationEntered.countDown();
+ assertTrue(releaseOperation.await(10, TimeUnit.SECONDS));
+ LaunchTemplate template = new LaunchTemplate();
+ template.setLaunchTemplateId("lt-original");
+ template.setLatestVersionNumber("1");
+ template.getData().setImageId("ami-original");
+ template.getData().setInstanceType("t3.micro");
+ return List.of(template);
+ }).when(ec2).describeLaunchTemplates(eq(REGION), anyList(), anyList(), anyMap());
+ } else {
+ group.setDesiredCapacity(0);
+ AsgInstance existing = spy(group.getInstances().getFirst());
+ group.getInstances().set(0, existing);
+ doAnswer(invocation -> {
+ operationEntered.countDown();
+ assertTrue(releaseOperation.await(10, TimeUnit.SECONDS));
+ return false;
+ }).when(existing).isProtectedFromScaleIn();
+ }
+ service.createAutoScalingGroup(REGION, "unrelated-group", null, "lt-original", null, "1", null,
+ 0, 3, 0, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,
+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());
+ ExecutorService workers = Executors.newFixedThreadPool(3);
+ try {
+ Future<?> reconciliation = workers.submit(() -> reconciler.reconcile(group));
+ assertTrue(operationEntered.await(5, TimeUnit.SECONDS));
+ Future<?> suspension = workers.submit(() -> {
+ suspensionThread.set(Thread.currentThread());
+ service.suspendProcesses(REGION, "suspended-group", allProcesses ? List.of() : List.of(process));
+ });
+ await().atMost(Duration.ofSeconds(5)).until(() -> suspension.isDone()
+ || suspensionThread.get() != null && suspensionThread.get().getState() == Thread.State.BLOCKED);
+ assertFalse(suspension.isDone(), "Suspension must wait for the capacity operation it raced");
+ workers.submit(() -> service.suspendProcesses(REGION, "unrelated-group", List.of(process)))
+ .get(5, TimeUnit.SECONDS);
+ releaseOperation.countDown();
+ reconciliation.get(5, TimeUnit.SECONDS);
+ suspension.get(5, TimeUnit.SECONDS);
+ int completedCapacity = group.getInstances().size();
+ group.setDesiredCapacity("Launch".equals(process) ? completedCapacity + 1 : 0);
+ clearInvocations(ec2);
+ reconciler.reconcile(group);
+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),
+ any(), any(), anyList(), any(), any(), any());
+ verify(ec2, never()).terminateInstances(anyString(), anyList());
+ } finally {
+ releaseOperation.countDown();
+ workers.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void staleGroupCannotAdjustCapacityAfterDeletionOrRecreation(boolean recreate) {
+ group.setDesiredCapacity(2);
+ service.deleteAutoScalingGroup(REGION, "suspended-group", true);
+ if (recreate) {
+ service.createAutoScalingGroup(REGION, "suspended-group", null, "lt-original", null, "1", null,
+ 0, 3, 0, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,
+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());
+ }
+ clearInvocations(ec2);
+ reconciler.reconcile(group);
+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),
+ any(), any(), anyList(), any(), any(), any());
+ if (recreate) {
+ assertTrue(service.describeAutoScalingGroups(REGION, List.of("suspended-group"))
+ .getFirst().getInstances().isEmpty());
+ } else {
+ assertTrue(service.describeAutoScalingGroups(REGION, List.of("suspended-group")).isEmpty());
+ }
+ }
+
+ private static Reservation reservation(String id) {
+ Instance instance = new Instance();
+ instance.setInstanceId(id);
+ instance.setState(InstanceState.running());
+ Reservation reservation = new Reservation();
+ reservation.setInstances(List.of(instance));
+ return reservation;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
index 45626d18ac..126f954d5a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
@@ -29,8 +29,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyList;
import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -40,6 +42,16 @@ import static org.mockito.Mockito.when;
class AutoScalingReconcilerTest {
+ private static AutoScalingService scalingService() {
+ AutoScalingService service = mock(AutoScalingService.class);
+ doAnswer(invocation -> {
+ Runnable adjustment = invocation.getArgument(1);
+ adjustment.run();
+ return null;
+ }).when(service).adjustCapacityIfPresent(any(), any());
+ return service;
+ }
+
@Test
void pendingInstancesCountAsActiveCapacity() {
AutoScalingGroup asg = new AutoScalingGroup();
@@ -54,7 +66,7 @@ class AutoScalingReconcilerTest {
@Test
void stopTerminatesReconcilerThread() throws Exception {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -73,7 +85,7 @@ class AutoScalingReconcilerTest {
@Test
void reconcileCompletesSettledInstanceRefreshes() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -90,7 +102,7 @@ class AutoScalingReconcilerTest {
@Test
void scaleOutUsesRequestedLaunchTemplateVersionData() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -154,7 +166,7 @@ class AutoScalingReconcilerTest {
@Test
void scaleOutTerminatesInstancesWhenGroupDisappears() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -178,7 +190,7 @@ class AutoScalingReconcilerTest {
@Test
void scaleOutPersistsInstancesWhenGroupStillExists() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -211,7 +223,7 @@ class AutoScalingReconcilerTest {
}
private void assertLaunchConfigurationForwards(Boolean associatePublicIp) {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -247,7 +259,7 @@ class AutoScalingReconcilerTest {
@Test
void scaleOutStoresResolvedLaunchTemplateVersionForAliases() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -284,7 +296,7 @@ class AutoScalingReconcilerTest {
@Test
void scaleOutUsesMixedInstancesLaunchTemplateSpecification() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -341,7 +353,7 @@ class AutoScalingReconcilerTest {
@Test
void reconcileDeregistersTargetsThatAreNotActiveAsgInstances() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -370,7 +382,7 @@ class AutoScalingReconcilerTest {
@Test
void reconcileKeepsPendingInstancesWhileContainerLaunchIsStillInFlight() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -411,7 +423,7 @@ class AutoScalingReconcilerTest {
@Test
void reconcilePrunesPendingInstancesWhenEc2InstanceIsTerminal() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -442,7 +454,7 @@ class AutoScalingReconcilerTest {
@Test
void reconcileFailsActiveSsmInvocationsBeforePruningStaleInstance() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
SsmCommandService ssmCommandService = mock(SsmCommandService.class);
@@ -472,7 +484,7 @@ class AutoScalingReconcilerTest {
@Test
void reconcileDeregistersStaleInstanceFromClassicLoadBalancerBeforePruning() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
ElbClassicService elbClassicService = mock(ElbClassicService.class);
@@ -495,7 +507,7 @@ class AutoScalingReconcilerTest {
@Test
void reconcilePromotesPendingAsgInstanceWhenEc2InstanceIsRunning() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -529,7 +541,7 @@ class AutoScalingReconcilerTest {
@Test
void deletedLaunchTemplateDoesNotAbortTheReconcilePass() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
AutoScalingGroup asg = asgWhoseLaunchTemplateLookupFails(ec2Service,
new AwsException("InvalidLaunchTemplateName.NotFoundException",
@@ -546,7 +558,7 @@ class AutoScalingReconcilerTest {
@Test
void otherLaunchTemplateErrorsSurfaceRatherThanSilentlySkippingScaling() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
AutoScalingGroup asg = asgWhoseLaunchTemplateLookupFails(ec2Service,
new AwsException("RequestLimitExceeded", "Request limit exceeded.", 503));
@@ -560,7 +572,7 @@ class AutoScalingReconcilerTest {
@Test
void scaleInTerminationFailureKeepsMembershipAndRecordsFailedActivity() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
stubActivityRecording(asgService);
@@ -592,7 +604,7 @@ class AutoScalingReconcilerTest {
@Test
void repeatedTerminationFailureIsRecordedOnce() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
stubActivityRecording(asgService);
@@ -620,7 +632,7 @@ class AutoScalingReconcilerTest {
@Test
void scaleInTerminatesAndRemovesTheInstanceOnSuccess() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -648,7 +660,7 @@ class AutoScalingReconcilerTest {
@Test
void terminatingInstanceTerminationFailureKeepsMembershipAndRecordsFailedActivity() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
stubActivityRecording(asgService);
@@ -684,7 +696,7 @@ class AutoScalingReconcilerTest {
@Test
void terminatingInstanceWithMissingEc2RecordIsPrunedAsStale() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -722,7 +734,7 @@ class AutoScalingReconcilerTest {
@Test
void terminatingStoppedInstanceIsRetriedRatherThanPrunedAsStale() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
stubActivityRecording(asgService);
@@ -757,7 +769,7 @@ class AutoScalingReconcilerTest {
@Test
void pendingStoppedInstanceIsStillPrunedAsStale() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);
@@ -782,7 +794,7 @@ class AutoScalingReconcilerTest {
@Test
void repeatedTerminationFailureIsRecordedOnceWhenOtherActivitiesFollowIt() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
ElbV2Service elbV2Service = mock(ElbV2Service.class);
stubActivityRecording(asgService);
@@ -812,7 +824,7 @@ class AutoScalingReconcilerTest {
@Test
void terminationFailureWithADifferentErrorIsRecordedAgain() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
stubActivityRecording(asgService);
when(asgService.describeScalingActivities("us-east-1", "app-asg"))
@@ -841,7 +853,7 @@ class AutoScalingReconcilerTest {
@Test
void failedActivityStatusMessageIsCappedAt255Characters() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
stubActivityRecording(asgService);
AutoScalingReconciler reconciler =
@@ -862,7 +874,7 @@ class AutoScalingReconcilerTest {
@Test
void failedActivityWithoutAnErrorMessageNamesTheException() {
- AutoScalingService asgService = mock(AutoScalingService.class);
+ AutoScalingService asgService = scalingService();
Ec2Service ec2Service = mock(Ec2Service.class);
stubActivityRecording(asgService);
AutoScalingReconciler reconciler =