Light, fluffy, and always free - The AWS Local Emulator alternative
fix(autoscaling): pause suspended capacity adjustments (#5545)
David Phillips41517f3c2ffb
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingCapacitySuspensionTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingCapacitySuspensionTest.javanew file mode 100644index 0000000000..99829d7303--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingCapacitySuspensionTest.java@@ -0,0 +1,49 @@+package com.floci.test;++import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.autoscaling.AutoScalingClient;+import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup;+import software.amazon.awssdk.services.autoscaling.model.SuspendedProcess;++import java.util.List;++import static org.assertj.core.api.Assertions.assertThat;++@DisplayName("Auto Scaling capacity process suspension")+class AutoScalingCapacitySuspensionTest {+ @Test+ @DisplayName("Launch and Terminate suspension round trips retain desired capacity")+ void suspendAndResumeRoundTrip() {+ String name = TestFixtures.uniqueName("capacity-pause");+ try (AutoScalingClient asg = TestFixtures.autoScalingClient()) {+ asg.createLaunchConfiguration(r -> r.launchConfigurationName(name)+ .imageId("ami-amazonlinux2023").instanceType("t3.micro"));+ try {+ asg.createAutoScalingGroup(r -> r.autoScalingGroupName(name).launchConfigurationName(name)+ .minSize(0).maxSize(1).desiredCapacity(0).availabilityZones("us-east-1a"));+ try {+ for (String process : List.of("Launch", "Terminate")) {+ asg.suspendProcesses(r -> r.autoScalingGroupName(name).scalingProcesses(process));+ AutoScalingGroup paused = group(asg, name);+ assertThat(paused.suspendedProcesses()).extracting(SuspendedProcess::processName)+ .containsExactly(process);+ assertThat(paused.desiredCapacity()).isZero();+ asg.resumeProcesses(r -> r.autoScalingGroupName(name).scalingProcesses(process));+ AutoScalingGroup resumed = group(asg, name);+ assertThat(resumed.suspendedProcesses()).isEmpty();+ assertThat(resumed.desiredCapacity()).isZero();+ }+ } finally {+ asg.deleteAutoScalingGroup(r -> r.autoScalingGroupName(name).forceDelete(true));+ }+ } finally {+ asg.deleteLaunchConfiguration(r -> r.launchConfigurationName(name));+ }+ }+ }++ private static AutoScalingGroup group(AutoScalingClient asg, String name) {+ return asg.describeAutoScalingGroups(r -> r.autoScalingGroupNames(name)).autoScalingGroups().get(0);+ }+}diff --git a/docs/services/autoscaling.md b/docs/services/autoscaling.mdindex da544e6ba3..e869b63914 100644--- a/docs/services/autoscaling.md+++ b/docs/services/autoscaling.md@@ -129,6 +129,19 @@ Floci runs a background reconciler (10 s fixed rate) that keeps each group's InS- **Scale-in**: selects InService instances not protected from scale-in, deregisters them from target groups, then calls `TerminateInstances`.- Activity records are written on each scale-out and scale-in event.+`SuspendProcesses` pauses desired-capacity scale-out for `Launch` and scale-in for+`Terminate`. The desired count remains stored, and `ResumeProcesses` allows the+next reconciliation pass to continue toward it. Omitting process names suspends+or resumes all processes. Suspending another process does not pause these capacity+adjustments. Suspension and resumption wait for any desired-capacity adjustment already+in progress for that group, so a completed suspension cannot be bypassed by an earlier+reconciliation check.++Suspension does not yet pause instance-refresh selection or replacement cleanup.+Explicitly requested termination and stale-instance cleanup continue independently.+Other suspended process names are stored and returned but do not pause their+associated activities.+## Launch Source CompatibilityAuto Scaling groups preserve either a launch configuration, a top-level launch template, or a `MixedInstancesPolicy`. These launch sources are mutually exclusive in create/update requests. When a mixed instances policy is supplied, Floci stores and returns:diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.javaindex 869418f3a7..691ffd2065 100644--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java@@ -129,14 +129,16 @@ public class AutoScalingReconciler {removeOrphanedTargetRegistrations(asg);promoteReadyInstances(asg);- long activeCapacity = activeCapacity(asg);- int desired = asg.getDesiredCapacity();-- if (activeCapacity < desired) {- scaleOut(asg, (int) (desired - activeCapacity));- } else if (activeCapacity > desired) {- scaleIn(asg, (int) (activeCapacity - desired));- }+ asgService.adjustCapacityIfPresent(asg, () -> {+ long activeCapacity = activeCapacity(asg);+ int desired = asg.getDesiredCapacity();++ if (activeCapacity < desired && !asg.getSuspendedProcesses().contains("Launch")) {+ scaleOut(asg, (int) (desired - activeCapacity));+ } else if (activeCapacity > desired && !asg.getSuspendedProcesses().contains("Terminate")) {+ scaleIn(asg, (int) (activeCapacity - desired));+ }+ });if (asgService.saveAutoScalingGroupIfPresent(asg)) {asgService.completeInstanceRefreshIfSettled(asg.getRegion(), asg.getAutoScalingGroupName());}diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.javaindex 2b9bd9387c..dca198c488 100644--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java@@ -493,8 +493,17 @@ public class AutoScalingService {}}+ void adjustCapacityIfPresent(AutoScalingGroup asg, Runnable adjustment) {+ String key = asgKey(asg.getRegion(), asg.getAutoScalingGroupName());+ synchronized (lockFor(key)) {+ if (groups.get(key) == asg) {+ adjustment.run();+ }+ }+ }+private Object lockFor(String key) {- return groupLocks.computeIfAbsent(key, ignored -> new Object());+ return groupLocks.computeIfAbsent(regionResolver.getAccountId() + "::" + key, ignored -> new Object());}Set<String> autoScalingGroupAccountIds() {@@ -544,36 +553,38 @@ public class AutoScalingService {return requireGroup(region, resourceId);}- // Real AWS suspends the named scaling processes ("Launch", "Terminate", "HealthCheck", ...)- // so they stop running until resumed; an empty list suspends all of them. Floci has no- // scaling loop for these processes to pause, so this only needs to record which processes- // are suspended for DescribeAutoScalingGroups to report back - the same "accept and- // remember" shape as terminationPolicies. The AWS provider calls this unconditionally- // around ASG creation whenever wait_for_capacity_timeout is non-zero (the default), so an- // unimplemented SuspendProcesses fails ASG creation outright, not just a feature gap.+ // An empty list suspends all processes. The reconciler honors Launch and Terminate+ // for desired-capacity adjustments; other names remain stored state. Refresh selection+ // and cleanup are separate paths and do not yet honor suspended processes.private static final List<String> ALL_SCALING_PROCESSES = List.of("Launch", "Terminate", "AddToLoadBalancer", "AlarmNotification", "AZRebalance","HealthCheck", "InstanceRefresh", "ReplaceUnhealthy", "ScheduledActions");public void suspendProcesses(String region, String name, List<String> processes) {- AutoScalingGroup asg = requireGroup(region, name);- List<String> toSuspend = processes.isEmpty() ? ALL_SCALING_PROCESSES : processes;- Set<String> suspended = new LinkedHashSet<>(asg.getSuspendedProcesses());- suspended.addAll(toSuspend);- asg.setSuspendedProcesses(new ArrayList<>(suspended));- groups.put(asgKey(region, name), asg);+ String key = asgKey(region, name);+ synchronized (lockFor(key)) {+ AutoScalingGroup asg = requireGroup(region, name);+ List<String> toSuspend = processes.isEmpty() ? ALL_SCALING_PROCESSES : processes;+ Set<String> suspended = new LinkedHashSet<>(asg.getSuspendedProcesses());+ suspended.addAll(toSuspend);+ asg.setSuspendedProcesses(new ArrayList<>(suspended));+ groups.put(key, asg);+ }}public void resumeProcesses(String region, String name, List<String> processes) {- AutoScalingGroup asg = requireGroup(region, name);- if (processes.isEmpty()) {- asg.setSuspendedProcesses(new ArrayList<>());- } else {- List<String> remaining = new ArrayList<>(asg.getSuspendedProcesses());- remaining.removeAll(processes);- asg.setSuspendedProcesses(remaining);+ String key = asgKey(region, name);+ synchronized (lockFor(key)) {+ AutoScalingGroup asg = requireGroup(region, name);+ if (processes.isEmpty()) {+ asg.setSuspendedProcesses(new ArrayList<>());+ } else {+ List<String> remaining = new ArrayList<>(asg.getSuspendedProcesses());+ remaining.removeAll(processes);+ asg.setSuspendedProcesses(remaining);+ }+ groups.put(key, asg);}- groups.put(asgKey(region, name), asg);}// ── Instance management ────────────────────────────────────────────────────diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.javanew file mode 100644index 0000000000..48faa51c1a--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java@@ -0,0 +1,79 @@+package io.github.hectorvent.floci.services.autoscaling;++import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.autoscaling.model.AsgOptionalFields;+import io.github.hectorvent.floci.services.autoscaling.model.AutoScalingGroup;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;++import java.util.List;+import java.util.Map;+import java.util.UUID;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertTrue;++@QuarkusTest+class AutoScalingCapacityAccountLockIntegrationTest {+ private static final String ACCOUNT = "111111111111";+ private static final String REGION = "us-east-1";++ @Inject+ AutoScalingService service;++ @ParameterizedTest+ @CsvSource({"222222222222,us-east-1", "111111111111,us-west-2"})+ void unrelatedAccountOrRegionDoesNotWaitForCapacityOperation(String otherAccount, String otherRegion)+ throws Exception {+ String name = "capacity-lock-" + UUID.randomUUID();+ CountDownLatch entered = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ ExecutorService workers = Executors.newFixedThreadPool(2);+ try {+ AutoScalingGroup group = RequestScopes.callAs(ACCOUNT, REGION, () -> create(REGION, name));+ RequestScopes.runAs(otherAccount, otherRegion, () -> create(otherRegion, name));+ Future<?> adjustment = workers.submit(() -> RequestScopes.runAs(ACCOUNT, REGION,+ () -> service.adjustCapacityIfPresent(group, () -> {+ entered.countDown();+ try {+ assertTrue(release.await(10, TimeUnit.SECONDS));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ })));+ assertTrue(entered.await(5, TimeUnit.SECONDS));+ Future<?> suspension = workers.submit(() -> RequestScopes.runAs(otherAccount, otherRegion,+ () -> service.suspendProcesses(otherRegion, name, List.of("Launch"))));+ suspension.get(2, TimeUnit.SECONDS);+ RequestScopes.runAs(otherAccount, otherRegion, () -> assertTrue(service+ .describeAutoScalingGroups(otherRegion, List.of(name)).getFirst()+ .getSuspendedProcesses().contains("Launch")));+ release.countDown();+ adjustment.get(5, TimeUnit.SECONDS);+ } finally {+ release.countDown();+ workers.shutdownNow();+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));+ try {+ RequestScopes.runAs(ACCOUNT, REGION, () -> service.deleteAutoScalingGroup(REGION, name, true));+ } finally {+ RequestScopes.runAs(otherAccount, otherRegion,+ () -> service.deleteAutoScalingGroup(otherRegion, name, true));+ }+ }+ }++ private AutoScalingGroup create(String region, String name) {+ service.createAutoScalingGroup(region, name, "unused-capacity-lock-source", null, null, null, null,+ 0, 1, 0, 300, List.of(region + "a"), List.of(), List.of(), List.of(), "EC2", 0,+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());+ return service.describeAutoScalingGroups(region, List.of(name)).getFirst();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionIntegrationTest.javanew file mode 100644index 0000000000..a3eee18abb--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionIntegrationTest.java@@ -0,0 +1,111 @@+package io.github.hectorvent.floci.services.autoscaling;++import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import io.restassured.specification.RequestSpecification;+import jakarta.inject.Inject;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;++@QuarkusTest+class AutoScalingCapacitySuspensionIntegrationTest {+ private static final String GROUP_PATH = "DescribeAutoScalingGroupsResponse."+ + "DescribeAutoScalingGroupsResult.AutoScalingGroups.member";++ @Inject+ AutoScalingReconciler reconciler;++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void suspendAndResumePreserveDesiredCapacityAndControlEc2Instances(boolean allProcesses) {+ String name = "capacity-pause-" + UUID.randomUUID().toString().substring(0, 8);+ request("CreateLaunchConfiguration")+ .formParam("LaunchConfigurationName", name)+ .formParam("ImageId", "ami-amazonlinux2023").formParam("InstanceType", "t3.micro")+ .post("/").then().statusCode(200);+ try {+ request("CreateAutoScalingGroup").formParam("AutoScalingGroupName", name)+ .formParam("LaunchConfigurationName", name)+ .formParam("MinSize", 0).formParam("MaxSize", 1).formParam("DesiredCapacity", 0)+ .formParam("AvailabilityZones.member.1", "us-east-1a")+ .post("/").then().statusCode(200);+ try {+ process("SuspendProcesses", name, allProcesses, "Launch");+ setDesired(name, 1);+ assertPaused(name, 1, 0);+ process("ResumeProcesses", name, allProcesses, "Launch");+ reconciler.reconcileAll();+ reconciler.reconcileAll();+ assertCapacity(name, 1, 1);+ String instanceId = describe(name).xmlPath().getString(GROUP_PATH + ".Instances.member.InstanceId");++ process("SuspendProcesses", name, allProcesses, "Terminate");+ setDesired(name, 0);+ assertPaused(name, 0, 1);+ assertEquals("running", instanceState(instanceId));+ process("ResumeProcesses", name, allProcesses, "Terminate");+ reconciler.reconcileAll();+ reconciler.reconcileAll();+ assertCapacity(name, 0, 0);+ assertEquals("terminated", instanceState(instanceId));+ } finally {+ request("DeleteAutoScalingGroup").formParam("AutoScalingGroupName", name)+ .formParam("ForceDelete", true).post("/").then().statusCode(200);+ }+ } finally {+ request("DeleteLaunchConfiguration").formParam("LaunchConfigurationName", name)+ .post("/").then().statusCode(200);+ }+ }++ private void assertPaused(String name, int desired, int instances) {+ reconciler.reconcileAll();+ reconciler.reconcileAll();+ assertCapacity(name, desired, instances);+ }++ private static void assertCapacity(String name, int desired, int instances) {+ Response response = describe(name);+ assertEquals(desired, response.xmlPath().getInt(GROUP_PATH + ".DesiredCapacity"));+ assertEquals(instances, response.xmlPath().getList(GROUP_PATH + ".Instances.member").size());+ }++ private static Response describe(String name) {+ return request("DescribeAutoScalingGroups").formParam("AutoScalingGroupNames.member.1", name)+ .post("/").then().statusCode(200).extract().response();+ }++ private static String instanceState(String id) {+ return given().header("Authorization", auth("ec2"))+ .formParam("Action", "DescribeInstances").formParam("InstanceId.1", id)+ .post("/").then().statusCode(200).extract().xmlPath()+ .getString("DescribeInstancesResponse.reservationSet.item.instancesSet.item.instanceState.name");+ }++ private static void setDesired(String name, int desired) {+ request("SetDesiredCapacity").formParam("AutoScalingGroupName", name)+ .formParam("DesiredCapacity", desired).post("/").then().statusCode(200);+ }++ private static void process(String action, String name, boolean allProcesses, String process) {+ RequestSpecification request = request(action).formParam("AutoScalingGroupName", name);+ if (!allProcesses) {+ request.formParam("ScalingProcesses.member.1", process);+ }+ request.post("/").then().statusCode(200);+ }++ private static RequestSpecification request(String action) {+ return given().header("Authorization", auth("autoscaling")).formParam("Action", action);+ }++ private static String auth(String service) {+ return "AWS4-HMAC-SHA256 Credential=AKID/20260215/us-east-1/" + service+ + "/aws4_request, SignedHeaders=host, Signature=abc";+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionTest.javanew file mode 100644index 0000000000..20ce95a78b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacitySuspensionTest.java@@ -0,0 +1,202 @@+package io.github.hectorvent.floci.services.autoscaling;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.autoscaling.model.AsgInstance;+import io.github.hectorvent.floci.services.autoscaling.model.AsgOptionalFields;+import io.github.hectorvent.floci.services.autoscaling.model.AutoScalingGroup;+import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.services.ec2.model.Instance;+import io.github.hectorvent.floci.services.ec2.model.InstanceState;+import io.github.hectorvent.floci.services.ec2.model.LaunchTemplate;+import io.github.hectorvent.floci.services.ec2.model.Reservation;+import io.github.hectorvent.floci.services.elbv2.ElbV2Service;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;+import org.junit.jupiter.params.provider.ValueSource;++import java.time.Duration;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import static org.awaitility.Awaitility.await;+import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.*;++class AutoScalingCapacitySuspensionTest {+ private static final String REGION = "us-east-1";+ private AutoScalingService service;+ private Ec2Service ec2;+ private AutoScalingReconciler reconciler;+ private AutoScalingGroup group;++ @BeforeEach+ void setUp() {+ service = new AutoScalingService();+ service.regionResolver = new RegionResolver(REGION, "000000000000");+ service.createAutoScalingGroup(REGION, "suspended-group", null, "lt-original", null, "1", null,+ 0, 3, 1, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());+ group = service.describeAutoScalingGroups(REGION, List.of("suspended-group")).getFirst();+ AsgInstance existing = new AsgInstance();+ existing.setInstanceId("i-original");+ existing.setLifecycleState("InService");+ existing.setHealthStatus("Healthy");+ existing.setLaunchTemplateId("lt-original");+ existing.setLaunchTemplateVersion("1");+ group.getInstances().add(existing);+ service.saveAutoScalingGroupIfPresent(group);+ ec2 = mock(Ec2Service.class);+ service.ec2Service = ec2;+ reconciler = new AutoScalingReconciler(service, ec2, mock(ElbV2Service.class));+ when(ec2.isInstanceContainerRunning(anyString())).thenReturn(true);+ when(ec2.describeInstances(eq(REGION), anyList(), isNull()))+ .thenReturn(List.of(reservation("i-original")));+ LaunchTemplate template = new LaunchTemplate();+ template.setLaunchTemplateId("lt-original");+ template.setLatestVersionNumber("1");+ template.getData().setImageId("ami-original");+ template.getData().setInstanceType("t3.micro");+ when(ec2.describeLaunchTemplates(eq(REGION), anyList(), anyList(), anyMap())).thenReturn(List.of(template));+ when(ec2.describeLaunchTemplateVersions(eq(REGION), eq("lt-original"), isNull(), anyList()))+ .thenReturn(List.of(template));+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),+ anyList(), any(), any(), any())).thenReturn(reservation("i-replacement"));+ }++ @ParameterizedTest+ @CsvSource({"Launch,2,true", "Terminate,0,true", "ALL,2,true", "ALL,0,true",+ "Launch,0,false", "Terminate,2,false", "HealthCheck,2,false", "HealthCheck,0,false"})+ void suspensionPausesOnlyItsCapacityDirectionAndResumeContinues(String process, int desired, boolean paused) {+ group.setDesiredCapacity(desired);+ List<String> processes = "ALL".equals(process) ? List.of() : List.of(process);+ service.suspendProcesses(REGION, "suspended-group", processes);+ reconciler.reconcile(group);+ assertEquals(desired, group.getDesiredCapacity());+ if (paused) {+ verify(ec2, never()).terminateInstances(anyString(), anyList());+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any());+ assertEquals(List.of("i-original"), group.getInstances().stream()+ .map(AsgInstance::getInstanceId).toList());+ service.resumeProcesses(REGION, "suspended-group", processes);+ reconciler.reconcile(group);+ }+ assertEquals(desired, group.getInstances().size());+ if (desired == 0) {+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ } else {+ verify(ec2).runInstances(eq(REGION), any(), any(), eq(1), eq(1), any(), anyList(),+ any(), any(), anyList(), any(), any(), any());+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"Terminate", "InstanceRefresh"})+ void suspensionDoesNotBlockExplicitManualTermination(String process) {+ service.suspendProcesses(REGION, "suspended-group", List.of(process));+ service.terminateInstanceInAutoScalingGroup(REGION, "i-original", true);+ reconciler.reconcile(group);+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ assertTrue(group.getInstances().isEmpty());+ }++ @ParameterizedTest+ @CsvSource({"Launch,false", "Launch,true", "Terminate,false", "Terminate,true"})+ void suspensionWaitsForCapacityOperationAlreadyPastItsGuard(String process, boolean allProcesses)+ throws Exception {+ CountDownLatch operationEntered = new CountDownLatch(1);+ CountDownLatch releaseOperation = new CountDownLatch(1);+ AtomicReference<Thread> suspensionThread = new AtomicReference<>();+ if ("Launch".equals(process)) {+ group.setDesiredCapacity(2);+ doAnswer(invocation -> {+ operationEntered.countDown();+ assertTrue(releaseOperation.await(10, TimeUnit.SECONDS));+ LaunchTemplate template = new LaunchTemplate();+ template.setLaunchTemplateId("lt-original");+ template.setLatestVersionNumber("1");+ template.getData().setImageId("ami-original");+ template.getData().setInstanceType("t3.micro");+ return List.of(template);+ }).when(ec2).describeLaunchTemplates(eq(REGION), anyList(), anyList(), anyMap());+ } else {+ group.setDesiredCapacity(0);+ AsgInstance existing = spy(group.getInstances().getFirst());+ group.getInstances().set(0, existing);+ doAnswer(invocation -> {+ operationEntered.countDown();+ assertTrue(releaseOperation.await(10, TimeUnit.SECONDS));+ return false;+ }).when(existing).isProtectedFromScaleIn();+ }+ service.createAutoScalingGroup(REGION, "unrelated-group", null, "lt-original", null, "1", null,+ 0, 3, 0, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());+ ExecutorService workers = Executors.newFixedThreadPool(3);+ try {+ Future<?> reconciliation = workers.submit(() -> reconciler.reconcile(group));+ assertTrue(operationEntered.await(5, TimeUnit.SECONDS));+ Future<?> suspension = workers.submit(() -> {+ suspensionThread.set(Thread.currentThread());+ service.suspendProcesses(REGION, "suspended-group", allProcesses ? List.of() : List.of(process));+ });+ await().atMost(Duration.ofSeconds(5)).until(() -> suspension.isDone()+ || suspensionThread.get() != null && suspensionThread.get().getState() == Thread.State.BLOCKED);+ assertFalse(suspension.isDone(), "Suspension must wait for the capacity operation it raced");+ workers.submit(() -> service.suspendProcesses(REGION, "unrelated-group", List.of(process)))+ .get(5, TimeUnit.SECONDS);+ releaseOperation.countDown();+ reconciliation.get(5, TimeUnit.SECONDS);+ suspension.get(5, TimeUnit.SECONDS);+ int completedCapacity = group.getInstances().size();+ group.setDesiredCapacity("Launch".equals(process) ? completedCapacity + 1 : 0);+ clearInvocations(ec2);+ reconciler.reconcile(group);+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any());+ verify(ec2, never()).terminateInstances(anyString(), anyList());+ } finally {+ releaseOperation.countDown();+ workers.shutdownNow();+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));+ }+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void staleGroupCannotAdjustCapacityAfterDeletionOrRecreation(boolean recreate) {+ group.setDesiredCapacity(2);+ service.deleteAutoScalingGroup(REGION, "suspended-group", true);+ if (recreate) {+ service.createAutoScalingGroup(REGION, "suspended-group", null, "lt-original", null, "1", null,+ 0, 3, 0, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());+ }+ clearInvocations(ec2);+ reconciler.reconcile(group);+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any());+ if (recreate) {+ assertTrue(service.describeAutoScalingGroups(REGION, List.of("suspended-group"))+ .getFirst().getInstances().isEmpty());+ } else {+ assertTrue(service.describeAutoScalingGroups(REGION, List.of("suspended-group")).isEmpty());+ }+ }++ private static Reservation reservation(String id) {+ Instance instance = new Instance();+ instance.setInstanceId(id);+ instance.setState(InstanceState.running());+ Reservation reservation = new Reservation();+ reservation.setInstances(List.of(instance));+ return reservation;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.javaindex 45626d18ac..126f954d5a 100644--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java@@ -29,8 +29,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.Mockito.any;import static org.mockito.Mockito.anyList;import static org.mockito.Mockito.anyString;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.eq;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;@@ -40,6 +42,16 @@ import static org.mockito.Mockito.when;class AutoScalingReconcilerTest {+ private static AutoScalingService scalingService() {+ AutoScalingService service = mock(AutoScalingService.class);+ doAnswer(invocation -> {+ Runnable adjustment = invocation.getArgument(1);+ adjustment.run();+ return null;+ }).when(service).adjustCapacityIfPresent(any(), any());+ return service;+ }+@Testvoid pendingInstancesCountAsActiveCapacity() {AutoScalingGroup asg = new AutoScalingGroup();@@ -54,7 +66,7 @@ class AutoScalingReconcilerTest {@Testvoid stopTerminatesReconcilerThread() throws Exception {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -73,7 +85,7 @@ class AutoScalingReconcilerTest {@Testvoid reconcileCompletesSettledInstanceRefreshes() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -90,7 +102,7 @@ class AutoScalingReconcilerTest {@Testvoid scaleOutUsesRequestedLaunchTemplateVersionData() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -154,7 +166,7 @@ class AutoScalingReconcilerTest {@Testvoid scaleOutTerminatesInstancesWhenGroupDisappears() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -178,7 +190,7 @@ class AutoScalingReconcilerTest {@Testvoid scaleOutPersistsInstancesWhenGroupStillExists() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -211,7 +223,7 @@ class AutoScalingReconcilerTest {}private void assertLaunchConfigurationForwards(Boolean associatePublicIp) {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -247,7 +259,7 @@ class AutoScalingReconcilerTest {@Testvoid scaleOutStoresResolvedLaunchTemplateVersionForAliases() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -284,7 +296,7 @@ class AutoScalingReconcilerTest {@Testvoid scaleOutUsesMixedInstancesLaunchTemplateSpecification() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -341,7 +353,7 @@ class AutoScalingReconcilerTest {@Testvoid reconcileDeregistersTargetsThatAreNotActiveAsgInstances() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -370,7 +382,7 @@ class AutoScalingReconcilerTest {@Testvoid reconcileKeepsPendingInstancesWhileContainerLaunchIsStillInFlight() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -411,7 +423,7 @@ class AutoScalingReconcilerTest {@Testvoid reconcilePrunesPendingInstancesWhenEc2InstanceIsTerminal() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -442,7 +454,7 @@ class AutoScalingReconcilerTest {@Testvoid reconcileFailsActiveSsmInvocationsBeforePruningStaleInstance() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);SsmCommandService ssmCommandService = mock(SsmCommandService.class);@@ -472,7 +484,7 @@ class AutoScalingReconcilerTest {@Testvoid reconcileDeregistersStaleInstanceFromClassicLoadBalancerBeforePruning() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);ElbClassicService elbClassicService = mock(ElbClassicService.class);@@ -495,7 +507,7 @@ class AutoScalingReconcilerTest {@Testvoid reconcilePromotesPendingAsgInstanceWhenEc2InstanceIsRunning() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -529,7 +541,7 @@ class AutoScalingReconcilerTest {@Testvoid deletedLaunchTemplateDoesNotAbortTheReconcilePass() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);AutoScalingGroup asg = asgWhoseLaunchTemplateLookupFails(ec2Service,new AwsException("InvalidLaunchTemplateName.NotFoundException",@@ -546,7 +558,7 @@ class AutoScalingReconcilerTest {@Testvoid otherLaunchTemplateErrorsSurfaceRatherThanSilentlySkippingScaling() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);AutoScalingGroup asg = asgWhoseLaunchTemplateLookupFails(ec2Service,new AwsException("RequestLimitExceeded", "Request limit exceeded.", 503));@@ -560,7 +572,7 @@ class AutoScalingReconcilerTest {@Testvoid scaleInTerminationFailureKeepsMembershipAndRecordsFailedActivity() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);stubActivityRecording(asgService);@@ -592,7 +604,7 @@ class AutoScalingReconcilerTest {@Testvoid repeatedTerminationFailureIsRecordedOnce() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);stubActivityRecording(asgService);@@ -620,7 +632,7 @@ class AutoScalingReconcilerTest {@Testvoid scaleInTerminatesAndRemovesTheInstanceOnSuccess() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -648,7 +660,7 @@ class AutoScalingReconcilerTest {@Testvoid terminatingInstanceTerminationFailureKeepsMembershipAndRecordsFailedActivity() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);stubActivityRecording(asgService);@@ -684,7 +696,7 @@ class AutoScalingReconcilerTest {@Testvoid terminatingInstanceWithMissingEc2RecordIsPrunedAsStale() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -722,7 +734,7 @@ class AutoScalingReconcilerTest {@Testvoid terminatingStoppedInstanceIsRetriedRatherThanPrunedAsStale() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);stubActivityRecording(asgService);@@ -757,7 +769,7 @@ class AutoScalingReconcilerTest {@Testvoid pendingStoppedInstanceIsStillPrunedAsStale() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);AutoScalingReconciler reconciler = new AutoScalingReconciler(asgService, ec2Service, elbV2Service);@@ -782,7 +794,7 @@ class AutoScalingReconcilerTest {@Testvoid repeatedTerminationFailureIsRecordedOnceWhenOtherActivitiesFollowIt() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);ElbV2Service elbV2Service = mock(ElbV2Service.class);stubActivityRecording(asgService);@@ -812,7 +824,7 @@ class AutoScalingReconcilerTest {@Testvoid terminationFailureWithADifferentErrorIsRecordedAgain() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);stubActivityRecording(asgService);when(asgService.describeScalingActivities("us-east-1", "app-asg"))@@ -841,7 +853,7 @@ class AutoScalingReconcilerTest {@Testvoid failedActivityStatusMessageIsCappedAt255Characters() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);stubActivityRecording(asgService);AutoScalingReconciler reconciler =@@ -862,7 +874,7 @@ class AutoScalingReconcilerTest {@Testvoid failedActivityWithoutAnErrorMessageNamesTheException() {- AutoScalingService asgService = mock(AutoScalingService.class);+ AutoScalingService asgService = scalingService();Ec2Service ec2Service = mock(Ec2Service.class);stubActivityRecording(asgService);AutoScalingReconciler reconciler =