Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(autoscaling): preserve paused refresh ownership (#5668)

David Phillipsb395560fe229
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingRefreshOwnershipTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingRefreshOwnershipTest.java
new file mode 100644
index 0000000000..7277936b75
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingRefreshOwnershipTest.java
@@ -0,0 +1,95 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.services.autoscaling.AutoScalingClient;
+import software.amazon.awssdk.services.autoscaling.model.AutoScalingException;
+import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup;
+import software.amazon.awssdk.services.autoscaling.model.InstanceRefresh;
+import software.amazon.awssdk.services.autoscaling.model.ResumeProcessesRequest;
+import software.amazon.awssdk.services.autoscaling.model.SuspendProcessesRequest;
+import software.amazon.awssdk.services.ec2.Ec2Client;
+import software.amazon.awssdk.services.ec2.model.InstanceType;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class AutoScalingRefreshOwnershipTest {
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ @DisplayName("A paused refresh exposes its selected work and resumes through the SDK")
+ void pausedRefreshMetadataAndResumeRoundTrip(boolean allProcesses) {
+ String name = TestFixtures.uniqueName("refresh-ownership");
+ try (AutoScalingClient autoScaling = TestFixtures.autoScalingClient();
+ Ec2Client ec2 = TestFixtures.ec2Client()) {
+ autoScaling.createLaunchConfiguration(request -> request.launchConfigurationName(name)
+ .imageId("ami-amazonlinux2023").instanceType("t3.micro"));
+ try {
+ String instanceId = ec2.runInstances(request -> request.imageId("ami-amazonlinux2023")
+ .instanceType(InstanceType.T3_MICRO).minCount(1).maxCount(1))
+ .instances().get(0).instanceId();
+ try {
+ autoScaling.createAutoScalingGroup(request -> request.autoScalingGroupName(name)
+ .launchConfigurationName(name).minSize(0).maxSize(1).desiredCapacity(0)
+ .availabilityZones("us-east-1a"));
+ try {
+ SuspendProcessesRequest.Builder suspend = SuspendProcessesRequest.builder()
+ .autoScalingGroupName(name);
+ if (!allProcesses) {
+ suspend.scalingProcesses("InstanceRefresh");
+ }
+ autoScaling.suspendProcesses(suspend.build());
+ autoScaling.attachInstances(request -> request.autoScalingGroupName(name)
+ .instanceIds(instanceId));
+ String refreshId = autoScaling.startInstanceRefresh(request ->
+ request.autoScalingGroupName(name)).instanceRefreshId();
+ assertThat(refreshId).isNotBlank();
+ InstanceRefresh refresh = autoScaling.describeInstanceRefreshes(request ->
+ request.autoScalingGroupName(name).instanceRefreshIds(refreshId))
+ .instanceRefreshes().get(0);
+ assertThat(refresh.instanceRefreshId()).isEqualTo(refreshId);
+ assertThat(refresh.autoScalingGroupName()).isEqualTo(name);
+ assertThat(refresh.statusAsString()).isEqualTo("InProgress");
+ assertThat(refresh.startTime()).isNotNull();
+ assertThat(refresh.instancesToUpdate()).isEqualTo(1);
+ assertThat(refresh.percentageComplete()).isZero();
+ AutoScalingGroup group = autoScaling.describeAutoScalingGroups(request ->
+ request.autoScalingGroupNames(name)).autoScalingGroups().get(0);
+ assertThat(group.desiredCapacity()).isEqualTo(1);
+ assertThat(group.instances()).hasSize(1);
+ assertThat(group.instances().get(0).instanceId()).isEqualTo(instanceId);
+ assertThat(group.suspendedProcesses()).extracting(process -> process.processName())
+ .contains("InstanceRefresh");
+ assertThatThrownBy(() -> autoScaling.startInstanceRefresh(request ->
+ request.autoScalingGroupName(name)))
+ .isInstanceOfSatisfying(AutoScalingException.class, error -> {
+ assertThat(error.statusCode()).isEqualTo(400);
+ assertThat(error.awsErrorDetails().errorCode()).isEqualTo("InstanceRefreshInProgress");
+ });
+ assertThat(autoScaling.describeInstanceRefreshes(request ->
+ request.autoScalingGroupName(name)).instanceRefreshes())
+ .extracting(InstanceRefresh::instanceRefreshId).containsExactly(refreshId);
+ ResumeProcessesRequest.Builder resume = ResumeProcessesRequest.builder()
+ .autoScalingGroupName(name);
+ if (!allProcesses) {
+ resume.scalingProcesses("InstanceRefresh");
+ }
+ autoScaling.resumeProcesses(resume.build());
+ assertThat(autoScaling.describeAutoScalingGroups(request ->
+ request.autoScalingGroupNames(name)).autoScalingGroups().get(0).suspendedProcesses())
+ .isEmpty();
+ } finally {
+ autoScaling.deleteAutoScalingGroup(request -> request.autoScalingGroupName(name)
+ .forceDelete(true));
+ }
+ } finally {
+ ec2.terminateInstances(request -> request.instanceIds(instanceId));
+ }
+ } finally {
+ autoScaling.deleteLaunchConfiguration(request -> request.launchConfigurationName(name));
+ }
+ }
+ }
+}
diff --git a/docs/services/autoscaling.md b/docs/services/autoscaling.md
index bd92cb1b59..c2c0fd3956 100644
--- a/docs/services/autoscaling.md
+++ b/docs/services/autoscaling.md
@@ -49,8 +49,12 @@ after a launch error and rebalancing without a capacity increase are not impleme
AWS launch errors appear as failed scaling activities after preserving partial
launches. Repeated identical failures reuse the recorded activity, following the
-scale-in behavior, and do not complete an instance refresh. Unexpected runtime,
-save and cleanup failures still propagate.
+scale-in behavior, and do not complete an instance refresh. Unexpected runtime
+failures in ordinary scale-out, and save and cleanup failures, still propagate.
+Refresh launch failures retain the original replacement attempt and appear as
+failed scaling activities while other replacements can continue. Refresh and
+ordinary launches share the same launch settings, including template block-device
+mappings; accepted refresh launches keep their recorded source identity.
For groups configured only with Availability Zones, Floci preserves its unpinned
EC2 launch behavior and counts each member under the requested zone. This permits
@@ -69,6 +73,27 @@ EC2 instance can use a default subnet in a different zone.
| `SuspendProcesses` | Records suspended scaling processes (empty list suspends all); reported back by `DescribeAutoScalingGroups` |
| `ResumeProcesses` | Clears suspended processes (empty list resumes all) |
+### Instance Refresh Recovery
+
+New instance refreshes retain the selected instances and their replacement progress
+across persistent, hybrid, and WAL storage reloads. Suspending `InstanceRefresh`,
+`Launch`, or `Terminate` pauses refresh replacements; resuming those processes lets
+the refresh continue. A failed replacement launch remains part of the refresh, so
+ordinary desired-capacity reconciliation does not bypass the pause. Explicit
+`TerminateInstanceInAutoScalingGroup` requests still run, and independent capacity
+increases can proceed while only `InstanceRefresh` is suspended. Paused original
+instances keep their target-group registrations until replacement resumes.
+Explicitly detached instances leave refresh ownership and are not adopted again
+when the refresh resumes. Recovery of an accepted EC2 launch keeps the source
+identity recorded for that attempt even if the group configuration has changed.
+Older attempts without a recorded source use the available EC2 instance metadata
+without assigning the current launch template to the recovered instance.
+
+This does not implement rolling health-percentage limits, checkpoints, rollback,
+or cancellation. Refreshes saved by older versions without replacement ownership
+retain the previous reconciliation behavior. Storage reload coverage does not
+imply an atomic transaction spanning Auto Scaling and EC2 after a process crash.
+
### Load Balancer Attachment
| Operation | Notes |
@@ -155,8 +180,9 @@ adjustments. Suspension and resumption wait for any desired-capacity adjustment
in progress for that group, so a completed suspension cannot be bypassed by an earlier
reconciliation check.
-Suspension does not yet pause instance-refresh selection or replacement cleanup.
-Explicitly requested termination and stale-instance cleanup continue independently.
+New owned refresh replacements also pause for `InstanceRefresh`, `Launch`, or
+`Terminate`, as described above. Explicitly requested termination and stale-instance
+cleanup continue independently.
Other suspended process names are stored and returned but do not pause their
associated activities.
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
index c698b48f38..c5323150fe 100644
--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
@@ -4,6 +4,9 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.services.autoscaling.model.AsgInstance;
import io.github.hectorvent.floci.services.autoscaling.model.AutoScalingGroup;
+import io.github.hectorvent.floci.services.autoscaling.model.InstanceRefresh;
+import io.github.hectorvent.floci.services.autoscaling.model.InstanceRefreshLaunch;
+import io.github.hectorvent.floci.services.autoscaling.model.InstanceRefreshReplacement;
import io.github.hectorvent.floci.services.autoscaling.model.LaunchConfiguration;
import io.github.hectorvent.floci.services.autoscaling.model.MixedInstancesPolicy;
import io.github.hectorvent.floci.services.autoscaling.model.ScalingActivity;
@@ -130,17 +133,18 @@ public class AutoScalingReconciler {
}
public void reconcile(AutoScalingGroup asg) {
- removeTerminatingInstances(asg);
- removeStaleInstances(asg);
- removeOrphanedTargetRegistrations(asg);
- promoteReadyInstances(asg);
+ asgService.reconcileIfPresent(asg, () -> {
+ removeTerminatingInstances(asg);
+ removeStaleInstances(asg);
+ removeOrphanedTargetRegistrations(asg);
+ promoteReadyInstances(asg);
+ reconcileRefreshReplacements(asg);
- asgService.adjustCapacityIfPresent(asg, () -> {
long activeCapacity = activeCapacity(asg);
+ long reservedCapacity = reservedRefreshCapacity(asg);
int desired = asg.getDesiredCapacity();
-
- if (activeCapacity < desired && !asg.getSuspendedProcesses().contains("Launch")) {
- if (!scaleOut(asg, (int) (desired - activeCapacity))) {
+ if (activeCapacity + reservedCapacity < desired && !asg.getSuspendedProcesses().contains("Launch")) {
+ if (!scaleOut(asg, (int) (desired - activeCapacity - reservedCapacity))) {
return;
}
} else if (activeCapacity > desired && !asg.getSuspendedProcesses().contains("Terminate")) {
@@ -152,6 +156,76 @@ public class AutoScalingReconciler {
});
}
+ private static boolean refreshPaused(AutoScalingGroup asg) {
+ return asg.getSuspendedProcesses().contains("InstanceRefresh")
+ || asg.getSuspendedProcesses().contains("Launch")
+ || asg.getSuspendedProcesses().contains("Terminate");
+ }
+
+ private boolean pausedRefreshOwns(AutoScalingGroup asg, String instanceId) {
+ InstanceRefresh refresh = asgService.activeInstanceRefresh(asg);
+ return refreshPaused(asg) && refresh != null && refresh.getReplacements() != null
+ && refresh.getReplacements().stream()
+ .anyMatch(replacement -> instanceId.equals(replacement.getOriginalInstanceId()));
+ }
+
+ private long reservedRefreshCapacity(AutoScalingGroup asg) {
+ InstanceRefresh refresh = asgService.activeInstanceRefresh(asg);
+ if (refresh == null || refresh.getReplacements() == null) {
+ return 0;
+ }
+ return refresh.getReplacements().stream()
+ .filter(replacement -> !replacement.isComplete())
+ .filter(replacement -> asg.getInstances().stream().noneMatch(instance ->
+ instance.getInstanceId().equals(replacement.getReplacementInstanceId())
+ && isActiveLifecycleState(instance.getLifecycleState())))
+ .count();
+ }
+
+ private void reconcileRefreshReplacements(AutoScalingGroup asg) {
+ InstanceRefresh refresh = asgService.activeInstanceRefresh(asg);
+ if (refresh == null || refresh.getReplacements() == null || refreshPaused(asg)) {
+ return;
+ }
+ Map<String, List<Instance>> acceptedLaunches = null;
+ for (InstanceRefreshReplacement replacement : refresh.getReplacements()) {
+ if (replacement.isComplete()) {
+ continue;
+ }
+ AsgInstance launchedInstance = asg.getInstances().stream()
+ .filter(instance -> instance.getInstanceId().equals(replacement.getReplacementInstanceId()))
+ .findFirst().orElse(null);
+ if (launchedInstance != null) {
+ if ("InService".equals(launchedInstance.getLifecycleState())) {
+ replacement.setPhase("Complete");
+ asgService.saveInstanceRefresh(refresh);
+ }
+ continue;
+ }
+ if (asg.getInstances().stream().anyMatch(instance ->
+ instance.getInstanceId().equals(replacement.getOriginalInstanceId()))) {
+ continue;
+ }
+ if (acceptedLaunches == null) {
+ acceptedLaunches = refreshLaunches(asg);
+ }
+ List<Instance> existing = acceptedLaunches.getOrDefault(replacement.getLaunchClientToken(), List.of());
+ if (activeCapacity(asg) >= asg.getDesiredCapacity() && existing.isEmpty()) {
+ replacement.setPhase("NoLongerRequired");
+ asgService.saveInstanceRefresh(refresh);
+ continue;
+ }
+ replacement.setPhase("AwaitingReplacement");
+ asgService.saveInstanceRefresh(refresh);
+ List<String> launched = launchInstances(asg, refresh, replacement, existing, acceptedLaunches);
+ if (!launched.isEmpty()) {
+ replacement.setReplacementInstanceId(launched.getFirst());
+ replacement.setPhase("Warming");
+ asgService.saveInstanceRefresh(refresh);
+ }
+ }
+ }
+
static long activeCapacity(AutoScalingGroup asg) {
return asg.getInstances().stream()
.filter(i -> {
@@ -279,7 +353,8 @@ public class AutoScalingReconciler {
}
Set<String> activeInstanceIds = asg.getInstances().stream()
- .filter(instance -> isActiveLifecycleState(instance.getLifecycleState()))
+ .filter(instance -> isActiveLifecycleState(instance.getLifecycleState())
+ || pausedRefreshOwns(asg, instance.getInstanceId()))
.map(AsgInstance::getInstanceId)
.collect(Collectors.toSet());
@@ -308,6 +383,7 @@ public class AutoScalingReconciler {
private void removeTerminatingInstances(AutoScalingGroup asg) {
List<AsgInstance> terminatingInstances = asg.getInstances().stream()
.filter(instance -> "Terminating".equals(instance.getLifecycleState()))
+ .filter(instance -> !pausedRefreshOwns(asg, instance.getInstanceId()))
.collect(Collectors.toList());
if (terminatingInstances.isEmpty()) {
return;
@@ -362,45 +438,12 @@ public class AutoScalingReconciler {
List<LaunchPlacement> placements = new ArrayList<>(launchPlacements(asg));
for (int index = 0; index < count; index++) {
LaunchPlacement placement = leastPopulatedPlacement(asg, placements);
- Reservation reservation = ec2Service.runInstances(
- asg.getRegion(),
- launchSource.imageId(),
- launchSource.instanceType(),
- 1, 1,
- launchSource.keyName(),
- launchSource.securityGroupIds(),
- placement.subnetId(),
- null,
- propagatedInstanceTags(asg, launchSource),
- launchSource.userData(),
- launchSource.iamInstanceProfile(),
- launchSource.associatePublicIpAddress(), null, 0, null, null, null,
- launchSource.encodedUserData() != null ? launchSource.encodedUserData()
- : encodeLegacyUserData(launchSource.userData()),
- false, null, null, launchSource.blockDeviceMappings());
-
- for (Instance ec2Inst : reservation.getInstances()) {
- AsgInstance asgInst = new AsgInstance();
- asgInst.setInstanceId(ec2Inst.getInstanceId());
- asgInst.setAvailabilityZone(placement.subnetId() == null && placement.availabilityZone() != null
- ? placement.availabilityZone() : ec2Inst.getPlacement() != null
- ? ec2Inst.getPlacement().getAvailabilityZone() : placement.availabilityZone());
- asgInst.setLifecycleState("Pending");
- asgInst.setHealthStatus("Healthy");
- asgInst.setLaunchConfigurationName(launchSource.launchConfigurationName());
- asgInst.setLaunchTemplateId(launchSource.launchTemplateId());
- asgInst.setLaunchTemplateName(launchSource.launchTemplateName());
- asgInst.setLaunchTemplateVersion(launchSource.launchTemplateVersion());
- asgInst.setInstanceType(launchSource.instanceType());
- asg.getInstances().add(asgInst);
- launchedInstanceIds.add(ec2Inst.getInstanceId());
- LOG.infov("ASG {0}: launched instance {1} (Pending)",
- asg.getAutoScalingGroupName(), ec2Inst.getInstanceId());
- }
+ List<Instance> launched = launchInstance(asg, launchSource, placement, null, null,
+ List.of(), launchedInstanceIds, false);
if (placement.subnetId() != null) {
placements.set(placements.indexOf(placement), new LaunchPlacement(placement.subnetId(),
placement.availabilityZone(), Math.max(0,
- placement.availableIpAddresses() - reservation.getInstances().size()),
+ placement.availableIpAddresses() - launched.size()),
placement.requiresIpv4Address()));
}
}
@@ -455,6 +498,114 @@ public class AutoScalingReconciler {
}
}
+ private Map<String, List<Instance>> refreshLaunches(AutoScalingGroup asg) {
+ return ec2Service.describeInstances(asg.getRegion(), List.of(), null).stream()
+ .flatMap(result -> result.getInstances().stream())
+ .filter(instance -> instance.getClientToken() != null)
+ .filter(instance -> instance.getState() != null
+ && !ABANDONED_LAUNCH_STATES.contains(instance.getState().getName()))
+ .collect(Collectors.groupingBy(Instance::getClientToken));
+ }
+
+ private List<String> launchInstances(AutoScalingGroup asg, InstanceRefresh refresh,
+ InstanceRefreshReplacement replacement, List<Instance> existing,
+ Map<String, List<Instance>> acceptedLaunches) {
+ String clientToken = replacement.getLaunchClientToken();
+ LaunchSource launchSource = existing.isEmpty() ? resolveLaunchSource(asg) : null;
+ if (existing.isEmpty() && launchSource == null) {
+ LOG.warnv("ASG {0}: no launch source found, cannot scale out", asg.getAutoScalingGroupName());
+ return List.of();
+ }
+ List<String> launchedInstanceIds = new ArrayList<>();
+ LaunchPlacement placement = existing.isEmpty() ? leastPopulatedPlacement(asg, launchPlacements(asg)) : null;
+ InstanceRefreshLaunch metadata = replacement.getLaunch();
+ if (existing.isEmpty()) {
+ metadata = new InstanceRefreshLaunch(launchSource.launchConfigurationName(),
+ launchSource.launchTemplateId(), launchSource.launchTemplateName(),
+ launchSource.launchTemplateVersion(), launchSource.instanceType(), placement.availabilityZone());
+ replacement.setLaunch(metadata);
+ asgService.saveInstanceRefresh(refresh);
+ }
+ List<Instance> launched = launchInstance(asg, launchSource, placement, clientToken, metadata,
+ existing, launchedInstanceIds, true);
+ if (!launched.isEmpty()) {
+ acceptedLaunches.put(clientToken, launched);
+ preserveScaleOutAllocations(asg, launchedInstanceIds);
+ }
+ // Cleanup after a removed group releases these allocations rather than publishing ownership.
+ launchedInstanceIds.removeIf(id -> asg.getInstances().stream()
+ .noneMatch(instance -> id.equals(instance.getInstanceId())));
+ return launchedInstanceIds;
+ }
+
+ private List<Instance> launchInstance(AutoScalingGroup asg, LaunchSource launchSource,
+ LaunchPlacement placement, String clientToken,
+ InstanceRefreshLaunch metadata, List<Instance> existing,
+ List<String> launchedInstanceIds, boolean refreshLaunch) {
+ List<Instance> instances = existing;
+ if (instances.isEmpty()) {
+ Reservation reservation;
+ try {
+ reservation = ec2Service.runInstances(
+ asg.getRegion(), launchSource.imageId(), launchSource.instanceType(), 1, 1,
+ launchSource.keyName(), launchSource.securityGroupIds(), placement.subnetId(), clientToken,
+ propagatedInstanceTags(asg, launchSource), launchSource.userData(),
+ launchSource.iamInstanceProfile(), launchSource.associatePublicIpAddress(), null, 0, null, null, null,
+ launchSource.encodedUserData() != null ? launchSource.encodedUserData()
+ : encodeLegacyUserData(launchSource.userData()),
+ false, null, null, launchSource.blockDeviceMappings());
+ } catch (RuntimeException launchFailure) {
+ if (!refreshLaunch) {
+ throw launchFailure;
+ }
+ recordFailedActivity(asg, "Launching a replacement EC2 instance.",
+ "An instance refresh requested replacement of active instances.", launchFailure);
+ return List.of();
+ }
+ instances = reservation.getInstances();
+ }
+ for (Instance instance : instances) {
+ launchedInstanceIds.add(instance.getInstanceId());
+ }
+ for (Instance ec2Inst : instances) {
+ if (asg.getInstances().stream().anyMatch(instance ->
+ instance.getInstanceId().equals(ec2Inst.getInstanceId()))) {
+ continue;
+ }
+ AsgInstance asgInst = new AsgInstance();
+ asgInst.setInstanceId(ec2Inst.getInstanceId());
+ asgInst.setLifecycleState("Pending");
+ asgInst.setHealthStatus("Healthy");
+ if (metadata != null) {
+ asgInst.setAvailabilityZone(metadata.availabilityZone() != null ? metadata.availabilityZone()
+ : ec2Inst.getPlacement() != null ? ec2Inst.getPlacement().getAvailabilityZone() : null);
+ asgInst.setLaunchConfigurationName(metadata.launchConfigurationName());
+ asgInst.setLaunchTemplateId(metadata.launchTemplateId());
+ asgInst.setLaunchTemplateName(metadata.launchTemplateName());
+ asgInst.setLaunchTemplateVersion(metadata.launchTemplateVersion());
+ asgInst.setInstanceType(metadata.instanceType());
+ } else if (launchSource != null) {
+ asgInst.setAvailabilityZone(placement.subnetId() == null && placement.availabilityZone() != null
+ ? placement.availabilityZone() : ec2Inst.getPlacement() != null
+ ? ec2Inst.getPlacement().getAvailabilityZone() : placement.availabilityZone());
+ asgInst.setLaunchConfigurationName(launchSource.launchConfigurationName());
+ asgInst.setLaunchTemplateId(launchSource.launchTemplateId());
+ asgInst.setLaunchTemplateName(launchSource.launchTemplateName());
+ asgInst.setLaunchTemplateVersion(launchSource.launchTemplateVersion());
+ asgInst.setInstanceType(launchSource.instanceType());
+ } else {
+ // Older accepted attempts have no saved source identity; retain actual EC2 metadata.
+ asgInst.setInstanceType(ec2Inst.getInstanceType());
+ if (ec2Inst.getPlacement() != null) {
+ asgInst.setAvailabilityZone(ec2Inst.getPlacement().getAvailabilityZone());
+ }
+ }
+ asg.getInstances().add(asgInst);
+ LOG.infov("ASG {0}: launched instance {1} (Pending)", asg.getAutoScalingGroupName(), ec2Inst.getInstanceId());
+ }
+ return instances;
+ }
+
private List<LaunchPlacement> launchPlacements(AutoScalingGroup asg) {
if (!asg.getSubnetIds().isEmpty()) {
Map<String, Subnet> subnets = ec2Service.describeSubnets(asg.getRegion(), asg.getSubnetIds(), Map.of())
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java
index f69faa7a20..0192826786 100644
--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java
@@ -468,11 +468,12 @@ public class AutoScalingService {
});
}
groups.remove(key);
+ instanceRefreshes.entrySet().removeIf(e -> region.equals(e.getValue().getRegion())
+ && name.equals(e.getValue().getAutoScalingGroupName()));
}
// clean up associated hooks and policies
hooks.entrySet().removeIf(e -> e.getValue().getAutoScalingGroupName().equals(name));
policies.entrySet().removeIf(e -> e.getValue().getAutoScalingGroupName().equals(name));
- instanceRefreshes.entrySet().removeIf(e -> e.getValue().getAutoScalingGroupName().equals(name));
scheduledActions.entrySet().removeIf(e -> region.equals(e.getValue().getRegion())
&& name.equals(e.getValue().getAutoScalingGroupName()));
warmPools.remove(warmPoolKey(region, name));
@@ -502,11 +503,11 @@ public class AutoScalingService {
}
}
- void adjustCapacityIfPresent(AutoScalingGroup asg, Runnable adjustment) {
+ void reconcileIfPresent(AutoScalingGroup asg, Runnable reconciliation) {
String key = asgKey(asg.getRegion(), asg.getAutoScalingGroupName());
synchronized (lockFor(key)) {
if (groups.get(key) == asg) {
- adjustment.run();
+ reconciliation.run();
}
}
}
@@ -563,8 +564,8 @@ public class AutoScalingService {
}
// An empty list suspends all processes. The reconciler honors Launch and Terminate
- // for desired-capacity adjustments; other names remain stored state. Refresh selection
- // and cleanup are separate paths and do not yet honor suspended processes.
+ // for desired-capacity adjustments and owned refresh replacements. Other names remain
+ // stored state; InstanceRefresh also pauses its owned replacement work.
private static final List<String> ALL_SCALING_PROCESSES = List.of(
"Launch", "Terminate", "AddToLoadBalancer", "AlarmNotification", "AZRebalance",
"HealthCheck", "InstanceRefresh", "ReplaceUnhealthy", "ScheduledActions");
@@ -683,13 +684,19 @@ public class AutoScalingService {
public void detachInstances(String region, String name, List<String> instanceIds,
boolean decrementDesiredCapacity) {
- AutoScalingGroup asg = requireGroup(region, name);
- asg.getInstances().removeIf(i -> instanceIds.contains(i.getInstanceId()));
- if (decrementDesiredCapacity) {
- int newDesired = Math.max(asg.getMinSize(), asg.getDesiredCapacity() - instanceIds.size());
- asg.setDesiredCapacity(newDesired);
+ String key = asgKey(region, name);
+ synchronized (lockFor(key)) {
+ AutoScalingGroup asg = requireGroup(region, name);
+ List<String> detached = asg.getInstances().stream().map(AsgInstance::getInstanceId)
+ .filter(instanceIds::contains).toList();
+ releaseRefreshOwnership(asg, detached);
+ asg.getInstances().removeIf(i -> instanceIds.contains(i.getInstanceId()));
+ if (decrementDesiredCapacity) {
+ int newDesired = Math.max(asg.getMinSize(), asg.getDesiredCapacity() - instanceIds.size());
+ asg.setDesiredCapacity(newDesired);
+ }
+ groups.put(key, asg);
}
- groups.put(asgKey(region, name), asg);
}
public void terminateInstanceInAutoScalingGroup(String region, String instanceId,
@@ -700,67 +707,109 @@ public class AutoScalingService {
.findFirst()
.orElseThrow(() -> new AwsException("ValidationError",
"Instance '" + instanceId + "' not found in any Auto Scaling group.", 400));
- asg.getInstances().stream()
- .filter(i -> instanceId.equals(i.getInstanceId()))
- .findFirst()
- .ifPresent(i -> i.setLifecycleState("Terminating"));
- if (decrementDesiredCapacity) {
- int newDesired = Math.max(asg.getMinSize(), asg.getDesiredCapacity() - 1);
- asg.setDesiredCapacity(newDesired);
+ String key = asgKey(region, asg.getAutoScalingGroupName());
+ synchronized (lockFor(key)) {
+ if (groups.get(key) != asg) {
+ throw new AwsException("ValidationError",
+ "Instance '" + instanceId + "' not found in any Auto Scaling group.", 400);
+ }
+ AsgInstance instance = asg.getInstances().stream()
+ .filter(i -> instanceId.equals(i.getInstanceId())).findFirst()
+ .orElseThrow(() -> new AwsException("ValidationError",
+ "Instance '" + instanceId + "' not found in any Auto Scaling group.", 400));
+ releaseRefreshOwnership(asg, List.of(instanceId));
+ instance.setLifecycleState("Terminating");
+ if (decrementDesiredCapacity) {
+ int newDesired = Math.max(asg.getMinSize(), asg.getDesiredCapacity() - 1);
+ asg.setDesiredCapacity(newDesired);
+ }
+ groups.put(key, asg);
+ }
+ }
+
+ private void releaseRefreshOwnership(AutoScalingGroup asg, List<String> instanceIds) {
+ InstanceRefresh refresh = activeInstanceRefresh(asg);
+ if (refresh != null && refresh.getReplacements() != null) {
+ refresh.getReplacements().removeIf(replacement -> instanceIds.stream().anyMatch(instanceId ->
+ instanceId.equals(replacement.getOriginalInstanceId())
+ || instanceId.equals(replacement.getReplacementInstanceId())));
+ saveInstanceRefresh(refresh);
}
- groups.put(asgKey(region, asg.getAutoScalingGroupName()), asg);
}
// ── Instance refreshes ────────────────────────────────────────────────────
public InstanceRefresh startInstanceRefresh(String region, String asgName, InstanceRefresh requestedRefresh) {
- AutoScalingGroup asg = requireGroup(region, asgName);
- boolean activeRefresh = instanceRefreshes.values().stream()
- .filter(r -> region.equals(r.getRegion()))
- .filter(r -> asgName.equals(r.getAutoScalingGroupName()))
- .anyMatch(r -> isActiveRefreshStatus(r.getStatus()));
- if (activeRefresh) {
- throw new AwsException("InstanceRefreshInProgress",
- "An active instance refresh already exists for Auto Scaling group '" + asgName + "'.", 400);
- }
-
- Instant now = Instant.now();
- InstanceRefresh refresh = new InstanceRefresh();
- refresh.setInstanceRefreshId(UUID.randomUUID().toString());
- refresh.setAutoScalingGroupName(asgName);
- refresh.setStrategy(normalizeRefreshStrategy(requestedRefresh.getStrategy()));
- refresh.setStartTime(now);
- refresh.setRegion(region);
- copyDesiredConfiguration(requestedRefresh, refresh);
- copyPreferences(requestedRefresh, refresh);
-
- applyDesiredConfiguration(asg, refresh);
- List<String> instanceIds = markInstancesForRefresh(asg, refresh);
- if (instanceIds.isEmpty()) {
- refresh.setStatus("Successful");
- refresh.setStatusReason("Instance refresh completed.");
- refresh.setPercentageComplete(100);
- refresh.setInstancesToUpdate(0);
- refresh.setEndTime(now);
- } else {
- refresh.setStatus("InProgress");
- refresh.setStatusReason("Instance refresh in progress.");
- refresh.setPercentageComplete(0);
- refresh.setInstancesToUpdate(instanceIds.size());
- }
- instanceRefreshes.put(instanceRefreshKey(region, asgName, refresh.getInstanceRefreshId()), refresh);
- groups.put(asgKey(region, asgName), asg);
- if (!instanceIds.isEmpty()) {
+ synchronized (lockFor(asgKey(region, asgName))) {
+ AutoScalingGroup asg = requireGroup(region, asgName);
+ boolean activeRefresh = instanceRefreshes.values().stream()
+ .filter(r -> region.equals(r.getRegion()))
+ .filter(r -> asgName.equals(r.getAutoScalingGroupName()))
+ .anyMatch(r -> isActiveRefreshStatus(r.getStatus()));
+ if (activeRefresh) {
+ throw new AwsException("InstanceRefreshInProgress",
+ "An active instance refresh already exists for Auto Scaling group '" + asgName + "'.", 400);
+ }
+
+ Instant now = Instant.now();
+ InstanceRefresh refresh = new InstanceRefresh();
+ refresh.setInstanceRefreshId(UUID.randomUUID().toString());
+ refresh.setAutoScalingGroupName(asgName);
+ refresh.setStrategy(normalizeRefreshStrategy(requestedRefresh.getStrategy()));
+ refresh.setStartTime(now);
+ refresh.setRegion(region);
+ copyDesiredConfiguration(requestedRefresh, refresh);
+ copyPreferences(requestedRefresh, refresh);
+
+ applyDesiredConfiguration(asg, refresh);
+ List<String> instanceIds = markInstancesForRefresh(asg, refresh);
+ List<InstanceRefreshReplacement> replacements = new ArrayList<>();
+ for (String instanceId : instanceIds) {
+ InstanceRefreshReplacement replacement = new InstanceRefreshReplacement();
+ replacement.setOriginalInstanceId(instanceId);
+ replacement.setLaunchClientToken(UUID.randomUUID().toString());
+ replacements.add(replacement);
+ }
+ refresh.setReplacements(replacements);
+ if (instanceIds.isEmpty()) {
+ refresh.setStatus("Successful");
+ refresh.setStatusReason("Instance refresh completed.");
+ refresh.setPercentageComplete(100);
+ refresh.setInstancesToUpdate(0);
+ refresh.setEndTime(now);
+ } else {
+ refresh.setStatus("InProgress");
+ refresh.setStatusReason("Instance refresh in progress.");
+ refresh.setPercentageComplete(0);
+ refresh.setInstancesToUpdate(instanceIds.size());
+ }
+ instanceRefreshes.put(instanceRefreshKey(region, asgName, refresh.getInstanceRefreshId()), refresh);
+ groups.put(asgKey(region, asgName), asg);
+ if (!instanceIds.isEmpty()) {
+ recordActivity(region, asgName,
+ "Marked EC2 instance(s) for refresh: " + instanceIds,
+ "At " + now + " an instance refresh selected active instances for replacement.",
+ "Successful");
+ }
recordActivity(region, asgName,
- "Marked EC2 instance(s) for refresh: " + instanceIds,
- "At " + now + " an instance refresh selected active instances for replacement.",
+ "Started instance refresh " + refresh.getInstanceRefreshId() + ".",
+ "At " + now + " an instance refresh was started.",
"Successful");
+ return refresh;
}
- recordActivity(region, asgName,
- "Started instance refresh " + refresh.getInstanceRefreshId() + ".",
- "At " + now + " an instance refresh was started.",
- "Successful");
- return refresh;
+ }
+
+ InstanceRefresh activeInstanceRefresh(AutoScalingGroup asg) {
+ return instanceRefreshes.values().stream()
+ .filter(refresh -> asg.getRegion().equals(refresh.getRegion()))
+ .filter(refresh -> asg.getAutoScalingGroupName().equals(refresh.getAutoScalingGroupName()))
+ .filter(refresh -> isActiveRefreshStatus(refresh.getStatus()))
+ .findFirst().orElse(null);
+ }
+
+ void saveInstanceRefresh(InstanceRefresh refresh) {
+ instanceRefreshes.put(instanceRefreshKey(refresh.getRegion(), refresh.getAutoScalingGroupName(),
+ refresh.getInstanceRefreshId()), refresh);
}
public void completeInstanceRefreshIfSettled(String region, String asgName) {
@@ -771,7 +820,19 @@ public class AutoScalingService {
.filter(r -> isActiveRefreshStatus(r.getStatus()))
.collect(Collectors.toList());
for (InstanceRefresh refresh : activeRefreshes) {
- int remaining = remainingInstancesToUpdate(asg);
+ if (refresh.getReplacements() != null) {
+ for (InstanceRefreshReplacement replacement : refresh.getReplacements()) {
+ if (asg.getInstances().stream().anyMatch(instance ->
+ instance.getInstanceId().equals(replacement.getReplacementInstanceId())
+ && "InService".equals(instance.getLifecycleState()))) {
+ replacement.setPhase("Complete");
+ }
+ }
+ }
+ int remaining = refresh.getReplacements() == null ? remainingInstancesToUpdate(asg)
+ : (int) refresh.getReplacements().stream()
+ .filter(replacement -> !replacement.isComplete())
+ .count();
refresh.setInstancesToUpdate(remaining);
refresh.setPercentageComplete(remaining == 0 ? 100 : 0);
if (remaining == 0) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefresh.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefresh.java
index 0a95d164e1..6ef01f3cd4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefresh.java
+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefresh.java
@@ -21,6 +21,11 @@ public class InstanceRefresh {
private Instant startTime;
private Instant endTime;
private String region;
+ private List<InstanceRefreshReplacement> replacements;
+
+ public List<InstanceRefreshReplacement> getReplacements() { return replacements; }
+ public void setReplacements(List<InstanceRefreshReplacement> value) { replacements = value; }
+
private String desiredLaunchTemplateId;
private String desiredLaunchTemplateName;
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshLaunch.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshLaunch.java
new file mode 100644
index 0000000000..86e140e8a0
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshLaunch.java
@@ -0,0 +1,12 @@
+package io.github.hectorvent.floci.services.autoscaling.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+/** Source identity recorded before a refresh launch can be accepted by EC2. */
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record InstanceRefreshLaunch(String launchConfigurationName, String launchTemplateId,
+ String launchTemplateName, String launchTemplateVersion,
+ String instanceType, String availabilityZone) {
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshReplacement.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshReplacement.java
new file mode 100644
index 0000000000..2bb9ca0fe6
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshReplacement.java
@@ -0,0 +1,32 @@
+package io.github.hectorvent.floci.services.autoscaling.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class InstanceRefreshReplacement {
+ private String originalInstanceId;
+ private String replacementInstanceId;
+ private String launchClientToken;
+ private String phase = "Selected";
+ private InstanceRefreshLaunch launch;
+
+ public InstanceRefreshLaunch getLaunch() { return launch; }
+ public void setLaunch(InstanceRefreshLaunch value) { launch = value; }
+
+ public String getPhase() { return phase; }
+ public void setPhase(String value) { phase = value; }
+
+ public boolean isComplete() { return "Complete".equals(phase) || "NoLongerRequired".equals(phase); }
+
+
+ public String getOriginalInstanceId() { return originalInstanceId; }
+ public void setOriginalInstanceId(String value) { originalInstanceId = value; }
+
+ public String getReplacementInstanceId() { return replacementInstanceId; }
+ public void setReplacementInstanceId(String value) { replacementInstanceId = value; }
+
+ public String getLaunchClientToken() { return launchClientToken; }
+ public void setLaunchClientToken(String value) { launchClientToken = value; }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java
index 48faa51c1a..777f820a80 100644
--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java
@@ -39,7 +39,7 @@ class AutoScalingCapacityAccountLockIntegrationTest {
AutoScalingGroup group = RequestScopes.callAs(ACCOUNT, REGION, () -> create(REGION, name));
RequestScopes.runAs(otherAccount, otherRegion, () -> create(otherRegion, name));
Future<?> adjustment = workers.submit(() -> RequestScopes.runAs(ACCOUNT, REGION,
- () -> service.adjustCapacityIfPresent(group, () -> {
+ () -> service.reconcileIfPresent(group, () -> {
entered.countDown();
try {
assertTrue(release.await(10, TimeUnit.SECONDS));
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingPlacementTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingPlacementTest.java
index 388704d819..765cc7f61f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingPlacementTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingPlacementTest.java
@@ -307,7 +307,7 @@ class AutoScalingPlacementTest {
doAnswer(call -> {
call.<Runnable>getArgument(1).run();
return null;
- }).when(groups).adjustCapacityIfPresent(any(), any());
+ }).when(groups).reconcileIfPresent(any(), any());
when(groups.describeScalingActivities(anyString(), anyString())).thenAnswer(call ->
activityStore.describeScalingActivities(call.getArgument(0), call.getArgument(1)));
when(groups.recordActivity(anyString(), anyString(), anyString(), anyString(), anyString()))
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
index 1fbf91ecd8..ba8bfcdf64 100644
--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
@@ -55,7 +55,7 @@ class AutoScalingReconcilerTest {
Runnable adjustment = invocation.getArgument(1);
adjustment.run();
return null;
- }).when(service).adjustCapacityIfPresent(any(), any());
+ }).when(service).reconcileIfPresent(any(), any());
return service;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshStorageTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshStorageTest.java
new file mode 100644
index 0000000000..d51875bc45
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshStorageTest.java
@@ -0,0 +1,1011 @@
+package io.github.hectorvent.floci.services.autoscaling;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.ServiceConfigAccess;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.autoscaling.model.AsgInstance;
+import io.github.hectorvent.floci.services.autoscaling.model.AsgOptionalFields;
+import io.github.hectorvent.floci.services.autoscaling.model.AutoScalingGroup;
+import io.github.hectorvent.floci.services.autoscaling.model.InstanceRefresh;
+import io.github.hectorvent.floci.services.autoscaling.model.ScalingActivity;
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.ec2.model.Instance;
+import io.github.hectorvent.floci.services.ec2.model.InstanceState;
+import io.github.hectorvent.floci.services.ec2.model.LaunchTemplate;
+import io.github.hectorvent.floci.services.ec2.model.Placement;
+import io.github.hectorvent.floci.services.ec2.model.Reservation;
+import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
+import io.github.hectorvent.floci.services.elbv2.model.TargetDescription;
+import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+class AutoScalingRefreshStorageTest {
+ @TempDir Path storagePath;
+ private final List<StorageFactory> factories = new ArrayList<>();
+ private static final String REGION = "us-east-1";
+ private AutoScalingService service;
+ private Ec2Service ec2;
+ private AutoScalingReconciler reconciler;
+ private AutoScalingGroup group;
+
+ private void setUp(String mode) {
+ service = storedService(mode);
+ service.createAutoScalingGroup(REGION, "suspended-group", null, "lt-original", null, "1", null,
+ 0, 3, 1, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,
+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());
+ group = service.describeAutoScalingGroups(REGION, List.of("suspended-group")).getFirst();
+ AsgInstance existing = new AsgInstance();
+ existing.setInstanceId("i-original");
+ existing.setLifecycleState("InService");
+ existing.setHealthStatus("Healthy");
+ existing.setLaunchTemplateId("lt-original");
+ existing.setLaunchTemplateVersion("1");
+ group.getInstances().add(existing);
+ service.saveAutoScalingGroupIfPresent(group);
+ ec2 = mock(Ec2Service.class);
+ service.ec2Service = ec2;
+ reconciler = new AutoScalingReconciler(service, ec2, mock(ElbV2Service.class));
+ when(ec2.isInstanceContainerRunning(anyString())).thenReturn(true);
+ when(ec2.describeInstances(eq(REGION), anyList(), isNull()))
+ .thenReturn(List.of(reservation("i-original")));
+ LaunchTemplate template = new LaunchTemplate();
+ template.setLaunchTemplateId("lt-original");
+ template.setLatestVersionNumber("1");
+ template.getData().setImageId("ami-original");
+ template.getData().setInstanceType("t3.micro");
+ when(ec2.describeLaunchTemplates(eq(REGION), anyList(), anyList(), anyMap())).thenReturn(List.of(template));
+ when(ec2.describeLaunchTemplateVersions(eq(REGION), eq("lt-original"), isNull(), anyList()))
+ .thenReturn(List.of(template));
+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),
+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any())).thenReturn(reservation("i-replacement"));
+ }
+
+ private AutoScalingService storedService(String mode) {
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ EmulatorConfig.StorageConfig storage = mock(EmulatorConfig.StorageConfig.class);
+ EmulatorConfig.WalConfig wal = mock(EmulatorConfig.WalConfig.class);
+ when(config.storage()).thenReturn(storage);
+ when(config.defaultAccountId()).thenReturn("000000000000");
+ when(storage.persistentPath()).thenReturn(storagePath.toString());
+ when(storage.wal()).thenReturn(wal);
+ when(wal.compactionIntervalMs()).thenReturn(60000L);
+ ServiceConfigAccess access = mock(ServiceConfigAccess.class);
+ when(access.storageMode("autoscaling")).thenReturn(mode);
+ when(access.storageFlushInterval("autoscaling")).thenReturn(60000L);
+ StorageFactory factory = new StorageFactory(config, access);
+ factories.add(factory);
+ AutoScalingService created = new AutoScalingService();
+ created.regionResolver = new RegionResolver(REGION, "000000000000");
+ created.storageFactory = factory;
+ created.initializeStorage();
+ return created;
+ }
+
+ private void restart(String mode) throws IOException {
+ AutoScalingGroup previous = group;
+ factories.removeFirst().shutdownAll();
+ try (Stream<Path> files = Files.list(storagePath)) {
+ assertTrue(files.anyMatch(p -> p.getFileName().toString().contains("autoscaling-groups")),
+ "No real group snapshot was written");
+ }
+ service = storedService(mode);
+ service.ec2Service = ec2;
+ group = service.describeAutoScalingGroups(REGION, List.of("suspended-group")).getFirst();
+ assertNotSame(previous, group, "Restart reused the in-memory group");
+ reconciler = new AutoScalingReconciler(service, ec2, mock(ElbV2Service.class));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void replacementWithoutDeclaredZoneUsesTheAcceptedEc2Placement(String mode) throws IOException {
+ setUp(mode);
+ group.setAvailabilityZones(List.of());
+ service.saveAutoScalingGroupIfPresent(group);
+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of());
+ Reservation accepted = reservation("i-replacement");
+ accepted.getInstances().getFirst().setPlacement(new Placement("us-east-1b"));
+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),
+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any())).thenReturn(accepted);
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ reconciler.reconcile(group);
+ assertEquals("i-replacement", group.getInstances().getFirst().getInstanceId());
+ assertEquals("us-east-1b", group.getInstances().getFirst().getAvailabilityZone());
+ assertEquals("i-replacement", refresh.getReplacements().getFirst().getReplacementInstanceId());
+ restart(mode);
+ assertEquals("us-east-1b", group.getInstances().getFirst().getAvailabilityZone());
+ }
+
+ @AfterEach
+ void closeStorage() {
+ factories.forEach(StorageFactory::shutdownAll);
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "persistent,Launch,true", "persistent,Terminate,true", "persistent,InstanceRefresh,true",
+ "persistent,Launch,false", "persistent,Terminate,false", "persistent,InstanceRefresh,false",
+ "hybrid,Launch,true", "hybrid,Terminate,true", "hybrid,InstanceRefresh,true",
+ "hybrid,Launch,false", "hybrid,Terminate,false", "hybrid,InstanceRefresh,false",
+ "wal,Launch,true", "wal,Terminate,true", "wal,InstanceRefresh,true",
+ "wal,Launch,false", "wal,Terminate,false", "wal,InstanceRefresh,false"
+ })
+ void refreshPauseSurvivesRealStorageRestart(String mode, String process, boolean suspendBefore) throws IOException {
+ setUp(mode);
+ if (suspendBefore) {
+ service.suspendProcesses(REGION, "suspended-group", List.of(process));
+ }
+ InstanceRefresh started = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ if (!suspendBefore) {
+ service.suspendProcesses(REGION, "suspended-group", List.of(process));
+ }
+ restart(mode);
+ InstanceRefresh restored = service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)
+ .instanceRefreshes().getFirst();
+ assertNotSame(started, restored);
+ assertEquals(started.getInstanceRefreshId(), restored.getInstanceRefreshId());
+ assertTrue(group.getSuspendedProcesses().contains(process));
+ reconciler.reconcile(group);
+ assertAll(
+ () -> verify(ec2, never()).terminateInstances(anyString(), anyList()),
+ () -> verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),
+ any(), any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()),
+ () -> assertEquals(List.of("i-original"), group.getInstances().stream()
+ .map(AsgInstance::getInstanceId).toList()),
+ () -> assertNotEquals("Successful", service.describeInstanceRefreshes(
+ REGION, "suspended-group", null, null, null).instanceRefreshes().getFirst().getStatus()));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void failedReplacementLaunchRemainsPausedAfterRestart(String mode) throws IOException {
+ setUp(mode);
+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),
+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()))
+ .thenThrow(new IllegalStateException("launch unavailable"))
+ .thenReturn(reservation("i-replacement"));
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ reconciler.reconcile(group);
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ assertTrue(group.getInstances().isEmpty());
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ restart(mode);
+ InstanceRefresh restored = service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)
+ .instanceRefreshes().getFirst();
+ assertEquals("InProgress", restored.getStatus());
+ assertEquals(1, restored.getInstancesToUpdate());
+ reconciler.reconcile(group);
+ assertAll(
+ () -> verify(ec2, times(1)).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(),
+ anyList(), any(), any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()),
+ () -> assertTrue(group.getInstances().isEmpty()));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void resumedRefreshReplacesOriginalAfterRestart(String mode) throws IOException {
+ setUp(mode);
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ restart(mode);
+ service.resumeProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ reconciler.reconcile(group);
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ assertEquals(List.of("i-replacement"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void manualTerminationDuringPausedRefreshStillRunsAfterRestart(String mode) throws IOException {
+ setUp(mode);
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ AsgInstance manual = new AsgInstance();
+ manual.setInstanceId("i-manual");
+ manual.setLifecycleState("InService");
+ manual.setHealthStatus("Healthy");
+ group.getInstances().add(manual);
+ group.setDesiredCapacity(2);
+ service.saveAutoScalingGroupIfPresent(group);
+ service.terminateInstanceInAutoScalingGroup(REGION, "i-manual", true);
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ restart(mode);
+ reconciler.reconcile(group);
+ assertAll(
+ () -> verify(ec2).terminateInstances(REGION, List.of("i-manual")),
+ () -> assertEquals(List.of("i-original"), group.getInstances().stream()
+ .map(AsgInstance::getInstanceId).toList()));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void manualTerminationStillRunsAfterRestart(String mode) throws IOException {
+ setUp(mode);
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ service.terminateInstanceInAutoScalingGroup(REGION, "i-original", true);
+ restart(mode);
+ reconciler.reconcile(group);
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ assertTrue(group.getInstances().isEmpty());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void failedReplacementRecordsActivityAndRetainsRetryOwnership(String mode) throws IOException {
+ setUp(mode);
+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),
+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()))
+ .thenThrow(new AwsException(
+ "InsufficientInstanceCapacity", "replacement capacity unavailable", 500))
+ .thenThrow(new AwsException("InsufficientInstanceCapacity", "replacement capacity unavailable", 500))
+ .thenThrow(new AwsException("RequestLimitExceeded", "replacement throttled", 503))
+ .thenReturn(reservation("i-replacement"));
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ String token = refresh.getReplacements().getFirst().getLaunchClientToken();
+ reconciler.reconcile(group);
+ assertTrue(group.getInstances().isEmpty());
+ assertEquals("InProgress", refresh.getStatus());
+ assertEquals(token, refresh.getReplacements().getFirst().getLaunchClientToken());
+ List<ScalingActivity> failed = service.describeScalingActivities(REGION, "suspended-group").stream()
+ .filter(activity -> "Failed".equals(activity.getStatusCode())).toList();
+ assertEquals(1, failed.size());
+ assertEquals("replacement capacity unavailable", failed.getFirst().getStatusMessage());
+ assertEquals(100, failed.getFirst().getProgress());
+ assertNotNull(failed.getFirst().getEndTime());
+ reconciler.reconcile(group);
+ assertEquals(1, service.describeScalingActivities(REGION, "suspended-group").stream()
+ .filter(activity -> "Failed".equals(activity.getStatusCode())).count());
+ reconciler.reconcile(group);
+ assertEquals(List.of("replacement throttled", "replacement capacity unavailable"),
+ service.describeScalingActivities(REGION, "suspended-group").stream()
+ .filter(activity -> "Failed".equals(activity.getStatusCode()))
+ .map(ScalingActivity::getStatusMessage).toList());
+ restart(mode);
+ assertEquals(2, service.describeScalingActivities(REGION, "suspended-group").stream()
+ .filter(activity -> "Failed".equals(activity.getStatusCode())).count());
+ reconciler.reconcile(group);
+ reconciler.reconcile(group);
+ verify(ec2, times(4)).runInstances(eq(REGION), any(), any(), eq(1), eq(1), any(), anyList(), any(),
+ eq(token), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ assertEquals("Successful", service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)
+ .instanceRefreshes().getFirst().getStatus());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void failedReplacementAllowsSiblingProgressWithOneInventoryScan(String mode) {
+ setUp(mode);
+ group.setDesiredCapacity(2);
+ AsgInstance second = new AsgInstance();
+ second.setInstanceId("i-second-original");
+ second.setLifecycleState("InService");
+ second.setHealthStatus("Healthy");
+ group.getInstances().add(second);
+ service.saveAutoScalingGroupIfPresent(group);
+ Reservation unrelated = reservation("i-unrelated");
+ unrelated.getInstances().getFirst().setClientToken("unrelated-attempt");
+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of(unrelated));
+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),
+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(),
+ anyBoolean(), any(), any(), any()))
+ .thenThrow(new IllegalStateException("first replacement launch unavailable"))
+ .thenReturn(reservation("i-sibling"))
+ .thenReturn(reservation("i-retry"));
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ String token = refresh.getReplacements().getFirst().getLaunchClientToken();
+ reconciler.reconcile(group);
+ assertEquals(List.of("i-sibling"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());
+ assertEquals("InProgress", refresh.getStatus());
+ assertEquals(token, refresh.getReplacements().getFirst().getLaunchClientToken());
+ assertEquals("first replacement launch unavailable", service.describeScalingActivities(REGION, "suspended-group")
+ .stream().filter(activity -> "Failed".equals(activity.getStatusCode())).findFirst().orElseThrow()
+ .getStatusMessage());
+ verify(ec2, times(1)).describeInstances(REGION, List.of(), null);
+ reconciler.reconcile(group);
+ verify(ec2, times(2)).describeInstances(REGION, List.of(), null);
+ assertEquals(List.of("i-sibling", "i-retry"),
+ group.getInstances().stream().map(AsgInstance::getInstanceId).toList());
+ reconciler.reconcile(group);
+ assertEquals("Successful", refresh.getStatus());
+ verify(ec2, times(2)).describeInstances(REGION, List.of(), null);
+ }
+
+ @Test
+ void refreshCleanupFailureRetainsOwnershipAndPropagates() {
+ setUp("persistent");
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ String token = refresh.getReplacements().getFirst().getLaunchClientToken();
+ AutoScalingService intercepted = spy(service);
+ doReturn(false).when(intercepted).saveAutoScalingGroupIfPresent(group);
+ IllegalStateException cleanupFailure = new IllegalStateException("cleanup refused");
+ when(ec2.terminateInstances(REGION, List.of("i-replacement"))).thenThrow(cleanupFailure);
+ AutoScalingReconciler worker = new AutoScalingReconciler(intercepted, ec2, mock(ElbV2Service.class));
+ assertSame(cleanupFailure, assertThrows(IllegalStateException.class, () -> worker.reconcile(group)));
+ assertEquals(token, refresh.getReplacements().getFirst().getLaunchClientToken());
+ assertEquals("InProgress", refresh.getStatus());
+ assertNull(refresh.getReplacements().getFirst().getReplacementInstanceId());
+ assertEquals(List.of("i-replacement"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void launchRetryResumesWithSavedToken(String mode) throws IOException {
+ setUp(mode);
+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),
+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()))
+ .thenThrow(new IllegalStateException("launch unavailable"))
+ .thenReturn(reservation("i-replacement"));
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ String token = refresh.getReplacements().getFirst().getLaunchClientToken();
+ reconciler.reconcile(group);
+ restart(mode);
+ reconciler.reconcile(group);
+ reconciler.reconcile(group);
+ verify(ec2, times(2)).runInstances(eq(REGION), any(), any(), eq(1), eq(1), any(), anyList(), any(),
+ eq(token), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ verify(ec2, times(1)).terminateInstances(REGION, List.of("i-original"));
+ assertEquals("Successful", service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)
+ .instanceRefreshes().getFirst().getStatus());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void recoversEc2LaunchWhoseMembershipWasNotSaved(String mode) throws IOException {
+ setUp(mode);
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ Reservation launched = reservation("i-recovered");
+ launched.getInstances().getFirst().setClientToken(refresh.getReplacements().getFirst().getLaunchClientToken());
+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of(launched));
+ group.getInstances().clear();
+ service.saveAutoScalingGroupIfPresent(group);
+ restart(mode);
+ reconciler.reconcile(group);
+ assertEquals(List.of("i-recovered"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());
+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),
+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ reconciler.reconcile(group);
+ assertEquals(1, group.getInstances().size());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void explicitTerminationOfSelectedOriginalIsNotPaused(String mode) throws IOException {
+ setUp(mode);
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ service.terminateInstanceInAutoScalingGroup(REGION, "i-original", true);
+ restart(mode);
+ reconciler.reconcile(group);
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ assertTrue(group.getInstances().isEmpty());
+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),
+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void pausedRefreshDoesNotBlockIndependentCapacityIncrease(String mode) throws IOException {
+ setUp(mode);
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ group.setDesiredCapacity(2);
+ service.saveAutoScalingGroupIfPresent(group);
+ restart(mode);
+ reconciler.reconcile(group);
+ verify(ec2, never()).terminateInstances(anyString(), anyList());
+ verify(ec2).runInstances(eq(REGION), any(), any(), eq(1), eq(1), any(), anyList(), any(), isNull(),
+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ assertEquals(2, group.getInstances().size());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void capacityReductionDoesNotLaunchUnneededReplacement(String mode) throws IOException {
+ setUp(mode);
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ group.setDesiredCapacity(0);
+ service.saveAutoScalingGroupIfPresent(group);
+ restart(mode);
+ reconciler.reconcile(group);
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),
+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ assertTrue(group.getInstances().isEmpty());
+ assertEquals("Successful", service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)
+ .instanceRefreshes().getFirst().getStatus());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void missingOriginalRetainsPausedReplacementReservation(String mode) throws IOException {
+ setUp(mode);
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ when(ec2.describeInstances(REGION, List.of("i-original"), null)).thenReturn(List.of());
+ restart(mode);
+ reconciler.reconcile(group);
+ assertTrue(group.getInstances().isEmpty());
+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),
+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ service.resumeProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ restart(mode);
+ reconciler.reconcile(group);
+ assertEquals(List.of("i-replacement"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void recoveredLaunchAlreadyInGroupDoesNotDuplicateMembership(String mode) throws IOException {
+ setUp(mode);
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ Reservation launched = reservation("i-recovered");
+ launched.getInstances().getFirst().setClientToken(refresh.getReplacements().getFirst().getLaunchClientToken());
+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of(launched));
+ group.getInstances().clear();
+ AsgInstance member = new AsgInstance();
+ member.setInstanceId("i-recovered");
+ member.setLifecycleState("Pending");
+ member.setHealthStatus("Healthy");
+ group.getInstances().add(member);
+ service.saveAutoScalingGroupIfPresent(group);
+ restart(mode);
+ reconciler.reconcile(group);
+ assertEquals(1, group.getInstances().size());
+ InstanceRefresh restored = service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)
+ .instanceRefreshes().getFirst();
+ assertEquals("i-recovered", restored.getReplacements().getFirst().getReplacementInstanceId());
+ assertEquals("Successful", restored.getStatus());
+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),
+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ }
+
+ @ParameterizedTest
+ @CsvSource({"persistent,Launch", "persistent,Terminate", "persistent,InstanceRefresh",
+ "hybrid,Launch", "hybrid,Terminate", "hybrid,InstanceRefresh",
+ "wal,Launch", "wal,Terminate", "wal,InstanceRefresh"})
+ void pausedOriginalKeepsTargetRegistrationUntilResume(String mode, String process) throws IOException {
+ setUp(mode);
+ group.setTargetGroupARNs(List.of("test-target-group"));
+ service.saveAutoScalingGroupIfPresent(group);
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ service.suspendProcesses(REGION, "suspended-group", List.of(process));
+ restart(mode);
+ ElbV2Service elb = mock(ElbV2Service.class);
+ TargetDescription target = new TargetDescription();
+ target.setId("i-original");
+ TargetHealth health = new TargetHealth();
+ health.setTarget(target);
+ health.setState("healthy");
+ when(elb.describeTargetHealth(REGION, "test-target-group", List.of())).thenReturn(List.of(health));
+ reconciler = new AutoScalingReconciler(service, ec2, elb);
+ reconciler.reconcile(group);
+ verify(elb, never()).deregisterTargets(anyString(), anyString(), anyList());
+ verify(ec2, never()).terminateInstances(anyString(), anyList());
+ service.resumeProcesses(REGION, "suspended-group", List.of(process));
+ reconciler.reconcile(group);
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ verify(elb, atLeastOnce()).deregisterTargets(eq(REGION), eq("test-target-group"),
+ argThat(targets -> targets.stream().anyMatch(value -> "i-original".equals(value.getId()))));
+ assertEquals(List.of("i-replacement"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());
+ }
+
+ @ParameterizedTest
+ @CsvSource({"persistent,false,false", "persistent,false,true", "persistent,true,false", "persistent,true,true",
+ "hybrid,false,false", "hybrid,false,true", "hybrid,true,false", "hybrid,true,true",
+ "wal,false,false", "wal,false,true", "wal,true,false", "wal,true,true"})
+ void detachedInstancesAreNotReadoptedAfterRestart(String mode, boolean replacement, boolean decrement)
+ throws IOException {
+ setUp(mode);
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ String detachedId = replacement ? "i-detached" : "i-original";
+ if (replacement) {
+ group.getInstances().clear();
+ AsgInstance member = new AsgInstance();
+ member.setInstanceId(detachedId);
+ member.setLifecycleState("InService");
+ member.setHealthStatus("Healthy");
+ group.getInstances().add(member);
+ refresh.getReplacements().getFirst().setReplacementInstanceId(detachedId);
+ refresh.getReplacements().getFirst().setPhase("Warming");
+ service.saveInstanceRefresh(refresh);
+ service.saveAutoScalingGroupIfPresent(group);
+ Reservation launched = reservation(detachedId);
+ launched.getInstances().getFirst().setClientToken(refresh.getReplacements().getFirst().getLaunchClientToken());
+ when(ec2.describeInstances(eq(REGION), any(), isNull())).thenReturn(List.of(launched));
+ }
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ service.detachInstances(REGION, "suspended-group", List.of(detachedId), decrement);
+ restart(mode);
+ InstanceRefresh restored = service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)
+ .instanceRefreshes().getFirst();
+ assertTrue(restored.getReplacements().isEmpty(), "Explicit detach retained refresh ownership");
+ service.resumeProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ reconciler.reconcile(group);
+ reconciler.reconcile(group);
+ assertEquals(decrement ? 0 : 1, group.getDesiredCapacity());
+ assertEquals(decrement ? 0 : 1, group.getInstances().size());
+ assertTrue(group.getInstances().stream().noneMatch(member -> detachedId.equals(member.getInstanceId())));
+ verify(ec2, never()).terminateInstances(anyString(), argThat(ids -> ids.contains(detachedId)));
+ if (decrement) {
+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),
+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ }
+ assertEquals("Successful", restored.getStatus());
+ }
+
+ @ParameterizedTest
+ @CsvSource({"persistent,false,false", "persistent,true,false", "hybrid,false,false", "hybrid,true,false",
+ "wal,false,false", "wal,true,false", "persistent,false,true", "persistent,true,true",
+ "hybrid,false,true", "hybrid,true,true", "wal,false,true", "wal,true,true"})
+ void recoveredLaunchRetainsAttemptSourceAfterGroupChanges(String mode, boolean sourceDeleted,
+ boolean defaultPlacement) throws IOException {
+ setUp(mode);
+ if (defaultPlacement) {
+ group.setAvailabilityZones(List.of());
+ service.saveAutoScalingGroupIfPresent(group);
+ }
+ List<Reservation> launched = new ArrayList<>();
+ when(ec2.describeInstances(REGION, List.of(), null)).thenAnswer(invocation -> List.copyOf(launched));
+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),
+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any())).thenAnswer(invocation -> {
+ Reservation accepted = reservation("i-accepted");
+ accepted.getInstances().getFirst().setClientToken(invocation.getArgument(8));
+ accepted.getInstances().getFirst().setInstanceType("t3.micro");
+ accepted.getInstances().getFirst().setPlacement(new Placement("us-east-1a"));
+ launched.add(accepted);
+ throw new IllegalStateException("EC2 accepted launch but response was lost");
+ });
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ reconciler.reconcile(group);
+ assertTrue(group.getInstances().isEmpty());
+ assertEquals(1, launched.size());
+ group.setLaunchTemplateId("lt-changed");
+ group.setAvailabilityZones(List.of("us-east-1b"));
+ service.saveAutoScalingGroupIfPresent(group);
+ LaunchTemplate changed = new LaunchTemplate();
+ changed.setLaunchTemplateId("lt-changed");
+ changed.setLatestVersionNumber("2");
+ changed.getData().setImageId("ami-changed");
+ changed.getData().setInstanceType("m5.large");
+ when(ec2.describeLaunchTemplates(eq(REGION), anyList(), anyList(), anyMap()))
+ .thenReturn(sourceDeleted ? List.of() : List.of(changed));
+ when(ec2.describeLaunchTemplateVersions(eq(REGION), eq("lt-changed"), isNull(), anyList()))
+ .thenReturn(sourceDeleted ? List.of() : List.of(changed));
+ restart(mode);
+ reconciler.reconcile(group);
+ assertEquals(List.of("i-accepted"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());
+ AsgInstance recovered = group.getInstances().getFirst();
+ assertAll(() -> assertEquals("lt-original", recovered.getLaunchTemplateId()),
+ () -> assertEquals("1", recovered.getLaunchTemplateVersion()),
+ () -> assertEquals("t3.micro", recovered.getInstanceType()),
+ () -> assertEquals("us-east-1a", recovered.getAvailabilityZone()));
+ verify(ec2, times(1)).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),
+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void legacyRecoveredLaunchUsesOnlyKnownEc2Metadata(String mode) throws IOException {
+ setUp(mode);
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ Reservation launched = reservation("i-legacy");
+ Instance instance = launched.getInstances().getFirst();
+ instance.setClientToken(refresh.getReplacements().getFirst().getLaunchClientToken());
+ instance.setInstanceType("m5.large");
+ instance.setPlacement(new Placement("us-east-1b"));
+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of(launched));
+ group.getInstances().clear();
+ service.saveAutoScalingGroupIfPresent(group);
+ restart(mode);
+ reconciler.reconcile(group);
+ AsgInstance recovered = group.getInstances().getFirst();
+ assertAll(() -> assertEquals("i-legacy", recovered.getInstanceId()),
+ () -> assertEquals("m5.large", recovered.getInstanceType()),
+ () -> assertEquals("us-east-1b", recovered.getAvailabilityZone()),
+ () -> assertNull(recovered.getLaunchConfigurationName()),
+ () -> assertNull(recovered.getLaunchTemplateId()),
+ () -> assertNull(recovered.getLaunchTemplateName()),
+ () -> assertNull(recovered.getLaunchTemplateVersion()));
+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),
+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void partialMultiInstanceRefreshResumesWithoutRepeatingCompletedLaunch(String mode) throws IOException {
+ setUp(mode);
+ AsgInstance second = new AsgInstance();
+ second.setInstanceId("i-second");
+ second.setLifecycleState("InService");
+ second.setHealthStatus("Healthy");
+ group.getInstances().add(second);
+ group.setDesiredCapacity(2);
+ service.saveAutoScalingGroupIfPresent(group);
+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of());
+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),
+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()))
+ .thenReturn(reservation("i-first-replacement"))
+ .thenThrow(new IllegalStateException("Second launch unavailable"))
+ .thenReturn(reservation("i-second-replacement"));
+ InstanceRefresh started = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ assertEquals(2, started.getReplacements().size());
+ assertNotEquals(started.getReplacements().getFirst().getLaunchClientToken(),
+ started.getReplacements().getLast().getLaunchClientToken());
+ reconciler.reconcile(group);
+ assertEquals(List.of("i-first-replacement"), group.getInstances().stream()
+ .map(AsgInstance::getInstanceId).toList());
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ restart(mode);
+ reconciler.reconcile(group);
+ InstanceRefresh restored = service.activeInstanceRefresh(group);
+ assertNotNull(restored);
+ assertEquals(1, restored.getInstancesToUpdate());
+ assertEquals("Complete", restored.getReplacements().getFirst().getPhase());
+ verify(ec2, times(2)).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),
+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ service.resumeProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ reconciler.reconcile(group);
+ reconciler.reconcile(group);
+ assertEquals("Successful", restored.getStatus());
+ assertEquals(0, restored.getInstancesToUpdate());
+ assertEquals(List.of("i-first-replacement", "i-second-replacement"), group.getInstances().stream()
+ .map(AsgInstance::getInstanceId).toList());
+ verify(ec2, times(3)).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),
+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());
+ verify(ec2).terminateInstances(REGION, List.of("i-original", "i-second"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void legacyNullOwnershipRetainsExistingCleanupAfterRestart(String mode) throws IOException {
+ setUp(mode);
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ refresh.setReplacements(null);
+ service.saveInstanceRefresh(refresh);
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ restart(mode);
+ InstanceRefresh restored = service.activeInstanceRefresh(group);
+ assertNotNull(restored);
+ assertNull(restored.getReplacements());
+ reconciler.reconcile(group);
+ reconciler.reconcile(group);
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ assertEquals(List.of("i-replacement"), group.getInstances().stream()
+ .map(AsgInstance::getInstanceId).toList());
+ assertEquals("Successful", restored.getStatus());
+ assertNull(restored.getReplacements(), "Legacy state was heuristically assigned new ownership");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void sameNameOtherRegionRefreshDoesNotOwnLocalInstances(String mode) throws IOException {
+ setUp(mode);
+ String otherRegion = "us-west-2";
+ LaunchTemplate otherTemplate = new LaunchTemplate();
+ otherTemplate.setLaunchTemplateId("lt-other");
+ otherTemplate.getData().setImageId("ami-other");
+ when(ec2.describeLaunchTemplateVersions(eq(otherRegion), eq("lt-other"), isNull(), anyList()))
+ .thenReturn(List.of(otherTemplate));
+ service.createAutoScalingGroup(otherRegion, "suspended-group", null, "lt-other", null, "1", null,
+ 0, 3, 1, 300, List.of("us-west-2a"), List.of(), List.of(), List.of(), "EC2", 0,
+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());
+ AutoScalingGroup other = service.describeAutoScalingGroups(otherRegion, List.of("suspended-group"))
+ .getFirst();
+ AsgInstance otherMember = new AsgInstance();
+ otherMember.setInstanceId("i-original");
+ otherMember.setLifecycleState("InService");
+ otherMember.setHealthStatus("Healthy");
+ other.getInstances().add(otherMember);
+ service.saveAutoScalingGroupIfPresent(other);
+ service.suspendProcesses(otherRegion, "suspended-group", List.of("InstanceRefresh"));
+ service.startInstanceRefresh(otherRegion, "suspended-group", new InstanceRefresh());
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ service.terminateInstanceInAutoScalingGroup(REGION, "i-original", true);
+ restart(mode);
+ reconciler.reconcile(group);
+ assertTrue(group.getInstances().isEmpty());
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ AutoScalingGroup restoredOther = service.describeAutoScalingGroups(otherRegion,
+ List.of("suspended-group")).getFirst();
+ InstanceRefresh otherRefresh = service.activeInstanceRefresh(restoredOther);
+ assertNotNull(otherRefresh);
+ assertEquals("InProgress", otherRefresh.getStatus());
+ assertEquals(1, otherRefresh.getReplacements().size());
+ assertEquals("i-original", otherRefresh.getReplacements().getFirst().getOriginalInstanceId());
+ assertEquals(1, restoredOther.getInstances().size());
+ verify(ec2, never()).terminateInstances(eq(otherRegion), anyList());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ @SuppressWarnings("unchecked")
+ void pausedSelectionCannotBeTerminatedBeforeOwnershipPublication(String mode) throws Exception {
+ setUp(mode);
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ service.createAutoScalingGroup(REGION, "unrelated-group", null, "lt-original", null, "1", null,
+ 0, 3, 0, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,
+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());
+ AutoScalingGroup unrelated = service.describeAutoScalingGroups(REGION, List.of("unrelated-group")).getFirst();
+ Field field = AutoScalingService.class.getDeclaredField("instanceRefreshes");
+ field.setAccessible(true);
+ Map<String, InstanceRefresh> original = (Map<String, InstanceRefresh>) field.get(service);
+ Map<String, InstanceRefresh> intercepted = spy(original);
+ CountDownLatch selected = new CountDownLatch(1);
+ CountDownLatch publish = new CountDownLatch(1);
+ CountDownLatch reconciling = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ selected.countDown();
+ assertTrue(publish.await(5, TimeUnit.SECONDS));
+ return original.put(invocation.getArgument(0), invocation.getArgument(1));
+ }).when(intercepted).put(anyString(), any(InstanceRefresh.class));
+ field.set(service, intercepted);
+ ExecutorService executor = Executors.newFixedThreadPool(3);
+ Future<?> starting = executor.submit(() ->
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh()));
+ Future<?> reconcilingFuture = null;
+ try {
+ assertTrue(selected.await(5, TimeUnit.SECONDS));
+ reconcilingFuture = executor.submit(() -> {
+ reconciling.countDown();
+ reconciler.reconcile(group);
+ });
+ assertTrue(reconciling.await(5, TimeUnit.SECONDS));
+ try {
+ reconcilingFuture.get(250, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException expected) {
+ // Correct group coordination waits for ownership publication before reconciliation.
+ }
+ executor.submit(() -> reconciler.reconcile(unrelated)).get(5, TimeUnit.SECONDS);
+ verify(ec2, never()).terminateInstances(anyString(), anyList());
+ } finally {
+ publish.countDown();
+ try {
+ starting.get(5, TimeUnit.SECONDS);
+ if (reconcilingFuture != null) {
+ reconcilingFuture.get(5, TimeUnit.SECONDS);
+ }
+ } finally {
+ field.set(service, original);
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"persistent,true", "persistent,false", "hybrid,true", "hybrid,false", "wal,true", "wal,false"})
+ @SuppressWarnings("unchecked")
+ void explicitRemovalWaitsForRefreshOwnershipPublication(String mode, boolean detach) throws Exception {
+ setUp(mode);
+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));
+ Field field = AutoScalingService.class.getDeclaredField("instanceRefreshes");
+ field.setAccessible(true);
+ Map<String, InstanceRefresh> original = (Map<String, InstanceRefresh>) field.get(service);
+ Map<String, InstanceRefresh> intercepted = spy(original);
+ CountDownLatch selected = new CountDownLatch(1);
+ CountDownLatch publish = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ selected.countDown();
+ assertTrue(publish.await(5, TimeUnit.SECONDS));
+ return original.put(invocation.getArgument(0), invocation.getArgument(1));
+ }).when(intercepted).put(anyString(), any(InstanceRefresh.class));
+ field.set(service, intercepted);
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ Future<?> starting = executor.submit(() ->
+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh()));
+ Future<?> removing = null;
+ try {
+ assertTrue(selected.await(5, TimeUnit.SECONDS));
+ CountDownLatch removalStarted = new CountDownLatch(1);
+ removing = executor.submit(() -> {
+ removalStarted.countDown();
+ if (detach) {
+ service.detachInstances(REGION, "suspended-group", List.of("i-original"), true);
+ } else {
+ service.terminateInstanceInAutoScalingGroup(REGION, "i-original", true);
+ }
+ });
+ assertTrue(removalStarted.await(5, TimeUnit.SECONDS));
+ try {
+ removing.get(250, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException expected) {
+ // Removal may wait for the shared group monitor before releasing ownership.
+ }
+ } finally {
+ publish.countDown();
+ try {
+ starting.get(5, TimeUnit.SECONDS);
+ if (removing != null) {
+ removing.get(5, TimeUnit.SECONDS);
+ }
+ } finally {
+ field.set(service, original);
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+ InstanceRefresh refresh = service.activeInstanceRefresh(group);
+ assertNotNull(refresh);
+ assertTrue(refresh.getReplacements().isEmpty(), "Explicit removal retained refresh ownership");
+ restart(mode);
+ reconciler.reconcile(group);
+ assertTrue(group.getInstances().isEmpty());
+ if (detach) {
+ verify(ec2, never()).terminateInstances(anyString(), anyList());
+ } else {
+ verify(ec2).terminateInstances(REGION, List.of("i-original"));
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"persistent,true", "persistent,false", "hybrid,true", "hybrid,false", "wal,true", "wal,false"})
+ void detachWaitsForReplacementIdentityPublication(String mode, boolean decrement) throws Exception {
+ setUp(mode);
+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ service = spy(service);
+ reconciler = new AutoScalingReconciler(service, ec2, mock(ElbV2Service.class));
+ CountDownLatch launched = new CountDownLatch(1);
+ CountDownLatch publish = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ if (group.getInstances().stream().anyMatch(i -> "i-replacement".equals(i.getInstanceId()))
+ && refresh.getReplacements().getFirst().getReplacementInstanceId() == null) {
+ launched.countDown();
+ assertTrue(publish.await(5, TimeUnit.SECONDS));
+ }
+ return invocation.callRealMethod();
+ }).when(service).saveAutoScalingGroupIfPresent(group);
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ Future<?> reconciling = executor.submit(() -> reconciler.reconcile(group));
+ Future<?> detaching = null;
+ try {
+ assertTrue(launched.await(5, TimeUnit.SECONDS));
+ CountDownLatch detachStarted = new CountDownLatch(1);
+ detaching = executor.submit(() -> {
+ detachStarted.countDown();
+ service.detachInstances(REGION, "suspended-group", List.of("i-replacement"), decrement);
+ });
+ assertTrue(detachStarted.await(5, TimeUnit.SECONDS));
+ try {
+ detaching.get(250, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException expected) {
+ // Detach may wait until the replacement ID is published under the group monitor.
+ }
+ } finally {
+ publish.countDown();
+ try {
+ reconciling.get(5, TimeUnit.SECONDS);
+ if (detaching != null) {
+ detaching.get(5, TimeUnit.SECONDS);
+ }
+ } finally {
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+ assertTrue(group.getInstances().isEmpty());
+ assertTrue(refresh.getReplacements().isEmpty(), "Detached replacement retained refresh ownership");
+ restart(mode);
+ assertTrue(service.activeInstanceRefresh(group).getReplacements().isEmpty());
+ assertEquals(decrement ? 0 : 1, group.getDesiredCapacity());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ void deletingSameNameOtherRegionPreservesPausedRefresh(String mode) throws IOException {
+ setUp(mode);
+ String otherRegion = "us-west-2";
+ LaunchTemplate template = new LaunchTemplate();
+ template.setLaunchTemplateId("lt-original");
+ template.getData().setImageId("ami-original");
+ when(ec2.describeLaunchTemplateVersions(eq(otherRegion), eq("lt-original"), isNull(), anyList()))
+ .thenReturn(List.of(template));
+ service.createAutoScalingGroup(otherRegion, "suspended-group", null, "lt-original", null, "1", null,
+ 0, 3, 1, 300, List.of("us-west-2a"), List.of(), List.of(), List.of(), "EC2", 0,
+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());
+ AutoScalingGroup other = service.describeAutoScalingGroups(otherRegion, List.of("suspended-group"))
+ .getFirst();
+ AsgInstance member = new AsgInstance();
+ member.setInstanceId("i-west");
+ member.setLifecycleState("InService");
+ other.getInstances().add(member);
+ service.saveAutoScalingGroupIfPresent(other);
+ service.suspendProcesses(otherRegion, "suspended-group", List.of("InstanceRefresh"));
+ InstanceRefresh refresh = service.startInstanceRefresh(otherRegion, "suspended-group", new InstanceRefresh());
+ service.deleteAutoScalingGroup(REGION, "suspended-group", true);
+ assertSame(refresh, service.activeInstanceRefresh(other), "Peer deletion erased refresh ownership");
+ when(ec2.describeInstances(eq(otherRegion), anyList(), isNull())).thenReturn(List.of(reservation("i-west")));
+ reconciler.reconcile(other);
+ verify(ec2, never()).terminateInstances(eq(otherRegion), anyList());
+ assertEquals(List.of("i-west"), other.getInstances().stream().map(AsgInstance::getInstanceId).toList());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "hybrid", "wal"})
+ @SuppressWarnings("unchecked")
+ void groupRecreationWaitsForOldRefreshCleanup(String mode) throws Exception {
+ setUp(mode);
+ Field field = AutoScalingService.class.getDeclaredField("instanceRefreshes");
+ field.setAccessible(true);
+ Map<String, InstanceRefresh> original = (Map<String, InstanceRefresh>) field.get(service);
+ Map<String, InstanceRefresh> intercepted = spy(original);
+ CountDownLatch cleanup = new CountDownLatch(1);
+ CountDownLatch finish = new CountDownLatch(1);
+ AtomicBoolean firstCleanup = new AtomicBoolean(true);
+ doAnswer(invocation -> {
+ if (firstCleanup.compareAndSet(true, false)) {
+ cleanup.countDown();
+ assertTrue(finish.await(5, TimeUnit.SECONDS));
+ }
+ return original.entrySet();
+ }).when(intercepted).entrySet();
+ field.set(service, intercepted);
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ Future<?> deleting = executor.submit(() -> service.deleteAutoScalingGroup(REGION, "suspended-group", true));
+ Future<InstanceRefresh> recreating = null;
+ InstanceRefresh replacement;
+ try {
+ assertTrue(cleanup.await(5, TimeUnit.SECONDS));
+ CountDownLatch creationStarted = new CountDownLatch(1);
+ recreating = executor.submit(() -> {
+ creationStarted.countDown();
+ service.createAutoScalingGroup(REGION, "suspended-group", null, "lt-original", null, "1", null,
+ 0, 3, 0, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,
+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());
+ return service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());
+ });
+ assertTrue(creationStarted.await(5, TimeUnit.SECONDS));
+ try {
+ recreating.get(250, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException expected) {
+ // Recreating the same group may wait for the old generation's refresh cleanup.
+ }
+ } finally {
+ finish.countDown();
+ try {
+ deleting.get(5, TimeUnit.SECONDS);
+ replacement = recreating == null ? null : recreating.get(5, TimeUnit.SECONDS);
+ } finally {
+ field.set(service, original);
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+ assertNotNull(replacement);
+ assertEquals(List.of(replacement.getInstanceRefreshId()), service.describeInstanceRefreshes(
+ REGION, "suspended-group", null, null, null).instanceRefreshes().stream()
+ .map(InstanceRefresh::getInstanceRefreshId).toList());
+ }
+
+ private static Reservation reservation(String id) {
+ Instance instance = new Instance();
+ instance.setInstanceId(id);
+ instance.setState(InstanceState.running());
+ Reservation reservation = new Reservation();
+ reservation.setInstances(List.of(instance));
+ return reservation;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshSuspensionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshSuspensionIntegrationTest.java
new file mode 100644
index 0000000000..3ca048910e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshSuspensionIntegrationTest.java
@@ -0,0 +1,284 @@
+package io.github.hectorvent.floci.services.autoscaling;
+
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import io.restassured.specification.RequestSpecification;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+@QuarkusTest
+class AutoScalingRefreshSuspensionIntegrationTest {
+ @Inject
+ AutoScalingReconciler reconciler;
+
+ @Inject
+ Ec2Service ec2;
+
+ private static final String GROUP = "DescribeAutoScalingGroupsResponse."
+ + "DescribeAutoScalingGroupsResult.AutoScalingGroups.member";
+ private static final String REFRESH = "DescribeInstanceRefreshesResponse."
+ + "DescribeInstanceRefreshesResult.InstanceRefreshes.member";
+
+ @ParameterizedTest
+ @ValueSource(strings = {"Launch", "Terminate", "InstanceRefresh", "all"})
+ void pausedRefreshKeepsOriginalAndResumesToOneHealthyReplacement(String process) {
+ String name = "refresh-pause-" + UUID.randomUUID().toString().substring(0, 8);
+ request("CreateLaunchConfiguration").formParam("LaunchConfigurationName", name)
+ .formParam("ImageId", "ami-amazonlinux2023").formParam("InstanceType", "t3.micro")
+ .post("/").then().statusCode(200);
+ try {
+ request("CreateAutoScalingGroup").formParam("AutoScalingGroupName", name)
+ .formParam("LaunchConfigurationName", name).formParam("MinSize", 0)
+ .formParam("MaxSize", 1).formParam("DesiredCapacity", 1)
+ .formParam("AvailabilityZones.member.1", "us-east-1a")
+ .post("/").then().statusCode(200);
+ try {
+ reconcileTwice();
+ assertEquals("InService", group(name).xmlPath().getString(GROUP + ".Instances.member.LifecycleState"));
+ String originalId = group(name).xmlPath().getString(GROUP + ".Instances.member.InstanceId");
+ process("SuspendProcesses", name, process);
+ String refreshId = request("StartInstanceRefresh").formParam("AutoScalingGroupName", name)
+ .post("/").then().statusCode(200).extract().xmlPath()
+ .getString("StartInstanceRefreshResponse.StartInstanceRefreshResult.InstanceRefreshId");
+ reconcileTwice();
+ Response paused = group(name);
+ assertEquals(1, paused.xmlPath().getInt(GROUP + ".DesiredCapacity"));
+ assertEquals(1, paused.xmlPath().getList(GROUP + ".Instances.member").size());
+ assertEquals(originalId, paused.xmlPath().getString(GROUP + ".Instances.member.InstanceId"));
+ assertEquals("running", instanceState(originalId));
+ Response refresh = refresh(name, refreshId);
+ assertEquals("InProgress", refresh.xmlPath().getString(REFRESH + ".Status"));
+ assertEquals(1, refresh.xmlPath().getInt(REFRESH + ".InstancesToUpdate"));
+ process("ResumeProcesses", name, process);
+ reconcileTwice();
+ assertEquals("Successful", refresh(name, refreshId).xmlPath().getString(REFRESH + ".Status"));
+ Response resumed = group(name);
+ assertEquals(1, resumed.xmlPath().getList(GROUP + ".Instances.member").size());
+ assertNotEquals(originalId, resumed.xmlPath().getString(GROUP + ".Instances.member.InstanceId"));
+ assertEquals("InService", resumed.xmlPath().getString(GROUP + ".Instances.member.LifecycleState"));
+ assertEquals("terminated", instanceState(originalId));
+ } finally {
+ request("DeleteAutoScalingGroup").formParam("AutoScalingGroupName", name)
+ .formParam("ForceDelete", true).post("/").then().statusCode(200);
+ }
+ } finally {
+ request("DeleteLaunchConfiguration").formParam("LaunchConfigurationName", name)
+ .post("/").then().statusCode(200);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"configuration", "template"})
+ void replacementRetainsEncodedUserDataAndSelectsTheCurrentSubnet(String source) {
+ String name = "refresh-placement-" + UUID.randomUUID().toString().substring(0, 8);
+ String encoded = "/w";
+ String vpcId = ec2.createVpc("us-east-1", "10.92.0.0/16", false).getVpcId();
+ List<String> subnets = new ArrayList<>();
+ try {
+ subnets.add(ec2.createSubnet("us-east-1", vpcId, "10.92.0.0/28", "us-east-1a").getSubnetId());
+ subnets.add(ec2.createSubnet("us-east-1", vpcId, "10.92.1.0/24", "us-east-1b").getSubnetId());
+ boolean configuration = "configuration".equals(source);
+ RequestSpecification create = given().header("Authorization", auth(configuration ? "autoscaling" : "ec2"))
+ .formParam("Action", configuration ? "CreateLaunchConfiguration" : "CreateLaunchTemplate")
+ .formParam(configuration ? "LaunchConfigurationName" : "LaunchTemplateName", name);
+ String prefix = configuration ? "" : "LaunchTemplateData.";
+ if (!configuration) {
+ create.formParam("LaunchTemplateData.BlockDeviceMapping.1.DeviceName", "/dev/xvda")
+ .formParam("LaunchTemplateData.BlockDeviceMapping.1.Ebs.VolumeType", "gp2");
+ }
+ create.formParam(prefix + "ImageId", "ami-amazonlinux2023")
+ .formParam(prefix + "InstanceType", "t3.micro").formParam(prefix + "UserData", encoded)
+ .post("/").then().statusCode(200);
+ try {
+ RequestSpecification createGroup = request("CreateAutoScalingGroup")
+ .formParam("AutoScalingGroupName", name).formParam("MinSize", 0)
+ .formParam("MaxSize", 1).formParam("DesiredCapacity", 1)
+ .formParam("VPCZoneIdentifier", subnets.getFirst())
+ .formParam("AvailabilityZones.member.1", "us-east-1a");
+ if (configuration) {
+ createGroup.formParam("LaunchConfigurationName", name);
+ } else {
+ createGroup.formParam("LaunchTemplate.LaunchTemplateName", name)
+ .formParam("LaunchTemplate.Version", "1");
+ }
+ createGroup.post("/").then().statusCode(200);
+ try {
+ reconcileTwice();
+ String original = group(name).xmlPath().getString(GROUP + ".Instances.member.InstanceId");
+ assertEquals(subnets.getFirst(), ec2.getInstance("us-east-1", original).orElseThrow().getSubnetId());
+ assertEquals(configuration ? "gp3" : "gp2", rootVolumeType(original));
+ request("UpdateAutoScalingGroup").formParam("AutoScalingGroupName", name)
+ .formParam("VPCZoneIdentifier", String.join(",", subnets))
+ .post("/").then().statusCode(200);
+ String refreshId = request("StartInstanceRefresh").formParam("AutoScalingGroupName", name)
+ .post("/").then().statusCode(200).extract().xmlPath()
+ .getString("StartInstanceRefreshResponse.StartInstanceRefreshResult.InstanceRefreshId");
+ reconcileTwice();
+ Response current = group(name);
+ assertEquals(1, current.xmlPath().getList(GROUP + ".Instances.member").size());
+ String replacement = current.xmlPath().getString(GROUP + ".Instances.member.InstanceId");
+ assertNotEquals(original, replacement);
+ assertEquals(configuration ? "gp3" : "gp2", rootVolumeType(replacement));
+ assertEquals("terminated", instanceState(original));
+ assertEquals(subnets.getLast(), ec2.getInstance("us-east-1", replacement).orElseThrow().getSubnetId());
+ assertEquals("us-east-1b", current.xmlPath().getString(GROUP + ".Instances.member.AvailabilityZone"));
+ assertEquals("Successful", refresh(name, refreshId).xmlPath().getString(REFRESH + ".Status"));
+ assertEquals(encoded, given().header("Authorization", auth("ec2"))
+ .formParam("Action", "DescribeInstanceAttribute").formParam("InstanceId", replacement)
+ .formParam("Attribute", "userData").post("/").then().statusCode(200)
+ .extract().xmlPath().getString("DescribeInstanceAttributeResponse.userData.value"));
+ } finally {
+ request("DeleteAutoScalingGroup").formParam("AutoScalingGroupName", name)
+ .formParam("ForceDelete", true).post("/").then().statusCode(200);
+ }
+ } finally {
+ given().header("Authorization", auth(configuration ? "autoscaling" : "ec2"))
+ .formParam("Action", configuration ? "DeleteLaunchConfiguration" : "DeleteLaunchTemplate")
+ .formParam(configuration ? "LaunchConfigurationName" : "LaunchTemplateName", name)
+ .post("/").then().statusCode(200);
+ }
+ } finally {
+ for (String subnet : subnets) {
+ ec2.deleteSubnet("us-east-1", subnet);
+ }
+ ec2.deleteVpc("us-east-1", vpcId);
+ }
+ }
+
+ @Test
+ void sameNameRefreshesInDifferentAccountsResumeIndependently() {
+ String name = "refresh-accounts-" + UUID.randomUUID().toString().substring(0, 8);
+ List<String> accounts = List.of("111122223333", "444455556666");
+ List<String> configurations = new ArrayList<>();
+ List<String> groups = new ArrayList<>();
+ List<String> originals = new ArrayList<>();
+ try {
+ for (String account : accounts) {
+ accountRequest(account, "CreateLaunchConfiguration")
+ .formParam("LaunchConfigurationName", name).formParam("ImageId", "ami-amazonlinux2023")
+ .formParam("InstanceType", "t3.micro").post("/").then().statusCode(200);
+ configurations.add(account);
+ accountRequest(account, "CreateAutoScalingGroup").formParam("AutoScalingGroupName", name)
+ .formParam("LaunchConfigurationName", name).formParam("MinSize", 0)
+ .formParam("MaxSize", 1).formParam("DesiredCapacity", 1)
+ .formParam("AvailabilityZones.member.1", "us-east-1a").post("/").then().statusCode(200);
+ groups.add(account);
+ }
+ reconcileTwice();
+ for (String account : accounts) {
+ originals.add(accountGroup(account, name).xmlPath().getString(GROUP + ".Instances.member.InstanceId"));
+ accountRequest(account, "SuspendProcesses").formParam("AutoScalingGroupName", name)
+ .formParam("ScalingProcesses.member.1", "InstanceRefresh").post("/").then().statusCode(200);
+ accountRequest(account, "StartInstanceRefresh").formParam("AutoScalingGroupName", name)
+ .post("/").then().statusCode(200);
+ }
+ assertNotEquals(originals.getFirst(), originals.getLast());
+ reconcileTwice();
+ for (int index = 0; index < accounts.size(); index++) {
+ assertEquals(originals.get(index), accountGroup(accounts.get(index), name)
+ .xmlPath().getString(GROUP + ".Instances.member.InstanceId"));
+ }
+ accountRequest(accounts.getFirst(), "ResumeProcesses").formParam("AutoScalingGroupName", name)
+ .formParam("ScalingProcesses.member.1", "InstanceRefresh").post("/").then().statusCode(200);
+ reconcileTwice();
+ assertNotEquals(originals.getFirst(), accountGroup(accounts.getFirst(), name)
+ .xmlPath().getString(GROUP + ".Instances.member.InstanceId"));
+ assertEquals("Successful", accountRefreshStatus(accounts.getFirst(), name));
+ assertEquals(originals.getLast(), accountGroup(accounts.getLast(), name)
+ .xmlPath().getString(GROUP + ".Instances.member.InstanceId"));
+ assertEquals("InProgress", accountRefreshStatus(accounts.getLast(), name));
+ accountRequest(accounts.getLast(), "ResumeProcesses").formParam("AutoScalingGroupName", name)
+ .formParam("ScalingProcesses.member.1", "InstanceRefresh").post("/").then().statusCode(200);
+ reconcileTwice();
+ assertNotEquals(originals.getLast(), accountGroup(accounts.getLast(), name)
+ .xmlPath().getString(GROUP + ".Instances.member.InstanceId"));
+ assertEquals("Successful", accountRefreshStatus(accounts.getLast(), name));
+ } finally {
+ try {
+ for (String account : groups) {
+ accountRequest(account, "DeleteAutoScalingGroup").formParam("AutoScalingGroupName", name)
+ .formParam("ForceDelete", true).post("/").then().statusCode(200);
+ }
+ } finally {
+ for (String account : configurations) {
+ accountRequest(account, "DeleteLaunchConfiguration").formParam("LaunchConfigurationName", name)
+ .post("/").then().statusCode(200);
+ }
+ }
+ }
+ }
+
+ private String rootVolumeType(String instanceId) {
+ String volumeId = ec2.getInstance("us-east-1", instanceId).orElseThrow().getRootVolumeId();
+ return given().header("Authorization", auth("ec2")).formParam("Action", "DescribeVolumes")
+ .formParam("VolumeId.1", volumeId).post("/").then().statusCode(200)
+ .extract().xmlPath().getString("DescribeVolumesResponse.volumeSet.item.volumeType");
+ }
+
+ private static String accountRefreshStatus(String account, String name) {
+ return accountRequest(account, "DescribeInstanceRefreshes").formParam("AutoScalingGroupName", name)
+ .post("/").then().statusCode(200).extract().xmlPath().getString(REFRESH + ".Status");
+ }
+
+ private static Response accountGroup(String account, String name) {
+ return accountRequest(account, "DescribeAutoScalingGroups").formParam("AutoScalingGroupNames.member.1", name)
+ .post("/").then().statusCode(200).extract().response();
+ }
+
+ private static RequestSpecification accountRequest(String account, String action) {
+ return given().header("Authorization", "AWS4-HMAC-SHA256 Credential=" + account
+ + "/20260215/us-east-1/autoscaling/aws4_request, SignedHeaders=host, Signature=abc")
+ .formParam("Action", action);
+ }
+
+ private void reconcileTwice() {
+ reconciler.reconcileAll();
+ reconciler.reconcileAll();
+ }
+
+ private static Response group(String name) {
+ return request("DescribeAutoScalingGroups").formParam("AutoScalingGroupNames.member.1", name)
+ .post("/").then().statusCode(200).extract().response();
+ }
+
+ private static Response refresh(String name, String refreshId) {
+ return request("DescribeInstanceRefreshes").formParam("AutoScalingGroupName", name)
+ .formParam("InstanceRefreshIds.member.1", refreshId)
+ .post("/").then().statusCode(200).extract().response();
+ }
+
+ private static String instanceState(String id) {
+ return given().header("Authorization", auth("ec2"))
+ .formParam("Action", "DescribeInstances").formParam("InstanceId.1", id)
+ .post("/").then().statusCode(200).extract().xmlPath()
+ .getString("DescribeInstancesResponse.reservationSet.item.instancesSet.item.instanceState.name");
+ }
+
+ private static void process(String action, String name, String process) {
+ RequestSpecification request = request(action).formParam("AutoScalingGroupName", name);
+ if (!"all".equals(process)) {
+ request.formParam("ScalingProcesses.member.1", process);
+ }
+ request.post("/").then().statusCode(200);
+ }
+
+ private static RequestSpecification request(String action) {
+ return given().header("Authorization", auth("autoscaling")).formParam("Action", action);
+ }
+
+ private static String auth(String service) {
+ return "AWS4-HMAC-SHA256 Credential=AKID/20260215/us-east-1/" + service
+ + "/aws4_request, SignedHeaders=host, Signature=abc";
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java
index de792100f3..502de53a4c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java
@@ -542,6 +542,7 @@ class AutoScalingServiceTest {
group.getInstances().clear();
AutoScalingGroupFixture.addInstance(service, REGION, "test-asg", "i-replacement", "InService", "lt-original", "2");
+ refresh.getReplacements().getFirst().setReplacementInstanceId("i-replacement");
service.completeInstanceRefreshIfSettled(REGION, "test-asg");
InstanceRefresh completed = service.describeInstanceRefreshes(
@@ -569,6 +570,7 @@ class AutoScalingServiceTest {
group.getInstances().clear();
AutoScalingGroupFixture.addInstance(service, REGION, "test-asg", "i-replacement", "InService", "lt-original", "2");
+ refresh.getReplacements().getFirst().setReplacementInstanceId("i-replacement");
service.completeInstanceRefreshIfSettled(REGION, "test-asg");
InstanceRefresh completed = service.describeInstanceRefreshes(
@@ -622,6 +624,7 @@ class AutoScalingServiceTest {
group.getInstances().clear();
AutoScalingGroupFixture.addInstance(service, REGION, "test-asg", "i-replacement", "InService", "lt-updated", "2");
+ refresh.getReplacements().getFirst().setReplacementInstanceId("i-replacement");
service.completeInstanceRefreshIfSettled(REGION, "test-asg");
InstanceRefresh completed = service.describeInstanceRefreshes(