Light, fluffy, and always free - The AWS Local Emulator alternative
fix(autoscaling): preserve paused refresh ownership (#5668)
David Phillipsb395560fe229
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingRefreshOwnershipTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingRefreshOwnershipTest.javanew file mode 100644index 0000000000..7277936b75--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AutoScalingRefreshOwnershipTest.java@@ -0,0 +1,95 @@+package com.floci.test;++import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import software.amazon.awssdk.services.autoscaling.AutoScalingClient;+import software.amazon.awssdk.services.autoscaling.model.AutoScalingException;+import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup;+import software.amazon.awssdk.services.autoscaling.model.InstanceRefresh;+import software.amazon.awssdk.services.autoscaling.model.ResumeProcessesRequest;+import software.amazon.awssdk.services.autoscaling.model.SuspendProcessesRequest;+import software.amazon.awssdk.services.ec2.Ec2Client;+import software.amazon.awssdk.services.ec2.model.InstanceType;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++class AutoScalingRefreshOwnershipTest {++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ @DisplayName("A paused refresh exposes its selected work and resumes through the SDK")+ void pausedRefreshMetadataAndResumeRoundTrip(boolean allProcesses) {+ String name = TestFixtures.uniqueName("refresh-ownership");+ try (AutoScalingClient autoScaling = TestFixtures.autoScalingClient();+ Ec2Client ec2 = TestFixtures.ec2Client()) {+ autoScaling.createLaunchConfiguration(request -> request.launchConfigurationName(name)+ .imageId("ami-amazonlinux2023").instanceType("t3.micro"));+ try {+ String instanceId = ec2.runInstances(request -> request.imageId("ami-amazonlinux2023")+ .instanceType(InstanceType.T3_MICRO).minCount(1).maxCount(1))+ .instances().get(0).instanceId();+ try {+ autoScaling.createAutoScalingGroup(request -> request.autoScalingGroupName(name)+ .launchConfigurationName(name).minSize(0).maxSize(1).desiredCapacity(0)+ .availabilityZones("us-east-1a"));+ try {+ SuspendProcessesRequest.Builder suspend = SuspendProcessesRequest.builder()+ .autoScalingGroupName(name);+ if (!allProcesses) {+ suspend.scalingProcesses("InstanceRefresh");+ }+ autoScaling.suspendProcesses(suspend.build());+ autoScaling.attachInstances(request -> request.autoScalingGroupName(name)+ .instanceIds(instanceId));+ String refreshId = autoScaling.startInstanceRefresh(request ->+ request.autoScalingGroupName(name)).instanceRefreshId();+ assertThat(refreshId).isNotBlank();+ InstanceRefresh refresh = autoScaling.describeInstanceRefreshes(request ->+ request.autoScalingGroupName(name).instanceRefreshIds(refreshId))+ .instanceRefreshes().get(0);+ assertThat(refresh.instanceRefreshId()).isEqualTo(refreshId);+ assertThat(refresh.autoScalingGroupName()).isEqualTo(name);+ assertThat(refresh.statusAsString()).isEqualTo("InProgress");+ assertThat(refresh.startTime()).isNotNull();+ assertThat(refresh.instancesToUpdate()).isEqualTo(1);+ assertThat(refresh.percentageComplete()).isZero();+ AutoScalingGroup group = autoScaling.describeAutoScalingGroups(request ->+ request.autoScalingGroupNames(name)).autoScalingGroups().get(0);+ assertThat(group.desiredCapacity()).isEqualTo(1);+ assertThat(group.instances()).hasSize(1);+ assertThat(group.instances().get(0).instanceId()).isEqualTo(instanceId);+ assertThat(group.suspendedProcesses()).extracting(process -> process.processName())+ .contains("InstanceRefresh");+ assertThatThrownBy(() -> autoScaling.startInstanceRefresh(request ->+ request.autoScalingGroupName(name)))+ .isInstanceOfSatisfying(AutoScalingException.class, error -> {+ assertThat(error.statusCode()).isEqualTo(400);+ assertThat(error.awsErrorDetails().errorCode()).isEqualTo("InstanceRefreshInProgress");+ });+ assertThat(autoScaling.describeInstanceRefreshes(request ->+ request.autoScalingGroupName(name)).instanceRefreshes())+ .extracting(InstanceRefresh::instanceRefreshId).containsExactly(refreshId);+ ResumeProcessesRequest.Builder resume = ResumeProcessesRequest.builder()+ .autoScalingGroupName(name);+ if (!allProcesses) {+ resume.scalingProcesses("InstanceRefresh");+ }+ autoScaling.resumeProcesses(resume.build());+ assertThat(autoScaling.describeAutoScalingGroups(request ->+ request.autoScalingGroupNames(name)).autoScalingGroups().get(0).suspendedProcesses())+ .isEmpty();+ } finally {+ autoScaling.deleteAutoScalingGroup(request -> request.autoScalingGroupName(name)+ .forceDelete(true));+ }+ } finally {+ ec2.terminateInstances(request -> request.instanceIds(instanceId));+ }+ } finally {+ autoScaling.deleteLaunchConfiguration(request -> request.launchConfigurationName(name));+ }+ }+ }+}diff --git a/docs/services/autoscaling.md b/docs/services/autoscaling.mdindex bd92cb1b59..c2c0fd3956 100644--- a/docs/services/autoscaling.md+++ b/docs/services/autoscaling.md@@ -49,8 +49,12 @@ after a launch error and rebalancing without a capacity increase are not implemeAWS launch errors appear as failed scaling activities after preserving partiallaunches. Repeated identical failures reuse the recorded activity, following the-scale-in behavior, and do not complete an instance refresh. Unexpected runtime,-save and cleanup failures still propagate.+scale-in behavior, and do not complete an instance refresh. Unexpected runtime+failures in ordinary scale-out, and save and cleanup failures, still propagate.+Refresh launch failures retain the original replacement attempt and appear as+failed scaling activities while other replacements can continue. Refresh and+ordinary launches share the same launch settings, including template block-device+mappings; accepted refresh launches keep their recorded source identity.For groups configured only with Availability Zones, Floci preserves its unpinnedEC2 launch behavior and counts each member under the requested zone. This permits@@ -69,6 +73,27 @@ EC2 instance can use a default subnet in a different zone.| `SuspendProcesses` | Records suspended scaling processes (empty list suspends all); reported back by `DescribeAutoScalingGroups` || `ResumeProcesses` | Clears suspended processes (empty list resumes all) |+### Instance Refresh Recovery++New instance refreshes retain the selected instances and their replacement progress+across persistent, hybrid, and WAL storage reloads. Suspending `InstanceRefresh`,+`Launch`, or `Terminate` pauses refresh replacements; resuming those processes lets+the refresh continue. A failed replacement launch remains part of the refresh, so+ordinary desired-capacity reconciliation does not bypass the pause. Explicit+`TerminateInstanceInAutoScalingGroup` requests still run, and independent capacity+increases can proceed while only `InstanceRefresh` is suspended. Paused original+instances keep their target-group registrations until replacement resumes.+Explicitly detached instances leave refresh ownership and are not adopted again+when the refresh resumes. Recovery of an accepted EC2 launch keeps the source+identity recorded for that attempt even if the group configuration has changed.+Older attempts without a recorded source use the available EC2 instance metadata+without assigning the current launch template to the recovered instance.++This does not implement rolling health-percentage limits, checkpoints, rollback,+or cancellation. Refreshes saved by older versions without replacement ownership+retain the previous reconciliation behavior. Storage reload coverage does not+imply an atomic transaction spanning Auto Scaling and EC2 after a process crash.+### Load Balancer Attachment| Operation | Notes |@@ -155,8 +180,9 @@ adjustments. Suspension and resumption wait for any desired-capacity adjustmentin progress for that group, so a completed suspension cannot be bypassed by an earlierreconciliation check.-Suspension does not yet pause instance-refresh selection or replacement cleanup.-Explicitly requested termination and stale-instance cleanup continue independently.+New owned refresh replacements also pause for `InstanceRefresh`, `Launch`, or+`Terminate`, as described above. Explicitly requested termination and stale-instance+cleanup continue independently.Other suspended process names are stored and returned but do not pause theirassociated activities.diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.javaindex c698b48f38..c5323150fe 100644--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java@@ -4,6 +4,9 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.services.autoscaling.model.AsgInstance;import io.github.hectorvent.floci.services.autoscaling.model.AutoScalingGroup;+import io.github.hectorvent.floci.services.autoscaling.model.InstanceRefresh;+import io.github.hectorvent.floci.services.autoscaling.model.InstanceRefreshLaunch;+import io.github.hectorvent.floci.services.autoscaling.model.InstanceRefreshReplacement;import io.github.hectorvent.floci.services.autoscaling.model.LaunchConfiguration;import io.github.hectorvent.floci.services.autoscaling.model.MixedInstancesPolicy;import io.github.hectorvent.floci.services.autoscaling.model.ScalingActivity;@@ -130,17 +133,18 @@ public class AutoScalingReconciler {}public void reconcile(AutoScalingGroup asg) {- removeTerminatingInstances(asg);- removeStaleInstances(asg);- removeOrphanedTargetRegistrations(asg);- promoteReadyInstances(asg);+ asgService.reconcileIfPresent(asg, () -> {+ removeTerminatingInstances(asg);+ removeStaleInstances(asg);+ removeOrphanedTargetRegistrations(asg);+ promoteReadyInstances(asg);+ reconcileRefreshReplacements(asg);- asgService.adjustCapacityIfPresent(asg, () -> {long activeCapacity = activeCapacity(asg);+ long reservedCapacity = reservedRefreshCapacity(asg);int desired = asg.getDesiredCapacity();-- if (activeCapacity < desired && !asg.getSuspendedProcesses().contains("Launch")) {- if (!scaleOut(asg, (int) (desired - activeCapacity))) {+ if (activeCapacity + reservedCapacity < desired && !asg.getSuspendedProcesses().contains("Launch")) {+ if (!scaleOut(asg, (int) (desired - activeCapacity - reservedCapacity))) {return;}} else if (activeCapacity > desired && !asg.getSuspendedProcesses().contains("Terminate")) {@@ -152,6 +156,76 @@ public class AutoScalingReconciler {});}+ private static boolean refreshPaused(AutoScalingGroup asg) {+ return asg.getSuspendedProcesses().contains("InstanceRefresh")+ || asg.getSuspendedProcesses().contains("Launch")+ || asg.getSuspendedProcesses().contains("Terminate");+ }++ private boolean pausedRefreshOwns(AutoScalingGroup asg, String instanceId) {+ InstanceRefresh refresh = asgService.activeInstanceRefresh(asg);+ return refreshPaused(asg) && refresh != null && refresh.getReplacements() != null+ && refresh.getReplacements().stream()+ .anyMatch(replacement -> instanceId.equals(replacement.getOriginalInstanceId()));+ }++ private long reservedRefreshCapacity(AutoScalingGroup asg) {+ InstanceRefresh refresh = asgService.activeInstanceRefresh(asg);+ if (refresh == null || refresh.getReplacements() == null) {+ return 0;+ }+ return refresh.getReplacements().stream()+ .filter(replacement -> !replacement.isComplete())+ .filter(replacement -> asg.getInstances().stream().noneMatch(instance ->+ instance.getInstanceId().equals(replacement.getReplacementInstanceId())+ && isActiveLifecycleState(instance.getLifecycleState())))+ .count();+ }++ private void reconcileRefreshReplacements(AutoScalingGroup asg) {+ InstanceRefresh refresh = asgService.activeInstanceRefresh(asg);+ if (refresh == null || refresh.getReplacements() == null || refreshPaused(asg)) {+ return;+ }+ Map<String, List<Instance>> acceptedLaunches = null;+ for (InstanceRefreshReplacement replacement : refresh.getReplacements()) {+ if (replacement.isComplete()) {+ continue;+ }+ AsgInstance launchedInstance = asg.getInstances().stream()+ .filter(instance -> instance.getInstanceId().equals(replacement.getReplacementInstanceId()))+ .findFirst().orElse(null);+ if (launchedInstance != null) {+ if ("InService".equals(launchedInstance.getLifecycleState())) {+ replacement.setPhase("Complete");+ asgService.saveInstanceRefresh(refresh);+ }+ continue;+ }+ if (asg.getInstances().stream().anyMatch(instance ->+ instance.getInstanceId().equals(replacement.getOriginalInstanceId()))) {+ continue;+ }+ if (acceptedLaunches == null) {+ acceptedLaunches = refreshLaunches(asg);+ }+ List<Instance> existing = acceptedLaunches.getOrDefault(replacement.getLaunchClientToken(), List.of());+ if (activeCapacity(asg) >= asg.getDesiredCapacity() && existing.isEmpty()) {+ replacement.setPhase("NoLongerRequired");+ asgService.saveInstanceRefresh(refresh);+ continue;+ }+ replacement.setPhase("AwaitingReplacement");+ asgService.saveInstanceRefresh(refresh);+ List<String> launched = launchInstances(asg, refresh, replacement, existing, acceptedLaunches);+ if (!launched.isEmpty()) {+ replacement.setReplacementInstanceId(launched.getFirst());+ replacement.setPhase("Warming");+ asgService.saveInstanceRefresh(refresh);+ }+ }+ }+static long activeCapacity(AutoScalingGroup asg) {return asg.getInstances().stream().filter(i -> {@@ -279,7 +353,8 @@ public class AutoScalingReconciler {}Set<String> activeInstanceIds = asg.getInstances().stream()- .filter(instance -> isActiveLifecycleState(instance.getLifecycleState()))+ .filter(instance -> isActiveLifecycleState(instance.getLifecycleState())+ || pausedRefreshOwns(asg, instance.getInstanceId())).map(AsgInstance::getInstanceId).collect(Collectors.toSet());@@ -308,6 +383,7 @@ public class AutoScalingReconciler {private void removeTerminatingInstances(AutoScalingGroup asg) {List<AsgInstance> terminatingInstances = asg.getInstances().stream().filter(instance -> "Terminating".equals(instance.getLifecycleState()))+ .filter(instance -> !pausedRefreshOwns(asg, instance.getInstanceId())).collect(Collectors.toList());if (terminatingInstances.isEmpty()) {return;@@ -362,45 +438,12 @@ public class AutoScalingReconciler {List<LaunchPlacement> placements = new ArrayList<>(launchPlacements(asg));for (int index = 0; index < count; index++) {LaunchPlacement placement = leastPopulatedPlacement(asg, placements);- Reservation reservation = ec2Service.runInstances(- asg.getRegion(),- launchSource.imageId(),- launchSource.instanceType(),- 1, 1,- launchSource.keyName(),- launchSource.securityGroupIds(),- placement.subnetId(),- null,- propagatedInstanceTags(asg, launchSource),- launchSource.userData(),- launchSource.iamInstanceProfile(),- launchSource.associatePublicIpAddress(), null, 0, null, null, null,- launchSource.encodedUserData() != null ? launchSource.encodedUserData()- : encodeLegacyUserData(launchSource.userData()),- false, null, null, launchSource.blockDeviceMappings());-- for (Instance ec2Inst : reservation.getInstances()) {- AsgInstance asgInst = new AsgInstance();- asgInst.setInstanceId(ec2Inst.getInstanceId());- asgInst.setAvailabilityZone(placement.subnetId() == null && placement.availabilityZone() != null- ? placement.availabilityZone() : ec2Inst.getPlacement() != null- ? ec2Inst.getPlacement().getAvailabilityZone() : placement.availabilityZone());- asgInst.setLifecycleState("Pending");- asgInst.setHealthStatus("Healthy");- asgInst.setLaunchConfigurationName(launchSource.launchConfigurationName());- asgInst.setLaunchTemplateId(launchSource.launchTemplateId());- asgInst.setLaunchTemplateName(launchSource.launchTemplateName());- asgInst.setLaunchTemplateVersion(launchSource.launchTemplateVersion());- asgInst.setInstanceType(launchSource.instanceType());- asg.getInstances().add(asgInst);- launchedInstanceIds.add(ec2Inst.getInstanceId());- LOG.infov("ASG {0}: launched instance {1} (Pending)",- asg.getAutoScalingGroupName(), ec2Inst.getInstanceId());- }+ List<Instance> launched = launchInstance(asg, launchSource, placement, null, null,+ List.of(), launchedInstanceIds, false);if (placement.subnetId() != null) {placements.set(placements.indexOf(placement), new LaunchPlacement(placement.subnetId(),placement.availabilityZone(), Math.max(0,- placement.availableIpAddresses() - reservation.getInstances().size()),+ placement.availableIpAddresses() - launched.size()),placement.requiresIpv4Address()));}}@@ -455,6 +498,114 @@ public class AutoScalingReconciler {}}+ private Map<String, List<Instance>> refreshLaunches(AutoScalingGroup asg) {+ return ec2Service.describeInstances(asg.getRegion(), List.of(), null).stream()+ .flatMap(result -> result.getInstances().stream())+ .filter(instance -> instance.getClientToken() != null)+ .filter(instance -> instance.getState() != null+ && !ABANDONED_LAUNCH_STATES.contains(instance.getState().getName()))+ .collect(Collectors.groupingBy(Instance::getClientToken));+ }++ private List<String> launchInstances(AutoScalingGroup asg, InstanceRefresh refresh,+ InstanceRefreshReplacement replacement, List<Instance> existing,+ Map<String, List<Instance>> acceptedLaunches) {+ String clientToken = replacement.getLaunchClientToken();+ LaunchSource launchSource = existing.isEmpty() ? resolveLaunchSource(asg) : null;+ if (existing.isEmpty() && launchSource == null) {+ LOG.warnv("ASG {0}: no launch source found, cannot scale out", asg.getAutoScalingGroupName());+ return List.of();+ }+ List<String> launchedInstanceIds = new ArrayList<>();+ LaunchPlacement placement = existing.isEmpty() ? leastPopulatedPlacement(asg, launchPlacements(asg)) : null;+ InstanceRefreshLaunch metadata = replacement.getLaunch();+ if (existing.isEmpty()) {+ metadata = new InstanceRefreshLaunch(launchSource.launchConfigurationName(),+ launchSource.launchTemplateId(), launchSource.launchTemplateName(),+ launchSource.launchTemplateVersion(), launchSource.instanceType(), placement.availabilityZone());+ replacement.setLaunch(metadata);+ asgService.saveInstanceRefresh(refresh);+ }+ List<Instance> launched = launchInstance(asg, launchSource, placement, clientToken, metadata,+ existing, launchedInstanceIds, true);+ if (!launched.isEmpty()) {+ acceptedLaunches.put(clientToken, launched);+ preserveScaleOutAllocations(asg, launchedInstanceIds);+ }+ // Cleanup after a removed group releases these allocations rather than publishing ownership.+ launchedInstanceIds.removeIf(id -> asg.getInstances().stream()+ .noneMatch(instance -> id.equals(instance.getInstanceId())));+ return launchedInstanceIds;+ }++ private List<Instance> launchInstance(AutoScalingGroup asg, LaunchSource launchSource,+ LaunchPlacement placement, String clientToken,+ InstanceRefreshLaunch metadata, List<Instance> existing,+ List<String> launchedInstanceIds, boolean refreshLaunch) {+ List<Instance> instances = existing;+ if (instances.isEmpty()) {+ Reservation reservation;+ try {+ reservation = ec2Service.runInstances(+ asg.getRegion(), launchSource.imageId(), launchSource.instanceType(), 1, 1,+ launchSource.keyName(), launchSource.securityGroupIds(), placement.subnetId(), clientToken,+ propagatedInstanceTags(asg, launchSource), launchSource.userData(),+ launchSource.iamInstanceProfile(), launchSource.associatePublicIpAddress(), null, 0, null, null, null,+ launchSource.encodedUserData() != null ? launchSource.encodedUserData()+ : encodeLegacyUserData(launchSource.userData()),+ false, null, null, launchSource.blockDeviceMappings());+ } catch (RuntimeException launchFailure) {+ if (!refreshLaunch) {+ throw launchFailure;+ }+ recordFailedActivity(asg, "Launching a replacement EC2 instance.",+ "An instance refresh requested replacement of active instances.", launchFailure);+ return List.of();+ }+ instances = reservation.getInstances();+ }+ for (Instance instance : instances) {+ launchedInstanceIds.add(instance.getInstanceId());+ }+ for (Instance ec2Inst : instances) {+ if (asg.getInstances().stream().anyMatch(instance ->+ instance.getInstanceId().equals(ec2Inst.getInstanceId()))) {+ continue;+ }+ AsgInstance asgInst = new AsgInstance();+ asgInst.setInstanceId(ec2Inst.getInstanceId());+ asgInst.setLifecycleState("Pending");+ asgInst.setHealthStatus("Healthy");+ if (metadata != null) {+ asgInst.setAvailabilityZone(metadata.availabilityZone() != null ? metadata.availabilityZone()+ : ec2Inst.getPlacement() != null ? ec2Inst.getPlacement().getAvailabilityZone() : null);+ asgInst.setLaunchConfigurationName(metadata.launchConfigurationName());+ asgInst.setLaunchTemplateId(metadata.launchTemplateId());+ asgInst.setLaunchTemplateName(metadata.launchTemplateName());+ asgInst.setLaunchTemplateVersion(metadata.launchTemplateVersion());+ asgInst.setInstanceType(metadata.instanceType());+ } else if (launchSource != null) {+ asgInst.setAvailabilityZone(placement.subnetId() == null && placement.availabilityZone() != null+ ? placement.availabilityZone() : ec2Inst.getPlacement() != null+ ? ec2Inst.getPlacement().getAvailabilityZone() : placement.availabilityZone());+ asgInst.setLaunchConfigurationName(launchSource.launchConfigurationName());+ asgInst.setLaunchTemplateId(launchSource.launchTemplateId());+ asgInst.setLaunchTemplateName(launchSource.launchTemplateName());+ asgInst.setLaunchTemplateVersion(launchSource.launchTemplateVersion());+ asgInst.setInstanceType(launchSource.instanceType());+ } else {+ // Older accepted attempts have no saved source identity; retain actual EC2 metadata.+ asgInst.setInstanceType(ec2Inst.getInstanceType());+ if (ec2Inst.getPlacement() != null) {+ asgInst.setAvailabilityZone(ec2Inst.getPlacement().getAvailabilityZone());+ }+ }+ asg.getInstances().add(asgInst);+ LOG.infov("ASG {0}: launched instance {1} (Pending)", asg.getAutoScalingGroupName(), ec2Inst.getInstanceId());+ }+ return instances;+ }+private List<LaunchPlacement> launchPlacements(AutoScalingGroup asg) {if (!asg.getSubnetIds().isEmpty()) {Map<String, Subnet> subnets = ec2Service.describeSubnets(asg.getRegion(), asg.getSubnetIds(), Map.of())diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.javaindex f69faa7a20..0192826786 100644--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingService.java@@ -468,11 +468,12 @@ public class AutoScalingService {});}groups.remove(key);+ instanceRefreshes.entrySet().removeIf(e -> region.equals(e.getValue().getRegion())+ && name.equals(e.getValue().getAutoScalingGroupName()));}// clean up associated hooks and policieshooks.entrySet().removeIf(e -> e.getValue().getAutoScalingGroupName().equals(name));policies.entrySet().removeIf(e -> e.getValue().getAutoScalingGroupName().equals(name));- instanceRefreshes.entrySet().removeIf(e -> e.getValue().getAutoScalingGroupName().equals(name));scheduledActions.entrySet().removeIf(e -> region.equals(e.getValue().getRegion())&& name.equals(e.getValue().getAutoScalingGroupName()));warmPools.remove(warmPoolKey(region, name));@@ -502,11 +503,11 @@ public class AutoScalingService {}}- void adjustCapacityIfPresent(AutoScalingGroup asg, Runnable adjustment) {+ void reconcileIfPresent(AutoScalingGroup asg, Runnable reconciliation) {String key = asgKey(asg.getRegion(), asg.getAutoScalingGroupName());synchronized (lockFor(key)) {if (groups.get(key) == asg) {- adjustment.run();+ reconciliation.run();}}}@@ -563,8 +564,8 @@ public class AutoScalingService {}// An empty list suspends all processes. The reconciler honors Launch and Terminate- // for desired-capacity adjustments; other names remain stored state. Refresh selection- // and cleanup are separate paths and do not yet honor suspended processes.+ // for desired-capacity adjustments and owned refresh replacements. Other names remain+ // stored state; InstanceRefresh also pauses its owned replacement work.private static final List<String> ALL_SCALING_PROCESSES = List.of("Launch", "Terminate", "AddToLoadBalancer", "AlarmNotification", "AZRebalance","HealthCheck", "InstanceRefresh", "ReplaceUnhealthy", "ScheduledActions");@@ -683,13 +684,19 @@ public class AutoScalingService {public void detachInstances(String region, String name, List<String> instanceIds,boolean decrementDesiredCapacity) {- AutoScalingGroup asg = requireGroup(region, name);- asg.getInstances().removeIf(i -> instanceIds.contains(i.getInstanceId()));- if (decrementDesiredCapacity) {- int newDesired = Math.max(asg.getMinSize(), asg.getDesiredCapacity() - instanceIds.size());- asg.setDesiredCapacity(newDesired);+ String key = asgKey(region, name);+ synchronized (lockFor(key)) {+ AutoScalingGroup asg = requireGroup(region, name);+ List<String> detached = asg.getInstances().stream().map(AsgInstance::getInstanceId)+ .filter(instanceIds::contains).toList();+ releaseRefreshOwnership(asg, detached);+ asg.getInstances().removeIf(i -> instanceIds.contains(i.getInstanceId()));+ if (decrementDesiredCapacity) {+ int newDesired = Math.max(asg.getMinSize(), asg.getDesiredCapacity() - instanceIds.size());+ asg.setDesiredCapacity(newDesired);+ }+ groups.put(key, asg);}- groups.put(asgKey(region, name), asg);}public void terminateInstanceInAutoScalingGroup(String region, String instanceId,@@ -700,67 +707,109 @@ public class AutoScalingService {.findFirst().orElseThrow(() -> new AwsException("ValidationError","Instance '" + instanceId + "' not found in any Auto Scaling group.", 400));- asg.getInstances().stream()- .filter(i -> instanceId.equals(i.getInstanceId()))- .findFirst()- .ifPresent(i -> i.setLifecycleState("Terminating"));- if (decrementDesiredCapacity) {- int newDesired = Math.max(asg.getMinSize(), asg.getDesiredCapacity() - 1);- asg.setDesiredCapacity(newDesired);+ String key = asgKey(region, asg.getAutoScalingGroupName());+ synchronized (lockFor(key)) {+ if (groups.get(key) != asg) {+ throw new AwsException("ValidationError",+ "Instance '" + instanceId + "' not found in any Auto Scaling group.", 400);+ }+ AsgInstance instance = asg.getInstances().stream()+ .filter(i -> instanceId.equals(i.getInstanceId())).findFirst()+ .orElseThrow(() -> new AwsException("ValidationError",+ "Instance '" + instanceId + "' not found in any Auto Scaling group.", 400));+ releaseRefreshOwnership(asg, List.of(instanceId));+ instance.setLifecycleState("Terminating");+ if (decrementDesiredCapacity) {+ int newDesired = Math.max(asg.getMinSize(), asg.getDesiredCapacity() - 1);+ asg.setDesiredCapacity(newDesired);+ }+ groups.put(key, asg);+ }+ }++ private void releaseRefreshOwnership(AutoScalingGroup asg, List<String> instanceIds) {+ InstanceRefresh refresh = activeInstanceRefresh(asg);+ if (refresh != null && refresh.getReplacements() != null) {+ refresh.getReplacements().removeIf(replacement -> instanceIds.stream().anyMatch(instanceId ->+ instanceId.equals(replacement.getOriginalInstanceId())+ || instanceId.equals(replacement.getReplacementInstanceId())));+ saveInstanceRefresh(refresh);}- groups.put(asgKey(region, asg.getAutoScalingGroupName()), asg);}// ── Instance refreshes ────────────────────────────────────────────────────public InstanceRefresh startInstanceRefresh(String region, String asgName, InstanceRefresh requestedRefresh) {- AutoScalingGroup asg = requireGroup(region, asgName);- boolean activeRefresh = instanceRefreshes.values().stream()- .filter(r -> region.equals(r.getRegion()))- .filter(r -> asgName.equals(r.getAutoScalingGroupName()))- .anyMatch(r -> isActiveRefreshStatus(r.getStatus()));- if (activeRefresh) {- throw new AwsException("InstanceRefreshInProgress",- "An active instance refresh already exists for Auto Scaling group '" + asgName + "'.", 400);- }-- Instant now = Instant.now();- InstanceRefresh refresh = new InstanceRefresh();- refresh.setInstanceRefreshId(UUID.randomUUID().toString());- refresh.setAutoScalingGroupName(asgName);- refresh.setStrategy(normalizeRefreshStrategy(requestedRefresh.getStrategy()));- refresh.setStartTime(now);- refresh.setRegion(region);- copyDesiredConfiguration(requestedRefresh, refresh);- copyPreferences(requestedRefresh, refresh);-- applyDesiredConfiguration(asg, refresh);- List<String> instanceIds = markInstancesForRefresh(asg, refresh);- if (instanceIds.isEmpty()) {- refresh.setStatus("Successful");- refresh.setStatusReason("Instance refresh completed.");- refresh.setPercentageComplete(100);- refresh.setInstancesToUpdate(0);- refresh.setEndTime(now);- } else {- refresh.setStatus("InProgress");- refresh.setStatusReason("Instance refresh in progress.");- refresh.setPercentageComplete(0);- refresh.setInstancesToUpdate(instanceIds.size());- }- instanceRefreshes.put(instanceRefreshKey(region, asgName, refresh.getInstanceRefreshId()), refresh);- groups.put(asgKey(region, asgName), asg);- if (!instanceIds.isEmpty()) {+ synchronized (lockFor(asgKey(region, asgName))) {+ AutoScalingGroup asg = requireGroup(region, asgName);+ boolean activeRefresh = instanceRefreshes.values().stream()+ .filter(r -> region.equals(r.getRegion()))+ .filter(r -> asgName.equals(r.getAutoScalingGroupName()))+ .anyMatch(r -> isActiveRefreshStatus(r.getStatus()));+ if (activeRefresh) {+ throw new AwsException("InstanceRefreshInProgress",+ "An active instance refresh already exists for Auto Scaling group '" + asgName + "'.", 400);+ }++ Instant now = Instant.now();+ InstanceRefresh refresh = new InstanceRefresh();+ refresh.setInstanceRefreshId(UUID.randomUUID().toString());+ refresh.setAutoScalingGroupName(asgName);+ refresh.setStrategy(normalizeRefreshStrategy(requestedRefresh.getStrategy()));+ refresh.setStartTime(now);+ refresh.setRegion(region);+ copyDesiredConfiguration(requestedRefresh, refresh);+ copyPreferences(requestedRefresh, refresh);++ applyDesiredConfiguration(asg, refresh);+ List<String> instanceIds = markInstancesForRefresh(asg, refresh);+ List<InstanceRefreshReplacement> replacements = new ArrayList<>();+ for (String instanceId : instanceIds) {+ InstanceRefreshReplacement replacement = new InstanceRefreshReplacement();+ replacement.setOriginalInstanceId(instanceId);+ replacement.setLaunchClientToken(UUID.randomUUID().toString());+ replacements.add(replacement);+ }+ refresh.setReplacements(replacements);+ if (instanceIds.isEmpty()) {+ refresh.setStatus("Successful");+ refresh.setStatusReason("Instance refresh completed.");+ refresh.setPercentageComplete(100);+ refresh.setInstancesToUpdate(0);+ refresh.setEndTime(now);+ } else {+ refresh.setStatus("InProgress");+ refresh.setStatusReason("Instance refresh in progress.");+ refresh.setPercentageComplete(0);+ refresh.setInstancesToUpdate(instanceIds.size());+ }+ instanceRefreshes.put(instanceRefreshKey(region, asgName, refresh.getInstanceRefreshId()), refresh);+ groups.put(asgKey(region, asgName), asg);+ if (!instanceIds.isEmpty()) {+ recordActivity(region, asgName,+ "Marked EC2 instance(s) for refresh: " + instanceIds,+ "At " + now + " an instance refresh selected active instances for replacement.",+ "Successful");+ }recordActivity(region, asgName,- "Marked EC2 instance(s) for refresh: " + instanceIds,- "At " + now + " an instance refresh selected active instances for replacement.",+ "Started instance refresh " + refresh.getInstanceRefreshId() + ".",+ "At " + now + " an instance refresh was started.","Successful");+ return refresh;}- recordActivity(region, asgName,- "Started instance refresh " + refresh.getInstanceRefreshId() + ".",- "At " + now + " an instance refresh was started.",- "Successful");- return refresh;+ }++ InstanceRefresh activeInstanceRefresh(AutoScalingGroup asg) {+ return instanceRefreshes.values().stream()+ .filter(refresh -> asg.getRegion().equals(refresh.getRegion()))+ .filter(refresh -> asg.getAutoScalingGroupName().equals(refresh.getAutoScalingGroupName()))+ .filter(refresh -> isActiveRefreshStatus(refresh.getStatus()))+ .findFirst().orElse(null);+ }++ void saveInstanceRefresh(InstanceRefresh refresh) {+ instanceRefreshes.put(instanceRefreshKey(refresh.getRegion(), refresh.getAutoScalingGroupName(),+ refresh.getInstanceRefreshId()), refresh);}public void completeInstanceRefreshIfSettled(String region, String asgName) {@@ -771,7 +820,19 @@ public class AutoScalingService {.filter(r -> isActiveRefreshStatus(r.getStatus())).collect(Collectors.toList());for (InstanceRefresh refresh : activeRefreshes) {- int remaining = remainingInstancesToUpdate(asg);+ if (refresh.getReplacements() != null) {+ for (InstanceRefreshReplacement replacement : refresh.getReplacements()) {+ if (asg.getInstances().stream().anyMatch(instance ->+ instance.getInstanceId().equals(replacement.getReplacementInstanceId())+ && "InService".equals(instance.getLifecycleState()))) {+ replacement.setPhase("Complete");+ }+ }+ }+ int remaining = refresh.getReplacements() == null ? remainingInstancesToUpdate(asg)+ : (int) refresh.getReplacements().stream()+ .filter(replacement -> !replacement.isComplete())+ .count();refresh.setInstancesToUpdate(remaining);refresh.setPercentageComplete(remaining == 0 ? 100 : 0);if (remaining == 0) {diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefresh.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefresh.javaindex 0a95d164e1..6ef01f3cd4 100644--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefresh.java+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefresh.java@@ -21,6 +21,11 @@ public class InstanceRefresh {private Instant startTime;private Instant endTime;private String region;+ private List<InstanceRefreshReplacement> replacements;++ public List<InstanceRefreshReplacement> getReplacements() { return replacements; }+ public void setReplacements(List<InstanceRefreshReplacement> value) { replacements = value; }+private String desiredLaunchTemplateId;private String desiredLaunchTemplateName;diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshLaunch.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshLaunch.javanew file mode 100644index 0000000000..86e140e8a0--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshLaunch.java@@ -0,0 +1,12 @@+package io.github.hectorvent.floci.services.autoscaling.model;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import io.quarkus.runtime.annotations.RegisterForReflection;++/** Source identity recorded before a refresh launch can be accepted by EC2. */+@RegisterForReflection+@JsonIgnoreProperties(ignoreUnknown = true)+public record InstanceRefreshLaunch(String launchConfigurationName, String launchTemplateId,+ String launchTemplateName, String launchTemplateVersion,+ String instanceType, String availabilityZone) {+}diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshReplacement.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshReplacement.javanew file mode 100644index 0000000000..2bb9ca0fe6--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/model/InstanceRefreshReplacement.java@@ -0,0 +1,32 @@+package io.github.hectorvent.floci.services.autoscaling.model;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+@JsonIgnoreProperties(ignoreUnknown = true)+public class InstanceRefreshReplacement {+ private String originalInstanceId;+ private String replacementInstanceId;+ private String launchClientToken;+ private String phase = "Selected";+ private InstanceRefreshLaunch launch;++ public InstanceRefreshLaunch getLaunch() { return launch; }+ public void setLaunch(InstanceRefreshLaunch value) { launch = value; }++ public String getPhase() { return phase; }+ public void setPhase(String value) { phase = value; }++ public boolean isComplete() { return "Complete".equals(phase) || "NoLongerRequired".equals(phase); }+++ public String getOriginalInstanceId() { return originalInstanceId; }+ public void setOriginalInstanceId(String value) { originalInstanceId = value; }++ public String getReplacementInstanceId() { return replacementInstanceId; }+ public void setReplacementInstanceId(String value) { replacementInstanceId = value; }++ public String getLaunchClientToken() { return launchClientToken; }+ public void setLaunchClientToken(String value) { launchClientToken = value; }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.javaindex 48faa51c1a..777f820a80 100644--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingCapacityAccountLockIntegrationTest.java@@ -39,7 +39,7 @@ class AutoScalingCapacityAccountLockIntegrationTest {AutoScalingGroup group = RequestScopes.callAs(ACCOUNT, REGION, () -> create(REGION, name));RequestScopes.runAs(otherAccount, otherRegion, () -> create(otherRegion, name));Future<?> adjustment = workers.submit(() -> RequestScopes.runAs(ACCOUNT, REGION,- () -> service.adjustCapacityIfPresent(group, () -> {+ () -> service.reconcileIfPresent(group, () -> {entered.countDown();try {assertTrue(release.await(10, TimeUnit.SECONDS));diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingPlacementTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingPlacementTest.javaindex 388704d819..765cc7f61f 100644--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingPlacementTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingPlacementTest.java@@ -307,7 +307,7 @@ class AutoScalingPlacementTest {doAnswer(call -> {call.<Runnable>getArgument(1).run();return null;- }).when(groups).adjustCapacityIfPresent(any(), any());+ }).when(groups).reconcileIfPresent(any(), any());when(groups.describeScalingActivities(anyString(), anyString())).thenAnswer(call ->activityStore.describeScalingActivities(call.getArgument(0), call.getArgument(1)));when(groups.recordActivity(anyString(), anyString(), anyString(), anyString(), anyString()))diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.javaindex 1fbf91ecd8..ba8bfcdf64 100644--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java@@ -55,7 +55,7 @@ class AutoScalingReconcilerTest {Runnable adjustment = invocation.getArgument(1);adjustment.run();return null;- }).when(service).adjustCapacityIfPresent(any(), any());+ }).when(service).reconcileIfPresent(any(), any());return service;}diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshStorageTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshStorageTest.javanew file mode 100644index 0000000000..d51875bc45--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshStorageTest.java@@ -0,0 +1,1011 @@+package io.github.hectorvent.floci.services.autoscaling;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.ServiceConfigAccess;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.autoscaling.model.AsgInstance;+import io.github.hectorvent.floci.services.autoscaling.model.AsgOptionalFields;+import io.github.hectorvent.floci.services.autoscaling.model.AutoScalingGroup;+import io.github.hectorvent.floci.services.autoscaling.model.InstanceRefresh;+import io.github.hectorvent.floci.services.autoscaling.model.ScalingActivity;+import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.services.ec2.model.Instance;+import io.github.hectorvent.floci.services.ec2.model.InstanceState;+import io.github.hectorvent.floci.services.ec2.model.LaunchTemplate;+import io.github.hectorvent.floci.services.ec2.model.Placement;+import io.github.hectorvent.floci.services.ec2.model.Reservation;+import io.github.hectorvent.floci.services.elbv2.ElbV2Service;+import io.github.hectorvent.floci.services.elbv2.model.TargetDescription;+import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;+import org.junit.jupiter.params.provider.ValueSource;++import java.io.IOException;+import java.lang.reflect.Field;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.stream.Stream;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.*;++class AutoScalingRefreshStorageTest {+ @TempDir Path storagePath;+ private final List<StorageFactory> factories = new ArrayList<>();+ private static final String REGION = "us-east-1";+ private AutoScalingService service;+ private Ec2Service ec2;+ private AutoScalingReconciler reconciler;+ private AutoScalingGroup group;++ private void setUp(String mode) {+ service = storedService(mode);+ service.createAutoScalingGroup(REGION, "suspended-group", null, "lt-original", null, "1", null,+ 0, 3, 1, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());+ group = service.describeAutoScalingGroups(REGION, List.of("suspended-group")).getFirst();+ AsgInstance existing = new AsgInstance();+ existing.setInstanceId("i-original");+ existing.setLifecycleState("InService");+ existing.setHealthStatus("Healthy");+ existing.setLaunchTemplateId("lt-original");+ existing.setLaunchTemplateVersion("1");+ group.getInstances().add(existing);+ service.saveAutoScalingGroupIfPresent(group);+ ec2 = mock(Ec2Service.class);+ service.ec2Service = ec2;+ reconciler = new AutoScalingReconciler(service, ec2, mock(ElbV2Service.class));+ when(ec2.isInstanceContainerRunning(anyString())).thenReturn(true);+ when(ec2.describeInstances(eq(REGION), anyList(), isNull()))+ .thenReturn(List.of(reservation("i-original")));+ LaunchTemplate template = new LaunchTemplate();+ template.setLaunchTemplateId("lt-original");+ template.setLatestVersionNumber("1");+ template.getData().setImageId("ami-original");+ template.getData().setInstanceType("t3.micro");+ when(ec2.describeLaunchTemplates(eq(REGION), anyList(), anyList(), anyMap())).thenReturn(List.of(template));+ when(ec2.describeLaunchTemplateVersions(eq(REGION), eq("lt-original"), isNull(), anyList()))+ .thenReturn(List.of(template));+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any())).thenReturn(reservation("i-replacement"));+ }++ private AutoScalingService storedService(String mode) {+ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.StorageConfig storage = mock(EmulatorConfig.StorageConfig.class);+ EmulatorConfig.WalConfig wal = mock(EmulatorConfig.WalConfig.class);+ when(config.storage()).thenReturn(storage);+ when(config.defaultAccountId()).thenReturn("000000000000");+ when(storage.persistentPath()).thenReturn(storagePath.toString());+ when(storage.wal()).thenReturn(wal);+ when(wal.compactionIntervalMs()).thenReturn(60000L);+ ServiceConfigAccess access = mock(ServiceConfigAccess.class);+ when(access.storageMode("autoscaling")).thenReturn(mode);+ when(access.storageFlushInterval("autoscaling")).thenReturn(60000L);+ StorageFactory factory = new StorageFactory(config, access);+ factories.add(factory);+ AutoScalingService created = new AutoScalingService();+ created.regionResolver = new RegionResolver(REGION, "000000000000");+ created.storageFactory = factory;+ created.initializeStorage();+ return created;+ }++ private void restart(String mode) throws IOException {+ AutoScalingGroup previous = group;+ factories.removeFirst().shutdownAll();+ try (Stream<Path> files = Files.list(storagePath)) {+ assertTrue(files.anyMatch(p -> p.getFileName().toString().contains("autoscaling-groups")),+ "No real group snapshot was written");+ }+ service = storedService(mode);+ service.ec2Service = ec2;+ group = service.describeAutoScalingGroups(REGION, List.of("suspended-group")).getFirst();+ assertNotSame(previous, group, "Restart reused the in-memory group");+ reconciler = new AutoScalingReconciler(service, ec2, mock(ElbV2Service.class));+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void replacementWithoutDeclaredZoneUsesTheAcceptedEc2Placement(String mode) throws IOException {+ setUp(mode);+ group.setAvailabilityZones(List.of());+ service.saveAutoScalingGroupIfPresent(group);+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of());+ Reservation accepted = reservation("i-replacement");+ accepted.getInstances().getFirst().setPlacement(new Placement("us-east-1b"));+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any())).thenReturn(accepted);+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ reconciler.reconcile(group);+ assertEquals("i-replacement", group.getInstances().getFirst().getInstanceId());+ assertEquals("us-east-1b", group.getInstances().getFirst().getAvailabilityZone());+ assertEquals("i-replacement", refresh.getReplacements().getFirst().getReplacementInstanceId());+ restart(mode);+ assertEquals("us-east-1b", group.getInstances().getFirst().getAvailabilityZone());+ }++ @AfterEach+ void closeStorage() {+ factories.forEach(StorageFactory::shutdownAll);+ }++ @ParameterizedTest+ @CsvSource({+ "persistent,Launch,true", "persistent,Terminate,true", "persistent,InstanceRefresh,true",+ "persistent,Launch,false", "persistent,Terminate,false", "persistent,InstanceRefresh,false",+ "hybrid,Launch,true", "hybrid,Terminate,true", "hybrid,InstanceRefresh,true",+ "hybrid,Launch,false", "hybrid,Terminate,false", "hybrid,InstanceRefresh,false",+ "wal,Launch,true", "wal,Terminate,true", "wal,InstanceRefresh,true",+ "wal,Launch,false", "wal,Terminate,false", "wal,InstanceRefresh,false"+ })+ void refreshPauseSurvivesRealStorageRestart(String mode, String process, boolean suspendBefore) throws IOException {+ setUp(mode);+ if (suspendBefore) {+ service.suspendProcesses(REGION, "suspended-group", List.of(process));+ }+ InstanceRefresh started = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ if (!suspendBefore) {+ service.suspendProcesses(REGION, "suspended-group", List.of(process));+ }+ restart(mode);+ InstanceRefresh restored = service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)+ .instanceRefreshes().getFirst();+ assertNotSame(started, restored);+ assertEquals(started.getInstanceRefreshId(), restored.getInstanceRefreshId());+ assertTrue(group.getSuspendedProcesses().contains(process));+ reconciler.reconcile(group);+ assertAll(+ () -> verify(ec2, never()).terminateInstances(anyString(), anyList()),+ () -> verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(),+ any(), any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()),+ () -> assertEquals(List.of("i-original"), group.getInstances().stream()+ .map(AsgInstance::getInstanceId).toList()),+ () -> assertNotEquals("Successful", service.describeInstanceRefreshes(+ REGION, "suspended-group", null, null, null).instanceRefreshes().getFirst().getStatus()));+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void failedReplacementLaunchRemainsPausedAfterRestart(String mode) throws IOException {+ setUp(mode);+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()))+ .thenThrow(new IllegalStateException("launch unavailable"))+ .thenReturn(reservation("i-replacement"));+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ reconciler.reconcile(group);+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ assertTrue(group.getInstances().isEmpty());+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ restart(mode);+ InstanceRefresh restored = service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)+ .instanceRefreshes().getFirst();+ assertEquals("InProgress", restored.getStatus());+ assertEquals(1, restored.getInstancesToUpdate());+ reconciler.reconcile(group);+ assertAll(+ () -> verify(ec2, times(1)).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(),+ anyList(), any(), any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()),+ () -> assertTrue(group.getInstances().isEmpty()));+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void resumedRefreshReplacesOriginalAfterRestart(String mode) throws IOException {+ setUp(mode);+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ restart(mode);+ service.resumeProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ reconciler.reconcile(group);+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ assertEquals(List.of("i-replacement"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void manualTerminationDuringPausedRefreshStillRunsAfterRestart(String mode) throws IOException {+ setUp(mode);+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ AsgInstance manual = new AsgInstance();+ manual.setInstanceId("i-manual");+ manual.setLifecycleState("InService");+ manual.setHealthStatus("Healthy");+ group.getInstances().add(manual);+ group.setDesiredCapacity(2);+ service.saveAutoScalingGroupIfPresent(group);+ service.terminateInstanceInAutoScalingGroup(REGION, "i-manual", true);+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ restart(mode);+ reconciler.reconcile(group);+ assertAll(+ () -> verify(ec2).terminateInstances(REGION, List.of("i-manual")),+ () -> assertEquals(List.of("i-original"), group.getInstances().stream()+ .map(AsgInstance::getInstanceId).toList()));+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void manualTerminationStillRunsAfterRestart(String mode) throws IOException {+ setUp(mode);+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ service.terminateInstanceInAutoScalingGroup(REGION, "i-original", true);+ restart(mode);+ reconciler.reconcile(group);+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ assertTrue(group.getInstances().isEmpty());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void failedReplacementRecordsActivityAndRetainsRetryOwnership(String mode) throws IOException {+ setUp(mode);+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()))+ .thenThrow(new AwsException(+ "InsufficientInstanceCapacity", "replacement capacity unavailable", 500))+ .thenThrow(new AwsException("InsufficientInstanceCapacity", "replacement capacity unavailable", 500))+ .thenThrow(new AwsException("RequestLimitExceeded", "replacement throttled", 503))+ .thenReturn(reservation("i-replacement"));+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ String token = refresh.getReplacements().getFirst().getLaunchClientToken();+ reconciler.reconcile(group);+ assertTrue(group.getInstances().isEmpty());+ assertEquals("InProgress", refresh.getStatus());+ assertEquals(token, refresh.getReplacements().getFirst().getLaunchClientToken());+ List<ScalingActivity> failed = service.describeScalingActivities(REGION, "suspended-group").stream()+ .filter(activity -> "Failed".equals(activity.getStatusCode())).toList();+ assertEquals(1, failed.size());+ assertEquals("replacement capacity unavailable", failed.getFirst().getStatusMessage());+ assertEquals(100, failed.getFirst().getProgress());+ assertNotNull(failed.getFirst().getEndTime());+ reconciler.reconcile(group);+ assertEquals(1, service.describeScalingActivities(REGION, "suspended-group").stream()+ .filter(activity -> "Failed".equals(activity.getStatusCode())).count());+ reconciler.reconcile(group);+ assertEquals(List.of("replacement throttled", "replacement capacity unavailable"),+ service.describeScalingActivities(REGION, "suspended-group").stream()+ .filter(activity -> "Failed".equals(activity.getStatusCode()))+ .map(ScalingActivity::getStatusMessage).toList());+ restart(mode);+ assertEquals(2, service.describeScalingActivities(REGION, "suspended-group").stream()+ .filter(activity -> "Failed".equals(activity.getStatusCode())).count());+ reconciler.reconcile(group);+ reconciler.reconcile(group);+ verify(ec2, times(4)).runInstances(eq(REGION), any(), any(), eq(1), eq(1), any(), anyList(), any(),+ eq(token), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ assertEquals("Successful", service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)+ .instanceRefreshes().getFirst().getStatus());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void failedReplacementAllowsSiblingProgressWithOneInventoryScan(String mode) {+ setUp(mode);+ group.setDesiredCapacity(2);+ AsgInstance second = new AsgInstance();+ second.setInstanceId("i-second-original");+ second.setLifecycleState("InService");+ second.setHealthStatus("Healthy");+ group.getInstances().add(second);+ service.saveAutoScalingGroupIfPresent(group);+ Reservation unrelated = reservation("i-unrelated");+ unrelated.getInstances().getFirst().setClientToken("unrelated-attempt");+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of(unrelated));+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(),+ anyBoolean(), any(), any(), any()))+ .thenThrow(new IllegalStateException("first replacement launch unavailable"))+ .thenReturn(reservation("i-sibling"))+ .thenReturn(reservation("i-retry"));+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ String token = refresh.getReplacements().getFirst().getLaunchClientToken();+ reconciler.reconcile(group);+ assertEquals(List.of("i-sibling"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());+ assertEquals("InProgress", refresh.getStatus());+ assertEquals(token, refresh.getReplacements().getFirst().getLaunchClientToken());+ assertEquals("first replacement launch unavailable", service.describeScalingActivities(REGION, "suspended-group")+ .stream().filter(activity -> "Failed".equals(activity.getStatusCode())).findFirst().orElseThrow()+ .getStatusMessage());+ verify(ec2, times(1)).describeInstances(REGION, List.of(), null);+ reconciler.reconcile(group);+ verify(ec2, times(2)).describeInstances(REGION, List.of(), null);+ assertEquals(List.of("i-sibling", "i-retry"),+ group.getInstances().stream().map(AsgInstance::getInstanceId).toList());+ reconciler.reconcile(group);+ assertEquals("Successful", refresh.getStatus());+ verify(ec2, times(2)).describeInstances(REGION, List.of(), null);+ }++ @Test+ void refreshCleanupFailureRetainsOwnershipAndPropagates() {+ setUp("persistent");+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ String token = refresh.getReplacements().getFirst().getLaunchClientToken();+ AutoScalingService intercepted = spy(service);+ doReturn(false).when(intercepted).saveAutoScalingGroupIfPresent(group);+ IllegalStateException cleanupFailure = new IllegalStateException("cleanup refused");+ when(ec2.terminateInstances(REGION, List.of("i-replacement"))).thenThrow(cleanupFailure);+ AutoScalingReconciler worker = new AutoScalingReconciler(intercepted, ec2, mock(ElbV2Service.class));+ assertSame(cleanupFailure, assertThrows(IllegalStateException.class, () -> worker.reconcile(group)));+ assertEquals(token, refresh.getReplacements().getFirst().getLaunchClientToken());+ assertEquals("InProgress", refresh.getStatus());+ assertNull(refresh.getReplacements().getFirst().getReplacementInstanceId());+ assertEquals(List.of("i-replacement"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void launchRetryResumesWithSavedToken(String mode) throws IOException {+ setUp(mode);+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()))+ .thenThrow(new IllegalStateException("launch unavailable"))+ .thenReturn(reservation("i-replacement"));+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ String token = refresh.getReplacements().getFirst().getLaunchClientToken();+ reconciler.reconcile(group);+ restart(mode);+ reconciler.reconcile(group);+ reconciler.reconcile(group);+ verify(ec2, times(2)).runInstances(eq(REGION), any(), any(), eq(1), eq(1), any(), anyList(), any(),+ eq(token), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ verify(ec2, times(1)).terminateInstances(REGION, List.of("i-original"));+ assertEquals("Successful", service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)+ .instanceRefreshes().getFirst().getStatus());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void recoversEc2LaunchWhoseMembershipWasNotSaved(String mode) throws IOException {+ setUp(mode);+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ Reservation launched = reservation("i-recovered");+ launched.getInstances().getFirst().setClientToken(refresh.getReplacements().getFirst().getLaunchClientToken());+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of(launched));+ group.getInstances().clear();+ service.saveAutoScalingGroupIfPresent(group);+ restart(mode);+ reconciler.reconcile(group);+ assertEquals(List.of("i-recovered"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ reconciler.reconcile(group);+ assertEquals(1, group.getInstances().size());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void explicitTerminationOfSelectedOriginalIsNotPaused(String mode) throws IOException {+ setUp(mode);+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ service.terminateInstanceInAutoScalingGroup(REGION, "i-original", true);+ restart(mode);+ reconciler.reconcile(group);+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ assertTrue(group.getInstances().isEmpty());+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void pausedRefreshDoesNotBlockIndependentCapacityIncrease(String mode) throws IOException {+ setUp(mode);+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ group.setDesiredCapacity(2);+ service.saveAutoScalingGroupIfPresent(group);+ restart(mode);+ reconciler.reconcile(group);+ verify(ec2, never()).terminateInstances(anyString(), anyList());+ verify(ec2).runInstances(eq(REGION), any(), any(), eq(1), eq(1), any(), anyList(), any(), isNull(),+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ assertEquals(2, group.getInstances().size());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void capacityReductionDoesNotLaunchUnneededReplacement(String mode) throws IOException {+ setUp(mode);+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ group.setDesiredCapacity(0);+ service.saveAutoScalingGroupIfPresent(group);+ restart(mode);+ reconciler.reconcile(group);+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ assertTrue(group.getInstances().isEmpty());+ assertEquals("Successful", service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)+ .instanceRefreshes().getFirst().getStatus());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void missingOriginalRetainsPausedReplacementReservation(String mode) throws IOException {+ setUp(mode);+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ when(ec2.describeInstances(REGION, List.of("i-original"), null)).thenReturn(List.of());+ restart(mode);+ reconciler.reconcile(group);+ assertTrue(group.getInstances().isEmpty());+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ service.resumeProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ restart(mode);+ reconciler.reconcile(group);+ assertEquals(List.of("i-replacement"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void recoveredLaunchAlreadyInGroupDoesNotDuplicateMembership(String mode) throws IOException {+ setUp(mode);+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ Reservation launched = reservation("i-recovered");+ launched.getInstances().getFirst().setClientToken(refresh.getReplacements().getFirst().getLaunchClientToken());+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of(launched));+ group.getInstances().clear();+ AsgInstance member = new AsgInstance();+ member.setInstanceId("i-recovered");+ member.setLifecycleState("Pending");+ member.setHealthStatus("Healthy");+ group.getInstances().add(member);+ service.saveAutoScalingGroupIfPresent(group);+ restart(mode);+ reconciler.reconcile(group);+ assertEquals(1, group.getInstances().size());+ InstanceRefresh restored = service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)+ .instanceRefreshes().getFirst();+ assertEquals("i-recovered", restored.getReplacements().getFirst().getReplacementInstanceId());+ assertEquals("Successful", restored.getStatus());+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ }++ @ParameterizedTest+ @CsvSource({"persistent,Launch", "persistent,Terminate", "persistent,InstanceRefresh",+ "hybrid,Launch", "hybrid,Terminate", "hybrid,InstanceRefresh",+ "wal,Launch", "wal,Terminate", "wal,InstanceRefresh"})+ void pausedOriginalKeepsTargetRegistrationUntilResume(String mode, String process) throws IOException {+ setUp(mode);+ group.setTargetGroupARNs(List.of("test-target-group"));+ service.saveAutoScalingGroupIfPresent(group);+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ service.suspendProcesses(REGION, "suspended-group", List.of(process));+ restart(mode);+ ElbV2Service elb = mock(ElbV2Service.class);+ TargetDescription target = new TargetDescription();+ target.setId("i-original");+ TargetHealth health = new TargetHealth();+ health.setTarget(target);+ health.setState("healthy");+ when(elb.describeTargetHealth(REGION, "test-target-group", List.of())).thenReturn(List.of(health));+ reconciler = new AutoScalingReconciler(service, ec2, elb);+ reconciler.reconcile(group);+ verify(elb, never()).deregisterTargets(anyString(), anyString(), anyList());+ verify(ec2, never()).terminateInstances(anyString(), anyList());+ service.resumeProcesses(REGION, "suspended-group", List.of(process));+ reconciler.reconcile(group);+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ verify(elb, atLeastOnce()).deregisterTargets(eq(REGION), eq("test-target-group"),+ argThat(targets -> targets.stream().anyMatch(value -> "i-original".equals(value.getId()))));+ assertEquals(List.of("i-replacement"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());+ }++ @ParameterizedTest+ @CsvSource({"persistent,false,false", "persistent,false,true", "persistent,true,false", "persistent,true,true",+ "hybrid,false,false", "hybrid,false,true", "hybrid,true,false", "hybrid,true,true",+ "wal,false,false", "wal,false,true", "wal,true,false", "wal,true,true"})+ void detachedInstancesAreNotReadoptedAfterRestart(String mode, boolean replacement, boolean decrement)+ throws IOException {+ setUp(mode);+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ String detachedId = replacement ? "i-detached" : "i-original";+ if (replacement) {+ group.getInstances().clear();+ AsgInstance member = new AsgInstance();+ member.setInstanceId(detachedId);+ member.setLifecycleState("InService");+ member.setHealthStatus("Healthy");+ group.getInstances().add(member);+ refresh.getReplacements().getFirst().setReplacementInstanceId(detachedId);+ refresh.getReplacements().getFirst().setPhase("Warming");+ service.saveInstanceRefresh(refresh);+ service.saveAutoScalingGroupIfPresent(group);+ Reservation launched = reservation(detachedId);+ launched.getInstances().getFirst().setClientToken(refresh.getReplacements().getFirst().getLaunchClientToken());+ when(ec2.describeInstances(eq(REGION), any(), isNull())).thenReturn(List.of(launched));+ }+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ service.detachInstances(REGION, "suspended-group", List.of(detachedId), decrement);+ restart(mode);+ InstanceRefresh restored = service.describeInstanceRefreshes(REGION, "suspended-group", null, null, null)+ .instanceRefreshes().getFirst();+ assertTrue(restored.getReplacements().isEmpty(), "Explicit detach retained refresh ownership");+ service.resumeProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ reconciler.reconcile(group);+ reconciler.reconcile(group);+ assertEquals(decrement ? 0 : 1, group.getDesiredCapacity());+ assertEquals(decrement ? 0 : 1, group.getInstances().size());+ assertTrue(group.getInstances().stream().noneMatch(member -> detachedId.equals(member.getInstanceId())));+ verify(ec2, never()).terminateInstances(anyString(), argThat(ids -> ids.contains(detachedId)));+ if (decrement) {+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ }+ assertEquals("Successful", restored.getStatus());+ }++ @ParameterizedTest+ @CsvSource({"persistent,false,false", "persistent,true,false", "hybrid,false,false", "hybrid,true,false",+ "wal,false,false", "wal,true,false", "persistent,false,true", "persistent,true,true",+ "hybrid,false,true", "hybrid,true,true", "wal,false,true", "wal,true,true"})+ void recoveredLaunchRetainsAttemptSourceAfterGroupChanges(String mode, boolean sourceDeleted,+ boolean defaultPlacement) throws IOException {+ setUp(mode);+ if (defaultPlacement) {+ group.setAvailabilityZones(List.of());+ service.saveAutoScalingGroupIfPresent(group);+ }+ List<Reservation> launched = new ArrayList<>();+ when(ec2.describeInstances(REGION, List.of(), null)).thenAnswer(invocation -> List.copyOf(launched));+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any())).thenAnswer(invocation -> {+ Reservation accepted = reservation("i-accepted");+ accepted.getInstances().getFirst().setClientToken(invocation.getArgument(8));+ accepted.getInstances().getFirst().setInstanceType("t3.micro");+ accepted.getInstances().getFirst().setPlacement(new Placement("us-east-1a"));+ launched.add(accepted);+ throw new IllegalStateException("EC2 accepted launch but response was lost");+ });+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ reconciler.reconcile(group);+ assertTrue(group.getInstances().isEmpty());+ assertEquals(1, launched.size());+ group.setLaunchTemplateId("lt-changed");+ group.setAvailabilityZones(List.of("us-east-1b"));+ service.saveAutoScalingGroupIfPresent(group);+ LaunchTemplate changed = new LaunchTemplate();+ changed.setLaunchTemplateId("lt-changed");+ changed.setLatestVersionNumber("2");+ changed.getData().setImageId("ami-changed");+ changed.getData().setInstanceType("m5.large");+ when(ec2.describeLaunchTemplates(eq(REGION), anyList(), anyList(), anyMap()))+ .thenReturn(sourceDeleted ? List.of() : List.of(changed));+ when(ec2.describeLaunchTemplateVersions(eq(REGION), eq("lt-changed"), isNull(), anyList()))+ .thenReturn(sourceDeleted ? List.of() : List.of(changed));+ restart(mode);+ reconciler.reconcile(group);+ assertEquals(List.of("i-accepted"), group.getInstances().stream().map(AsgInstance::getInstanceId).toList());+ AsgInstance recovered = group.getInstances().getFirst();+ assertAll(() -> assertEquals("lt-original", recovered.getLaunchTemplateId()),+ () -> assertEquals("1", recovered.getLaunchTemplateVersion()),+ () -> assertEquals("t3.micro", recovered.getInstanceType()),+ () -> assertEquals("us-east-1a", recovered.getAvailabilityZone()));+ verify(ec2, times(1)).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void legacyRecoveredLaunchUsesOnlyKnownEc2Metadata(String mode) throws IOException {+ setUp(mode);+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ Reservation launched = reservation("i-legacy");+ Instance instance = launched.getInstances().getFirst();+ instance.setClientToken(refresh.getReplacements().getFirst().getLaunchClientToken());+ instance.setInstanceType("m5.large");+ instance.setPlacement(new Placement("us-east-1b"));+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of(launched));+ group.getInstances().clear();+ service.saveAutoScalingGroupIfPresent(group);+ restart(mode);+ reconciler.reconcile(group);+ AsgInstance recovered = group.getInstances().getFirst();+ assertAll(() -> assertEquals("i-legacy", recovered.getInstanceId()),+ () -> assertEquals("m5.large", recovered.getInstanceType()),+ () -> assertEquals("us-east-1b", recovered.getAvailabilityZone()),+ () -> assertNull(recovered.getLaunchConfigurationName()),+ () -> assertNull(recovered.getLaunchTemplateId()),+ () -> assertNull(recovered.getLaunchTemplateName()),+ () -> assertNull(recovered.getLaunchTemplateVersion()));+ verify(ec2, never()).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void partialMultiInstanceRefreshResumesWithoutRepeatingCompletedLaunch(String mode) throws IOException {+ setUp(mode);+ AsgInstance second = new AsgInstance();+ second.setInstanceId("i-second");+ second.setLifecycleState("InService");+ second.setHealthStatus("Healthy");+ group.getInstances().add(second);+ group.setDesiredCapacity(2);+ service.saveAutoScalingGroupIfPresent(group);+ when(ec2.describeInstances(REGION, List.of(), null)).thenReturn(List.of());+ when(ec2.runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(), any(),+ anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any()))+ .thenReturn(reservation("i-first-replacement"))+ .thenThrow(new IllegalStateException("Second launch unavailable"))+ .thenReturn(reservation("i-second-replacement"));+ InstanceRefresh started = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ assertEquals(2, started.getReplacements().size());+ assertNotEquals(started.getReplacements().getFirst().getLaunchClientToken(),+ started.getReplacements().getLast().getLaunchClientToken());+ reconciler.reconcile(group);+ assertEquals(List.of("i-first-replacement"), group.getInstances().stream()+ .map(AsgInstance::getInstanceId).toList());+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ restart(mode);+ reconciler.reconcile(group);+ InstanceRefresh restored = service.activeInstanceRefresh(group);+ assertNotNull(restored);+ assertEquals(1, restored.getInstancesToUpdate());+ assertEquals("Complete", restored.getReplacements().getFirst().getPhase());+ verify(ec2, times(2)).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ service.resumeProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ reconciler.reconcile(group);+ reconciler.reconcile(group);+ assertEquals("Successful", restored.getStatus());+ assertEquals(0, restored.getInstancesToUpdate());+ assertEquals(List.of("i-first-replacement", "i-second-replacement"), group.getInstances().stream()+ .map(AsgInstance::getInstanceId).toList());+ verify(ec2, times(3)).runInstances(eq(REGION), any(), any(), anyInt(), anyInt(), any(), anyList(), any(),+ any(), anyList(), any(), any(), any(), any(), anyInt(), any(), any(), any(), any(), anyBoolean(), any(), any(), any());+ verify(ec2).terminateInstances(REGION, List.of("i-original", "i-second"));+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void legacyNullOwnershipRetainsExistingCleanupAfterRestart(String mode) throws IOException {+ setUp(mode);+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ refresh.setReplacements(null);+ service.saveInstanceRefresh(refresh);+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ restart(mode);+ InstanceRefresh restored = service.activeInstanceRefresh(group);+ assertNotNull(restored);+ assertNull(restored.getReplacements());+ reconciler.reconcile(group);+ reconciler.reconcile(group);+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ assertEquals(List.of("i-replacement"), group.getInstances().stream()+ .map(AsgInstance::getInstanceId).toList());+ assertEquals("Successful", restored.getStatus());+ assertNull(restored.getReplacements(), "Legacy state was heuristically assigned new ownership");+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void sameNameOtherRegionRefreshDoesNotOwnLocalInstances(String mode) throws IOException {+ setUp(mode);+ String otherRegion = "us-west-2";+ LaunchTemplate otherTemplate = new LaunchTemplate();+ otherTemplate.setLaunchTemplateId("lt-other");+ otherTemplate.getData().setImageId("ami-other");+ when(ec2.describeLaunchTemplateVersions(eq(otherRegion), eq("lt-other"), isNull(), anyList()))+ .thenReturn(List.of(otherTemplate));+ service.createAutoScalingGroup(otherRegion, "suspended-group", null, "lt-other", null, "1", null,+ 0, 3, 1, 300, List.of("us-west-2a"), List.of(), List.of(), List.of(), "EC2", 0,+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());+ AutoScalingGroup other = service.describeAutoScalingGroups(otherRegion, List.of("suspended-group"))+ .getFirst();+ AsgInstance otherMember = new AsgInstance();+ otherMember.setInstanceId("i-original");+ otherMember.setLifecycleState("InService");+ otherMember.setHealthStatus("Healthy");+ other.getInstances().add(otherMember);+ service.saveAutoScalingGroupIfPresent(other);+ service.suspendProcesses(otherRegion, "suspended-group", List.of("InstanceRefresh"));+ service.startInstanceRefresh(otherRegion, "suspended-group", new InstanceRefresh());+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ service.terminateInstanceInAutoScalingGroup(REGION, "i-original", true);+ restart(mode);+ reconciler.reconcile(group);+ assertTrue(group.getInstances().isEmpty());+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ AutoScalingGroup restoredOther = service.describeAutoScalingGroups(otherRegion,+ List.of("suspended-group")).getFirst();+ InstanceRefresh otherRefresh = service.activeInstanceRefresh(restoredOther);+ assertNotNull(otherRefresh);+ assertEquals("InProgress", otherRefresh.getStatus());+ assertEquals(1, otherRefresh.getReplacements().size());+ assertEquals("i-original", otherRefresh.getReplacements().getFirst().getOriginalInstanceId());+ assertEquals(1, restoredOther.getInstances().size());+ verify(ec2, never()).terminateInstances(eq(otherRegion), anyList());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ @SuppressWarnings("unchecked")+ void pausedSelectionCannotBeTerminatedBeforeOwnershipPublication(String mode) throws Exception {+ setUp(mode);+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ service.createAutoScalingGroup(REGION, "unrelated-group", null, "lt-original", null, "1", null,+ 0, 3, 0, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());+ AutoScalingGroup unrelated = service.describeAutoScalingGroups(REGION, List.of("unrelated-group")).getFirst();+ Field field = AutoScalingService.class.getDeclaredField("instanceRefreshes");+ field.setAccessible(true);+ Map<String, InstanceRefresh> original = (Map<String, InstanceRefresh>) field.get(service);+ Map<String, InstanceRefresh> intercepted = spy(original);+ CountDownLatch selected = new CountDownLatch(1);+ CountDownLatch publish = new CountDownLatch(1);+ CountDownLatch reconciling = new CountDownLatch(1);+ doAnswer(invocation -> {+ selected.countDown();+ assertTrue(publish.await(5, TimeUnit.SECONDS));+ return original.put(invocation.getArgument(0), invocation.getArgument(1));+ }).when(intercepted).put(anyString(), any(InstanceRefresh.class));+ field.set(service, intercepted);+ ExecutorService executor = Executors.newFixedThreadPool(3);+ Future<?> starting = executor.submit(() ->+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh()));+ Future<?> reconcilingFuture = null;+ try {+ assertTrue(selected.await(5, TimeUnit.SECONDS));+ reconcilingFuture = executor.submit(() -> {+ reconciling.countDown();+ reconciler.reconcile(group);+ });+ assertTrue(reconciling.await(5, TimeUnit.SECONDS));+ try {+ reconcilingFuture.get(250, TimeUnit.MILLISECONDS);+ } catch (TimeoutException expected) {+ // Correct group coordination waits for ownership publication before reconciliation.+ }+ executor.submit(() -> reconciler.reconcile(unrelated)).get(5, TimeUnit.SECONDS);+ verify(ec2, never()).terminateInstances(anyString(), anyList());+ } finally {+ publish.countDown();+ try {+ starting.get(5, TimeUnit.SECONDS);+ if (reconcilingFuture != null) {+ reconcilingFuture.get(5, TimeUnit.SECONDS);+ }+ } finally {+ field.set(service, original);+ executor.shutdownNow();+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));+ }+ }+ }++ @ParameterizedTest+ @CsvSource({"persistent,true", "persistent,false", "hybrid,true", "hybrid,false", "wal,true", "wal,false"})+ @SuppressWarnings("unchecked")+ void explicitRemovalWaitsForRefreshOwnershipPublication(String mode, boolean detach) throws Exception {+ setUp(mode);+ service.suspendProcesses(REGION, "suspended-group", List.of("InstanceRefresh"));+ Field field = AutoScalingService.class.getDeclaredField("instanceRefreshes");+ field.setAccessible(true);+ Map<String, InstanceRefresh> original = (Map<String, InstanceRefresh>) field.get(service);+ Map<String, InstanceRefresh> intercepted = spy(original);+ CountDownLatch selected = new CountDownLatch(1);+ CountDownLatch publish = new CountDownLatch(1);+ doAnswer(invocation -> {+ selected.countDown();+ assertTrue(publish.await(5, TimeUnit.SECONDS));+ return original.put(invocation.getArgument(0), invocation.getArgument(1));+ }).when(intercepted).put(anyString(), any(InstanceRefresh.class));+ field.set(service, intercepted);+ ExecutorService executor = Executors.newFixedThreadPool(2);+ Future<?> starting = executor.submit(() ->+ service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh()));+ Future<?> removing = null;+ try {+ assertTrue(selected.await(5, TimeUnit.SECONDS));+ CountDownLatch removalStarted = new CountDownLatch(1);+ removing = executor.submit(() -> {+ removalStarted.countDown();+ if (detach) {+ service.detachInstances(REGION, "suspended-group", List.of("i-original"), true);+ } else {+ service.terminateInstanceInAutoScalingGroup(REGION, "i-original", true);+ }+ });+ assertTrue(removalStarted.await(5, TimeUnit.SECONDS));+ try {+ removing.get(250, TimeUnit.MILLISECONDS);+ } catch (TimeoutException expected) {+ // Removal may wait for the shared group monitor before releasing ownership.+ }+ } finally {+ publish.countDown();+ try {+ starting.get(5, TimeUnit.SECONDS);+ if (removing != null) {+ removing.get(5, TimeUnit.SECONDS);+ }+ } finally {+ field.set(service, original);+ executor.shutdownNow();+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));+ }+ }+ InstanceRefresh refresh = service.activeInstanceRefresh(group);+ assertNotNull(refresh);+ assertTrue(refresh.getReplacements().isEmpty(), "Explicit removal retained refresh ownership");+ restart(mode);+ reconciler.reconcile(group);+ assertTrue(group.getInstances().isEmpty());+ if (detach) {+ verify(ec2, never()).terminateInstances(anyString(), anyList());+ } else {+ verify(ec2).terminateInstances(REGION, List.of("i-original"));+ }+ }++ @ParameterizedTest+ @CsvSource({"persistent,true", "persistent,false", "hybrid,true", "hybrid,false", "wal,true", "wal,false"})+ void detachWaitsForReplacementIdentityPublication(String mode, boolean decrement) throws Exception {+ setUp(mode);+ InstanceRefresh refresh = service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ service = spy(service);+ reconciler = new AutoScalingReconciler(service, ec2, mock(ElbV2Service.class));+ CountDownLatch launched = new CountDownLatch(1);+ CountDownLatch publish = new CountDownLatch(1);+ doAnswer(invocation -> {+ if (group.getInstances().stream().anyMatch(i -> "i-replacement".equals(i.getInstanceId()))+ && refresh.getReplacements().getFirst().getReplacementInstanceId() == null) {+ launched.countDown();+ assertTrue(publish.await(5, TimeUnit.SECONDS));+ }+ return invocation.callRealMethod();+ }).when(service).saveAutoScalingGroupIfPresent(group);+ ExecutorService executor = Executors.newFixedThreadPool(2);+ Future<?> reconciling = executor.submit(() -> reconciler.reconcile(group));+ Future<?> detaching = null;+ try {+ assertTrue(launched.await(5, TimeUnit.SECONDS));+ CountDownLatch detachStarted = new CountDownLatch(1);+ detaching = executor.submit(() -> {+ detachStarted.countDown();+ service.detachInstances(REGION, "suspended-group", List.of("i-replacement"), decrement);+ });+ assertTrue(detachStarted.await(5, TimeUnit.SECONDS));+ try {+ detaching.get(250, TimeUnit.MILLISECONDS);+ } catch (TimeoutException expected) {+ // Detach may wait until the replacement ID is published under the group monitor.+ }+ } finally {+ publish.countDown();+ try {+ reconciling.get(5, TimeUnit.SECONDS);+ if (detaching != null) {+ detaching.get(5, TimeUnit.SECONDS);+ }+ } finally {+ executor.shutdownNow();+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));+ }+ }+ assertTrue(group.getInstances().isEmpty());+ assertTrue(refresh.getReplacements().isEmpty(), "Detached replacement retained refresh ownership");+ restart(mode);+ assertTrue(service.activeInstanceRefresh(group).getReplacements().isEmpty());+ assertEquals(decrement ? 0 : 1, group.getDesiredCapacity());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ void deletingSameNameOtherRegionPreservesPausedRefresh(String mode) throws IOException {+ setUp(mode);+ String otherRegion = "us-west-2";+ LaunchTemplate template = new LaunchTemplate();+ template.setLaunchTemplateId("lt-original");+ template.getData().setImageId("ami-original");+ when(ec2.describeLaunchTemplateVersions(eq(otherRegion), eq("lt-original"), isNull(), anyList()))+ .thenReturn(List.of(template));+ service.createAutoScalingGroup(otherRegion, "suspended-group", null, "lt-original", null, "1", null,+ 0, 3, 1, 300, List.of("us-west-2a"), List.of(), List.of(), List.of(), "EC2", 0,+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());+ AutoScalingGroup other = service.describeAutoScalingGroups(otherRegion, List.of("suspended-group"))+ .getFirst();+ AsgInstance member = new AsgInstance();+ member.setInstanceId("i-west");+ member.setLifecycleState("InService");+ other.getInstances().add(member);+ service.saveAutoScalingGroupIfPresent(other);+ service.suspendProcesses(otherRegion, "suspended-group", List.of("InstanceRefresh"));+ InstanceRefresh refresh = service.startInstanceRefresh(otherRegion, "suspended-group", new InstanceRefresh());+ service.deleteAutoScalingGroup(REGION, "suspended-group", true);+ assertSame(refresh, service.activeInstanceRefresh(other), "Peer deletion erased refresh ownership");+ when(ec2.describeInstances(eq(otherRegion), anyList(), isNull())).thenReturn(List.of(reservation("i-west")));+ reconciler.reconcile(other);+ verify(ec2, never()).terminateInstances(eq(otherRegion), anyList());+ assertEquals(List.of("i-west"), other.getInstances().stream().map(AsgInstance::getInstanceId).toList());+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "hybrid", "wal"})+ @SuppressWarnings("unchecked")+ void groupRecreationWaitsForOldRefreshCleanup(String mode) throws Exception {+ setUp(mode);+ Field field = AutoScalingService.class.getDeclaredField("instanceRefreshes");+ field.setAccessible(true);+ Map<String, InstanceRefresh> original = (Map<String, InstanceRefresh>) field.get(service);+ Map<String, InstanceRefresh> intercepted = spy(original);+ CountDownLatch cleanup = new CountDownLatch(1);+ CountDownLatch finish = new CountDownLatch(1);+ AtomicBoolean firstCleanup = new AtomicBoolean(true);+ doAnswer(invocation -> {+ if (firstCleanup.compareAndSet(true, false)) {+ cleanup.countDown();+ assertTrue(finish.await(5, TimeUnit.SECONDS));+ }+ return original.entrySet();+ }).when(intercepted).entrySet();+ field.set(service, intercepted);+ ExecutorService executor = Executors.newFixedThreadPool(2);+ Future<?> deleting = executor.submit(() -> service.deleteAutoScalingGroup(REGION, "suspended-group", true));+ Future<InstanceRefresh> recreating = null;+ InstanceRefresh replacement;+ try {+ assertTrue(cleanup.await(5, TimeUnit.SECONDS));+ CountDownLatch creationStarted = new CountDownLatch(1);+ recreating = executor.submit(() -> {+ creationStarted.countDown();+ service.createAutoScalingGroup(REGION, "suspended-group", null, "lt-original", null, "1", null,+ 0, 3, 0, 300, List.of("us-east-1a"), List.of(), List.of(), List.of(), "EC2", 0,+ List.of("Default"), Map.of(), Map.of(), AsgOptionalFields.none());+ return service.startInstanceRefresh(REGION, "suspended-group", new InstanceRefresh());+ });+ assertTrue(creationStarted.await(5, TimeUnit.SECONDS));+ try {+ recreating.get(250, TimeUnit.MILLISECONDS);+ } catch (TimeoutException expected) {+ // Recreating the same group may wait for the old generation's refresh cleanup.+ }+ } finally {+ finish.countDown();+ try {+ deleting.get(5, TimeUnit.SECONDS);+ replacement = recreating == null ? null : recreating.get(5, TimeUnit.SECONDS);+ } finally {+ field.set(service, original);+ executor.shutdownNow();+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));+ }+ }+ assertNotNull(replacement);+ assertEquals(List.of(replacement.getInstanceRefreshId()), service.describeInstanceRefreshes(+ REGION, "suspended-group", null, null, null).instanceRefreshes().stream()+ .map(InstanceRefresh::getInstanceRefreshId).toList());+ }++ private static Reservation reservation(String id) {+ Instance instance = new Instance();+ instance.setInstanceId(id);+ instance.setState(InstanceState.running());+ Reservation reservation = new Reservation();+ reservation.setInstances(List.of(instance));+ return reservation;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshSuspensionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshSuspensionIntegrationTest.javanew file mode 100644index 0000000000..3ca048910e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingRefreshSuspensionIntegrationTest.java@@ -0,0 +1,284 @@+package io.github.hectorvent.floci.services.autoscaling;++import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import io.restassured.specification.RequestSpecification;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.ArrayList;+import java.util.List;+import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;++@QuarkusTest+class AutoScalingRefreshSuspensionIntegrationTest {+ @Inject+ AutoScalingReconciler reconciler;++ @Inject+ Ec2Service ec2;++ private static final String GROUP = "DescribeAutoScalingGroupsResponse."+ + "DescribeAutoScalingGroupsResult.AutoScalingGroups.member";+ private static final String REFRESH = "DescribeInstanceRefreshesResponse."+ + "DescribeInstanceRefreshesResult.InstanceRefreshes.member";++ @ParameterizedTest+ @ValueSource(strings = {"Launch", "Terminate", "InstanceRefresh", "all"})+ void pausedRefreshKeepsOriginalAndResumesToOneHealthyReplacement(String process) {+ String name = "refresh-pause-" + UUID.randomUUID().toString().substring(0, 8);+ request("CreateLaunchConfiguration").formParam("LaunchConfigurationName", name)+ .formParam("ImageId", "ami-amazonlinux2023").formParam("InstanceType", "t3.micro")+ .post("/").then().statusCode(200);+ try {+ request("CreateAutoScalingGroup").formParam("AutoScalingGroupName", name)+ .formParam("LaunchConfigurationName", name).formParam("MinSize", 0)+ .formParam("MaxSize", 1).formParam("DesiredCapacity", 1)+ .formParam("AvailabilityZones.member.1", "us-east-1a")+ .post("/").then().statusCode(200);+ try {+ reconcileTwice();+ assertEquals("InService", group(name).xmlPath().getString(GROUP + ".Instances.member.LifecycleState"));+ String originalId = group(name).xmlPath().getString(GROUP + ".Instances.member.InstanceId");+ process("SuspendProcesses", name, process);+ String refreshId = request("StartInstanceRefresh").formParam("AutoScalingGroupName", name)+ .post("/").then().statusCode(200).extract().xmlPath()+ .getString("StartInstanceRefreshResponse.StartInstanceRefreshResult.InstanceRefreshId");+ reconcileTwice();+ Response paused = group(name);+ assertEquals(1, paused.xmlPath().getInt(GROUP + ".DesiredCapacity"));+ assertEquals(1, paused.xmlPath().getList(GROUP + ".Instances.member").size());+ assertEquals(originalId, paused.xmlPath().getString(GROUP + ".Instances.member.InstanceId"));+ assertEquals("running", instanceState(originalId));+ Response refresh = refresh(name, refreshId);+ assertEquals("InProgress", refresh.xmlPath().getString(REFRESH + ".Status"));+ assertEquals(1, refresh.xmlPath().getInt(REFRESH + ".InstancesToUpdate"));+ process("ResumeProcesses", name, process);+ reconcileTwice();+ assertEquals("Successful", refresh(name, refreshId).xmlPath().getString(REFRESH + ".Status"));+ Response resumed = group(name);+ assertEquals(1, resumed.xmlPath().getList(GROUP + ".Instances.member").size());+ assertNotEquals(originalId, resumed.xmlPath().getString(GROUP + ".Instances.member.InstanceId"));+ assertEquals("InService", resumed.xmlPath().getString(GROUP + ".Instances.member.LifecycleState"));+ assertEquals("terminated", instanceState(originalId));+ } finally {+ request("DeleteAutoScalingGroup").formParam("AutoScalingGroupName", name)+ .formParam("ForceDelete", true).post("/").then().statusCode(200);+ }+ } finally {+ request("DeleteLaunchConfiguration").formParam("LaunchConfigurationName", name)+ .post("/").then().statusCode(200);+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"configuration", "template"})+ void replacementRetainsEncodedUserDataAndSelectsTheCurrentSubnet(String source) {+ String name = "refresh-placement-" + UUID.randomUUID().toString().substring(0, 8);+ String encoded = "/w";+ String vpcId = ec2.createVpc("us-east-1", "10.92.0.0/16", false).getVpcId();+ List<String> subnets = new ArrayList<>();+ try {+ subnets.add(ec2.createSubnet("us-east-1", vpcId, "10.92.0.0/28", "us-east-1a").getSubnetId());+ subnets.add(ec2.createSubnet("us-east-1", vpcId, "10.92.1.0/24", "us-east-1b").getSubnetId());+ boolean configuration = "configuration".equals(source);+ RequestSpecification create = given().header("Authorization", auth(configuration ? "autoscaling" : "ec2"))+ .formParam("Action", configuration ? "CreateLaunchConfiguration" : "CreateLaunchTemplate")+ .formParam(configuration ? "LaunchConfigurationName" : "LaunchTemplateName", name);+ String prefix = configuration ? "" : "LaunchTemplateData.";+ if (!configuration) {+ create.formParam("LaunchTemplateData.BlockDeviceMapping.1.DeviceName", "/dev/xvda")+ .formParam("LaunchTemplateData.BlockDeviceMapping.1.Ebs.VolumeType", "gp2");+ }+ create.formParam(prefix + "ImageId", "ami-amazonlinux2023")+ .formParam(prefix + "InstanceType", "t3.micro").formParam(prefix + "UserData", encoded)+ .post("/").then().statusCode(200);+ try {+ RequestSpecification createGroup = request("CreateAutoScalingGroup")+ .formParam("AutoScalingGroupName", name).formParam("MinSize", 0)+ .formParam("MaxSize", 1).formParam("DesiredCapacity", 1)+ .formParam("VPCZoneIdentifier", subnets.getFirst())+ .formParam("AvailabilityZones.member.1", "us-east-1a");+ if (configuration) {+ createGroup.formParam("LaunchConfigurationName", name);+ } else {+ createGroup.formParam("LaunchTemplate.LaunchTemplateName", name)+ .formParam("LaunchTemplate.Version", "1");+ }+ createGroup.post("/").then().statusCode(200);+ try {+ reconcileTwice();+ String original = group(name).xmlPath().getString(GROUP + ".Instances.member.InstanceId");+ assertEquals(subnets.getFirst(), ec2.getInstance("us-east-1", original).orElseThrow().getSubnetId());+ assertEquals(configuration ? "gp3" : "gp2", rootVolumeType(original));+ request("UpdateAutoScalingGroup").formParam("AutoScalingGroupName", name)+ .formParam("VPCZoneIdentifier", String.join(",", subnets))+ .post("/").then().statusCode(200);+ String refreshId = request("StartInstanceRefresh").formParam("AutoScalingGroupName", name)+ .post("/").then().statusCode(200).extract().xmlPath()+ .getString("StartInstanceRefreshResponse.StartInstanceRefreshResult.InstanceRefreshId");+ reconcileTwice();+ Response current = group(name);+ assertEquals(1, current.xmlPath().getList(GROUP + ".Instances.member").size());+ String replacement = current.xmlPath().getString(GROUP + ".Instances.member.InstanceId");+ assertNotEquals(original, replacement);+ assertEquals(configuration ? "gp3" : "gp2", rootVolumeType(replacement));+ assertEquals("terminated", instanceState(original));+ assertEquals(subnets.getLast(), ec2.getInstance("us-east-1", replacement).orElseThrow().getSubnetId());+ assertEquals("us-east-1b", current.xmlPath().getString(GROUP + ".Instances.member.AvailabilityZone"));+ assertEquals("Successful", refresh(name, refreshId).xmlPath().getString(REFRESH + ".Status"));+ assertEquals(encoded, given().header("Authorization", auth("ec2"))+ .formParam("Action", "DescribeInstanceAttribute").formParam("InstanceId", replacement)+ .formParam("Attribute", "userData").post("/").then().statusCode(200)+ .extract().xmlPath().getString("DescribeInstanceAttributeResponse.userData.value"));+ } finally {+ request("DeleteAutoScalingGroup").formParam("AutoScalingGroupName", name)+ .formParam("ForceDelete", true).post("/").then().statusCode(200);+ }+ } finally {+ given().header("Authorization", auth(configuration ? "autoscaling" : "ec2"))+ .formParam("Action", configuration ? "DeleteLaunchConfiguration" : "DeleteLaunchTemplate")+ .formParam(configuration ? "LaunchConfigurationName" : "LaunchTemplateName", name)+ .post("/").then().statusCode(200);+ }+ } finally {+ for (String subnet : subnets) {+ ec2.deleteSubnet("us-east-1", subnet);+ }+ ec2.deleteVpc("us-east-1", vpcId);+ }+ }++ @Test+ void sameNameRefreshesInDifferentAccountsResumeIndependently() {+ String name = "refresh-accounts-" + UUID.randomUUID().toString().substring(0, 8);+ List<String> accounts = List.of("111122223333", "444455556666");+ List<String> configurations = new ArrayList<>();+ List<String> groups = new ArrayList<>();+ List<String> originals = new ArrayList<>();+ try {+ for (String account : accounts) {+ accountRequest(account, "CreateLaunchConfiguration")+ .formParam("LaunchConfigurationName", name).formParam("ImageId", "ami-amazonlinux2023")+ .formParam("InstanceType", "t3.micro").post("/").then().statusCode(200);+ configurations.add(account);+ accountRequest(account, "CreateAutoScalingGroup").formParam("AutoScalingGroupName", name)+ .formParam("LaunchConfigurationName", name).formParam("MinSize", 0)+ .formParam("MaxSize", 1).formParam("DesiredCapacity", 1)+ .formParam("AvailabilityZones.member.1", "us-east-1a").post("/").then().statusCode(200);+ groups.add(account);+ }+ reconcileTwice();+ for (String account : accounts) {+ originals.add(accountGroup(account, name).xmlPath().getString(GROUP + ".Instances.member.InstanceId"));+ accountRequest(account, "SuspendProcesses").formParam("AutoScalingGroupName", name)+ .formParam("ScalingProcesses.member.1", "InstanceRefresh").post("/").then().statusCode(200);+ accountRequest(account, "StartInstanceRefresh").formParam("AutoScalingGroupName", name)+ .post("/").then().statusCode(200);+ }+ assertNotEquals(originals.getFirst(), originals.getLast());+ reconcileTwice();+ for (int index = 0; index < accounts.size(); index++) {+ assertEquals(originals.get(index), accountGroup(accounts.get(index), name)+ .xmlPath().getString(GROUP + ".Instances.member.InstanceId"));+ }+ accountRequest(accounts.getFirst(), "ResumeProcesses").formParam("AutoScalingGroupName", name)+ .formParam("ScalingProcesses.member.1", "InstanceRefresh").post("/").then().statusCode(200);+ reconcileTwice();+ assertNotEquals(originals.getFirst(), accountGroup(accounts.getFirst(), name)+ .xmlPath().getString(GROUP + ".Instances.member.InstanceId"));+ assertEquals("Successful", accountRefreshStatus(accounts.getFirst(), name));+ assertEquals(originals.getLast(), accountGroup(accounts.getLast(), name)+ .xmlPath().getString(GROUP + ".Instances.member.InstanceId"));+ assertEquals("InProgress", accountRefreshStatus(accounts.getLast(), name));+ accountRequest(accounts.getLast(), "ResumeProcesses").formParam("AutoScalingGroupName", name)+ .formParam("ScalingProcesses.member.1", "InstanceRefresh").post("/").then().statusCode(200);+ reconcileTwice();+ assertNotEquals(originals.getLast(), accountGroup(accounts.getLast(), name)+ .xmlPath().getString(GROUP + ".Instances.member.InstanceId"));+ assertEquals("Successful", accountRefreshStatus(accounts.getLast(), name));+ } finally {+ try {+ for (String account : groups) {+ accountRequest(account, "DeleteAutoScalingGroup").formParam("AutoScalingGroupName", name)+ .formParam("ForceDelete", true).post("/").then().statusCode(200);+ }+ } finally {+ for (String account : configurations) {+ accountRequest(account, "DeleteLaunchConfiguration").formParam("LaunchConfigurationName", name)+ .post("/").then().statusCode(200);+ }+ }+ }+ }++ private String rootVolumeType(String instanceId) {+ String volumeId = ec2.getInstance("us-east-1", instanceId).orElseThrow().getRootVolumeId();+ return given().header("Authorization", auth("ec2")).formParam("Action", "DescribeVolumes")+ .formParam("VolumeId.1", volumeId).post("/").then().statusCode(200)+ .extract().xmlPath().getString("DescribeVolumesResponse.volumeSet.item.volumeType");+ }++ private static String accountRefreshStatus(String account, String name) {+ return accountRequest(account, "DescribeInstanceRefreshes").formParam("AutoScalingGroupName", name)+ .post("/").then().statusCode(200).extract().xmlPath().getString(REFRESH + ".Status");+ }++ private static Response accountGroup(String account, String name) {+ return accountRequest(account, "DescribeAutoScalingGroups").formParam("AutoScalingGroupNames.member.1", name)+ .post("/").then().statusCode(200).extract().response();+ }++ private static RequestSpecification accountRequest(String account, String action) {+ return given().header("Authorization", "AWS4-HMAC-SHA256 Credential=" + account+ + "/20260215/us-east-1/autoscaling/aws4_request, SignedHeaders=host, Signature=abc")+ .formParam("Action", action);+ }++ private void reconcileTwice() {+ reconciler.reconcileAll();+ reconciler.reconcileAll();+ }++ private static Response group(String name) {+ return request("DescribeAutoScalingGroups").formParam("AutoScalingGroupNames.member.1", name)+ .post("/").then().statusCode(200).extract().response();+ }++ private static Response refresh(String name, String refreshId) {+ return request("DescribeInstanceRefreshes").formParam("AutoScalingGroupName", name)+ .formParam("InstanceRefreshIds.member.1", refreshId)+ .post("/").then().statusCode(200).extract().response();+ }++ private static String instanceState(String id) {+ return given().header("Authorization", auth("ec2"))+ .formParam("Action", "DescribeInstances").formParam("InstanceId.1", id)+ .post("/").then().statusCode(200).extract().xmlPath()+ .getString("DescribeInstancesResponse.reservationSet.item.instancesSet.item.instanceState.name");+ }++ private static void process(String action, String name, String process) {+ RequestSpecification request = request(action).formParam("AutoScalingGroupName", name);+ if (!"all".equals(process)) {+ request.formParam("ScalingProcesses.member.1", process);+ }+ request.post("/").then().statusCode(200);+ }++ private static RequestSpecification request(String action) {+ return given().header("Authorization", auth("autoscaling")).formParam("Action", action);+ }++ private static String auth(String service) {+ return "AWS4-HMAC-SHA256 Credential=AKID/20260215/us-east-1/" + service+ + "/aws4_request, SignedHeaders=host, Signature=abc";+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.javaindex de792100f3..502de53a4c 100644--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingServiceTest.java@@ -542,6 +542,7 @@ class AutoScalingServiceTest {group.getInstances().clear();AutoScalingGroupFixture.addInstance(service, REGION, "test-asg", "i-replacement", "InService", "lt-original", "2");+ refresh.getReplacements().getFirst().setReplacementInstanceId("i-replacement");service.completeInstanceRefreshIfSettled(REGION, "test-asg");InstanceRefresh completed = service.describeInstanceRefreshes(@@ -569,6 +570,7 @@ class AutoScalingServiceTest {group.getInstances().clear();AutoScalingGroupFixture.addInstance(service, REGION, "test-asg", "i-replacement", "InService", "lt-original", "2");+ refresh.getReplacements().getFirst().setReplacementInstanceId("i-replacement");service.completeInstanceRefreshIfSettled(REGION, "test-asg");InstanceRefresh completed = service.describeInstanceRefreshes(@@ -622,6 +624,7 @@ class AutoScalingServiceTest {group.getInstances().clear();AutoScalingGroupFixture.addInstance(service, REGION, "test-asg", "i-replacement", "InService", "lt-updated", "2");+ refresh.getReplacements().getFirst().setReplacementInstanceId("i-replacement");service.completeInstanceRefreshIfSettled(REGION, "test-asg");InstanceRefresh completed = service.describeInstanceRefreshes(