Light, fluffy, and always free - The AWS Local Emulator alternative
fix(autoscaling): fail SSM commands for stale ASG instances (#1600)
Jason van Zyla895802b8b25
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.javaindex 50cddfb693..1a6506c7f8 100644--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.services.ec2.model.Reservation;import io.github.hectorvent.floci.services.elbv2.ElbV2Service;import io.github.hectorvent.floci.services.elbv2.model.TargetDescription;import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;+import io.github.hectorvent.floci.services.ssm.SsmCommandService;import jakarta.annotation.PostConstruct;import jakarta.enterprise.context.ApplicationScoped;import jakarta.enterprise.event.Observes;@@ -34,15 +35,22 @@ public class AutoScalingReconciler {private final AutoScalingService asgService;private final Ec2Service ec2Service;private final ElbV2Service elbV2Service;+ private final SsmCommandService ssmCommandService;private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "asg-reconciler"));@InjectAutoScalingReconciler(AutoScalingService asgService, Ec2Service ec2Service,- ElbV2Service elbV2Service) {+ ElbV2Service elbV2Service, SsmCommandService ssmCommandService) {this.asgService = asgService;this.ec2Service = ec2Service;this.elbV2Service = elbV2Service;+ this.ssmCommandService = ssmCommandService;+ }++ AutoScalingReconciler(AutoScalingService asgService, Ec2Service ec2Service,+ ElbV2Service elbV2Service) {+ this(asgService, ec2Service, elbV2Service, null);}@PostConstruct@@ -138,6 +146,7 @@ public class AutoScalingReconciler {List<String> instanceIds = staleInstances.stream().map(AsgInstance::getInstanceId).collect(Collectors.toList());+ failActiveSsmInvocations(asg, instanceIds);asg.getInstances().removeIf(instance -> instanceIds.contains(instance.getInstanceId()));asgService.saveAutoScalingGroup(asg);asgService.recordActivity(asg.getRegion(), asg.getAutoScalingGroupName(),@@ -148,6 +157,20 @@ public class AutoScalingReconciler {asg.getAutoScalingGroupName(), instanceIds);}+ private void failActiveSsmInvocations(AutoScalingGroup asg, List<String> instanceIds) {+ if (ssmCommandService == null) {+ return;+ }+ int failed = ssmCommandService.failActiveInvocationsForInstances(+ asg.getRegion(),+ Set.copyOf(instanceIds),+ "Undeliverable");+ if (failed > 0) {+ LOG.infov("ASG {0}: marked {1} active SSM command invocation(s) failed for stale instances {2}",+ asg.getAutoScalingGroupName(), failed, instanceIds);+ }+ }+private boolean isStaleInstance(AutoScalingGroup asg, AsgInstance instance) {String lifecycleState = instance.getLifecycleState();if ("InService".equals(lifecycleState)) {diff --git a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.javaindex f43abc68e7..e92d6dd569 100644--- a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java@@ -21,9 +21,11 @@ import java.time.Instant;import java.util.ArrayList;import java.util.Base64;import java.util.Iterator;+import java.util.LinkedHashSet;import java.util.List;import java.util.Map;import java.util.Queue;+import java.util.Set;import java.util.UUID;import java.util.concurrent.CompletableFuture;import java.util.concurrent.ConcurrentHashMap;@@ -275,6 +277,51 @@ public class SsmCommandService implements Resettable {LOG.infov("CancelCommand: commandId={0}", commandId);}+ public int failActiveInvocationsForInstance(String region, String instanceId, String statusDetails) {+ return failActiveInvocationsForInstances(region, Set.of(instanceId), statusDetails);+ }++ public int failActiveInvocationsForInstances(String region, Set<String> instanceIds, String statusDetails) {+ if (instanceIds == null || instanceIds.isEmpty()) {+ return 0;+ }+ String prefix = region + "::";+ Instant now = Instant.now();+ Set<String> commandIds = new LinkedHashSet<>();+ int failed = 0;++ for (CommandInvocation invocation : invocationStore.scan(key -> key.startsWith(prefix))) {+ if (!instanceIds.contains(invocation.getInstanceId())) {+ continue;+ }+ if (!isActiveInvocation(invocation.getStatus())) {+ continue;+ }+ invocation.setStatus("Failed");+ invocation.setStatusDetails(statusDetails);+ invocation.setResponseCode(-1);+ invocation.setExecutionEndDateTime(now);+ invocationStore.put(invocationKey(region, invocation.getCommandId(), invocation.getInstanceId()), invocation);+ commandIds.add(invocation.getCommandId());+ failed++;+ }++ for (String instanceId : instanceIds) {+ Queue<PendingMessage> queue = messageQueues.remove(instanceId);+ if (queue != null) {+ queue.clear();+ }+ }+ messageIndex.entrySet().removeIf(entry -> {+ String[] meta = entry.getValue();+ return meta.length == 3+ && instanceIds.contains(meta[1])+ && region.equals(meta[2]);+ });+ commandIds.forEach(commandId -> updateCommandStatus(commandId, region));+ return failed;+ }+// ── ec2messages agent protocol ──────────────────────────────────────────public List<Map<String, Object>> getMessages(String instanceId, String messagesRequestId, int visibilityTimeout) {@@ -665,6 +712,10 @@ public class SsmCommandService implements Resettable {};}+ private static boolean isActiveInvocation(String status) {+ return "Pending".equals(status) || "InProgress".equals(status);+ }+private static String statusDetails(String status) {return switch (status) {case "InProgress" -> "In Progress";diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.javaindex 8d36e17338..d7b1f77fc4 100644--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java@@ -12,11 +12,13 @@ import io.github.hectorvent.floci.services.ec2.model.Tag;import io.github.hectorvent.floci.services.elbv2.ElbV2Service;import io.github.hectorvent.floci.services.elbv2.model.TargetDescription;import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;+import io.github.hectorvent.floci.services.ssm.SsmCommandService;import org.junit.jupiter.api.Test;import org.mockito.ArgumentCaptor;import java.util.List;import java.util.Map;+import java.util.Set;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.mockito.Mockito.eq;@@ -293,6 +295,36 @@ class AutoScalingReconcilerTest {eq("Successful"));}+ @Test+ void reconcileFailsActiveSsmInvocationsBeforePruningStaleInstance() {+ AutoScalingService asgService = mock(AutoScalingService.class);+ Ec2Service ec2Service = mock(Ec2Service.class);+ ElbV2Service elbV2Service = mock(ElbV2Service.class);+ SsmCommandService ssmCommandService = mock(SsmCommandService.class);+ AutoScalingReconciler reconciler = new AutoScalingReconciler(+ asgService, ec2Service, elbV2Service, ssmCommandService);+ AutoScalingGroup asg = new AutoScalingGroup();+ asg.setRegion("us-east-1");+ asg.setAutoScalingGroupName("app-asg");+ asg.setDesiredCapacity(0);+ asg.getInstances().add(instance("i-dead", "Pending"));++ Instance ec2Instance = new Instance();+ ec2Instance.setInstanceId("i-dead");+ ec2Instance.setState(InstanceState.terminated());+ Reservation reservation = new Reservation();+ reservation.setInstances(List.of(ec2Instance));+ when(ec2Service.describeInstances("us-east-1", List.of("i-dead"), null))+ .thenReturn(List.of(reservation));+ when(ssmCommandService.failActiveInvocationsForInstances("us-east-1", Set.of("i-dead"), "Undeliverable"))+ .thenReturn(1);++ reconciler.reconcile(asg);++ assertEquals(0, asg.getInstances().size());+ verify(ssmCommandService).failActiveInvocationsForInstances("us-east-1", Set.of("i-dead"), "Undeliverable");+ }+@Testvoid reconcilePromotesPendingAsgInstanceWhenEc2InstanceIsRunning() {AutoScalingService asgService = mock(AutoScalingService.class);diff --git a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.javaindex db704f2192..be6ebf5d17 100644--- a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java@@ -13,6 +13,7 @@ import org.junit.jupiter.api.Test;import java.time.Instant;import java.util.Map;import java.util.Optional;+import java.util.Set;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;@@ -98,6 +99,71 @@ class SsmCommandServiceDirectExecutionTest {assertEquals(1, service.getMessages("i-agent", "request-id", 30).size());}+ @Test+ void unavailableInstanceFailsQueuedCommandInvocation() throws Exception {+ SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);+ when(executor.supports(eq("i-stale"), eq("AWS-RunShellScript"))).thenReturn(false);++ SsmCommandService service = new SsmCommandService(+ new InMemoryStorageFactory(),+ objectMapper,+ regionResolver,+ executor);++ Command command = service.sendCommand(objectMapper.readTree("""+ {+ "InstanceIds": ["i-stale"],+ "DocumentName": "AWS-RunShellScript",+ "Parameters": {+ "commands": ["echo hello"]+ },+ "TimeoutSeconds": 60+ }+ """), "us-west-2");++ assertEquals(1, service.failActiveInvocationsForInstance("us-west-2", "i-stale", "Undeliverable"));++ CommandInvocation invocation = service.getCommandInvocation(command.getCommandId(), "i-stale", "us-west-2");+ assertEquals("Failed", invocation.getStatus());+ assertEquals("Undeliverable", invocation.getStatusDetails());+ assertEquals(-1, invocation.getResponseCode());+ assertEquals("Failed", service.listCommands(command.getCommandId(), null, "us-west-2").getFirst().getStatus());+ assertTrue(service.getMessages("i-stale", "request-id", 30).isEmpty());+ }++ @Test+ void unavailableInstancesFailQueuedCommandInvocationsInBulk() throws Exception {+ SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);+ when(executor.supports(any(), eq("AWS-RunShellScript"))).thenReturn(false);++ SsmCommandService service = new SsmCommandService(+ new InMemoryStorageFactory(),+ objectMapper,+ regionResolver,+ executor);++ Command command = service.sendCommand(objectMapper.readTree("""+ {+ "InstanceIds": ["i-stale-a", "i-stale-b", "i-active"],+ "DocumentName": "AWS-RunShellScript",+ "Parameters": {+ "commands": ["echo hello"]+ },+ "TimeoutSeconds": 60+ }+ """), "us-west-2");++ assertEquals(2, service.failActiveInvocationsForInstances(+ "us-west-2", Set.of("i-stale-a", "i-stale-b"), "Undeliverable"));++ assertEquals("Failed", service.getCommandInvocation(command.getCommandId(), "i-stale-a", "us-west-2").getStatus());+ assertEquals("Failed", service.getCommandInvocation(command.getCommandId(), "i-stale-b", "us-west-2").getStatus());+ assertEquals("Pending", service.getCommandInvocation(command.getCommandId(), "i-active", "us-west-2").getStatus());+ assertTrue(service.getMessages("i-stale-a", "request-id", 30).isEmpty());+ assertTrue(service.getMessages("i-stale-b", "request-id", 30).isEmpty());+ assertEquals(1, service.getMessages("i-active", "request-id", 30).size());+ }+@Testvoid directTimeoutMarksCommandTimedOut() throws Exception {SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);