Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(autoscaling): fail SSM commands for stale ASG instances (#1600)

Jason van Zyla895802b8b25
diff --git a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
index 50cddfb693..1a6506c7f8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconciler.java
@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.services.ec2.model.Reservation;
import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
import io.github.hectorvent.floci.services.elbv2.model.TargetDescription;
import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;
+import io.github.hectorvent.floci.services.ssm.SsmCommandService;
import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Observes;
@@ -34,15 +35,22 @@ public class AutoScalingReconciler {
private final AutoScalingService asgService;
private final Ec2Service ec2Service;
private final ElbV2Service elbV2Service;
+ private final SsmCommandService ssmCommandService;
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(
r -> new Thread(r, "asg-reconciler"));
@Inject
AutoScalingReconciler(AutoScalingService asgService, Ec2Service ec2Service,
- ElbV2Service elbV2Service) {
+ ElbV2Service elbV2Service, SsmCommandService ssmCommandService) {
this.asgService = asgService;
this.ec2Service = ec2Service;
this.elbV2Service = elbV2Service;
+ this.ssmCommandService = ssmCommandService;
+ }
+
+ AutoScalingReconciler(AutoScalingService asgService, Ec2Service ec2Service,
+ ElbV2Service elbV2Service) {
+ this(asgService, ec2Service, elbV2Service, null);
}
@PostConstruct
@@ -138,6 +146,7 @@ public class AutoScalingReconciler {
List<String> instanceIds = staleInstances.stream()
.map(AsgInstance::getInstanceId)
.collect(Collectors.toList());
+ failActiveSsmInvocations(asg, instanceIds);
asg.getInstances().removeIf(instance -> instanceIds.contains(instance.getInstanceId()));
asgService.saveAutoScalingGroup(asg);
asgService.recordActivity(asg.getRegion(), asg.getAutoScalingGroupName(),
@@ -148,6 +157,20 @@ public class AutoScalingReconciler {
asg.getAutoScalingGroupName(), instanceIds);
}
+ private void failActiveSsmInvocations(AutoScalingGroup asg, List<String> instanceIds) {
+ if (ssmCommandService == null) {
+ return;
+ }
+ int failed = ssmCommandService.failActiveInvocationsForInstances(
+ asg.getRegion(),
+ Set.copyOf(instanceIds),
+ "Undeliverable");
+ if (failed > 0) {
+ LOG.infov("ASG {0}: marked {1} active SSM command invocation(s) failed for stale instances {2}",
+ asg.getAutoScalingGroupName(), failed, instanceIds);
+ }
+ }
+
private boolean isStaleInstance(AutoScalingGroup asg, AsgInstance instance) {
String lifecycleState = instance.getLifecycleState();
if ("InService".equals(lifecycleState)) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
index f43abc68e7..e92d6dd569 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
@@ -21,9 +21,11 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Queue;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
@@ -275,6 +277,51 @@ public class SsmCommandService implements Resettable {
LOG.infov("CancelCommand: commandId={0}", commandId);
}
+ public int failActiveInvocationsForInstance(String region, String instanceId, String statusDetails) {
+ return failActiveInvocationsForInstances(region, Set.of(instanceId), statusDetails);
+ }
+
+ public int failActiveInvocationsForInstances(String region, Set<String> instanceIds, String statusDetails) {
+ if (instanceIds == null || instanceIds.isEmpty()) {
+ return 0;
+ }
+ String prefix = region + "::";
+ Instant now = Instant.now();
+ Set<String> commandIds = new LinkedHashSet<>();
+ int failed = 0;
+
+ for (CommandInvocation invocation : invocationStore.scan(key -> key.startsWith(prefix))) {
+ if (!instanceIds.contains(invocation.getInstanceId())) {
+ continue;
+ }
+ if (!isActiveInvocation(invocation.getStatus())) {
+ continue;
+ }
+ invocation.setStatus("Failed");
+ invocation.setStatusDetails(statusDetails);
+ invocation.setResponseCode(-1);
+ invocation.setExecutionEndDateTime(now);
+ invocationStore.put(invocationKey(region, invocation.getCommandId(), invocation.getInstanceId()), invocation);
+ commandIds.add(invocation.getCommandId());
+ failed++;
+ }
+
+ for (String instanceId : instanceIds) {
+ Queue<PendingMessage> queue = messageQueues.remove(instanceId);
+ if (queue != null) {
+ queue.clear();
+ }
+ }
+ messageIndex.entrySet().removeIf(entry -> {
+ String[] meta = entry.getValue();
+ return meta.length == 3
+ && instanceIds.contains(meta[1])
+ && region.equals(meta[2]);
+ });
+ commandIds.forEach(commandId -> updateCommandStatus(commandId, region));
+ return failed;
+ }
+
// ── ec2messages agent protocol ──────────────────────────────────────────
public List<Map<String, Object>> getMessages(String instanceId, String messagesRequestId, int visibilityTimeout) {
@@ -665,6 +712,10 @@ public class SsmCommandService implements Resettable {
};
}
+ private static boolean isActiveInvocation(String status) {
+ return "Pending".equals(status) || "InProgress".equals(status);
+ }
+
private static String statusDetails(String status) {
return switch (status) {
case "InProgress" -> "In Progress";
diff --git a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
index 8d36e17338..d7b1f77fc4 100644
--- a/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingReconcilerTest.java
@@ -12,11 +12,13 @@ import io.github.hectorvent.floci.services.ec2.model.Tag;
import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
import io.github.hectorvent.floci.services.elbv2.model.TargetDescription;
import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;
+import io.github.hectorvent.floci.services.ssm.SsmCommandService;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.eq;
@@ -293,6 +295,36 @@ class AutoScalingReconcilerTest {
eq("Successful"));
}
+ @Test
+ void reconcileFailsActiveSsmInvocationsBeforePruningStaleInstance() {
+ AutoScalingService asgService = mock(AutoScalingService.class);
+ Ec2Service ec2Service = mock(Ec2Service.class);
+ ElbV2Service elbV2Service = mock(ElbV2Service.class);
+ SsmCommandService ssmCommandService = mock(SsmCommandService.class);
+ AutoScalingReconciler reconciler = new AutoScalingReconciler(
+ asgService, ec2Service, elbV2Service, ssmCommandService);
+ AutoScalingGroup asg = new AutoScalingGroup();
+ asg.setRegion("us-east-1");
+ asg.setAutoScalingGroupName("app-asg");
+ asg.setDesiredCapacity(0);
+ asg.getInstances().add(instance("i-dead", "Pending"));
+
+ Instance ec2Instance = new Instance();
+ ec2Instance.setInstanceId("i-dead");
+ ec2Instance.setState(InstanceState.terminated());
+ Reservation reservation = new Reservation();
+ reservation.setInstances(List.of(ec2Instance));
+ when(ec2Service.describeInstances("us-east-1", List.of("i-dead"), null))
+ .thenReturn(List.of(reservation));
+ when(ssmCommandService.failActiveInvocationsForInstances("us-east-1", Set.of("i-dead"), "Undeliverable"))
+ .thenReturn(1);
+
+ reconciler.reconcile(asg);
+
+ assertEquals(0, asg.getInstances().size());
+ verify(ssmCommandService).failActiveInvocationsForInstances("us-east-1", Set.of("i-dead"), "Undeliverable");
+ }
+
@Test
void reconcilePromotesPendingAsgInstanceWhenEc2InstanceIsRunning() {
AutoScalingService asgService = mock(AutoScalingService.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java
index db704f2192..be6ebf5d17 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java
@@ -13,6 +13,7 @@ import org.junit.jupiter.api.Test;
import java.time.Instant;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -98,6 +99,71 @@ class SsmCommandServiceDirectExecutionTest {
assertEquals(1, service.getMessages("i-agent", "request-id", 30).size());
}
+ @Test
+ void unavailableInstanceFailsQueuedCommandInvocation() throws Exception {
+ SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);
+ when(executor.supports(eq("i-stale"), eq("AWS-RunShellScript"))).thenReturn(false);
+
+ SsmCommandService service = new SsmCommandService(
+ new InMemoryStorageFactory(),
+ objectMapper,
+ regionResolver,
+ executor);
+
+ Command command = service.sendCommand(objectMapper.readTree("""
+ {
+ "InstanceIds": ["i-stale"],
+ "DocumentName": "AWS-RunShellScript",
+ "Parameters": {
+ "commands": ["echo hello"]
+ },
+ "TimeoutSeconds": 60
+ }
+ """), "us-west-2");
+
+ assertEquals(1, service.failActiveInvocationsForInstance("us-west-2", "i-stale", "Undeliverable"));
+
+ CommandInvocation invocation = service.getCommandInvocation(command.getCommandId(), "i-stale", "us-west-2");
+ assertEquals("Failed", invocation.getStatus());
+ assertEquals("Undeliverable", invocation.getStatusDetails());
+ assertEquals(-1, invocation.getResponseCode());
+ assertEquals("Failed", service.listCommands(command.getCommandId(), null, "us-west-2").getFirst().getStatus());
+ assertTrue(service.getMessages("i-stale", "request-id", 30).isEmpty());
+ }
+
+ @Test
+ void unavailableInstancesFailQueuedCommandInvocationsInBulk() throws Exception {
+ SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);
+ when(executor.supports(any(), eq("AWS-RunShellScript"))).thenReturn(false);
+
+ SsmCommandService service = new SsmCommandService(
+ new InMemoryStorageFactory(),
+ objectMapper,
+ regionResolver,
+ executor);
+
+ Command command = service.sendCommand(objectMapper.readTree("""
+ {
+ "InstanceIds": ["i-stale-a", "i-stale-b", "i-active"],
+ "DocumentName": "AWS-RunShellScript",
+ "Parameters": {
+ "commands": ["echo hello"]
+ },
+ "TimeoutSeconds": 60
+ }
+ """), "us-west-2");
+
+ assertEquals(2, service.failActiveInvocationsForInstances(
+ "us-west-2", Set.of("i-stale-a", "i-stale-b"), "Undeliverable"));
+
+ assertEquals("Failed", service.getCommandInvocation(command.getCommandId(), "i-stale-a", "us-west-2").getStatus());
+ assertEquals("Failed", service.getCommandInvocation(command.getCommandId(), "i-stale-b", "us-west-2").getStatus());
+ assertEquals("Pending", service.getCommandInvocation(command.getCommandId(), "i-active", "us-west-2").getStatus());
+ assertTrue(service.getMessages("i-stale-a", "request-id", 30).isEmpty());
+ assertTrue(service.getMessages("i-stale-b", "request-id", 30).isEmpty());
+ assertEquals(1, service.getMessages("i-active", "request-id", 30).size());
+ }
+
@Test
void directTimeoutMarksCommandTimedOut() throws Exception {
SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);