Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ssm): retain command ownership during background execution (#5841)

David Phillipsd4158b954b14
diff --git a/docs/services/ssm.md b/docs/services/ssm.md
index eab4b49b30..0df68d320b 100644
--- a/docs/services/ssm.md
+++ b/docs/services/ssm.md
@@ -51,6 +51,11 @@
If the target is not a Floci EC2 container, or if the document is not supported for direct execution, Floci falls back to the SSM agent polling flow. In that mode, `SendCommand` queues an ec2messages payload and the invocation completes after an agent calls `SendReply`.
+Background execution retains the issuing account and region. Agent queues and message correlations
+use that same account, region and instance identity; polling or completing a message from another
+account or region leaves the owner's work untouched. Pending messages remain in memory and are lost
+on restart.
+
!!! note "Managed instance registration is independent of EC2 and IMDS"
Registering a container's SSM agent with `UpdateInstanceInformation` only creates the managed instance record. It does not create an EC2 instance, does not install the link-local `169.254.169.254` proxy in the container, and does not register the container with the [EC2 IMDS server](ec2.md#imds-and-ssm-managed-instances). A container that was only registered with SSM therefore has no working IMDS endpoint and no instance profile credentials, and `SendCommand` against it uses the agent polling flow rather than direct execution.
diff --git a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
index e8d6984363..5630a5bb54 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
@@ -57,10 +58,10 @@ public class SsmCommandService implements Resettable {
private final SsmDirectCommandExecutor directCommandExecutor;
private final ExecutorService directExecutionExecutor;
- // In-memory queues: instanceId → pending messages. Not persisted — lost on restart.
- private final ConcurrentHashMap<String, Queue<PendingMessage>> messageQueues = new ConcurrentHashMap<>();
- // messageId → (commandId, instanceId) for correlating SendReply back to an invocation
- private final ConcurrentHashMap<String, String[]> messageIndex = new ConcurrentHashMap<>();
+ // In-memory queues are owned by the issuing account, region and instance. Lost on restart.
+ private final ConcurrentHashMap<MessageOwner, Queue<PendingMessage>> messageQueues = new ConcurrentHashMap<>();
+ // Message correlations retain the same owner through every agent operation.
+ private final ConcurrentHashMap<String, MessageCorrelation> messageIndex = new ConcurrentHashMap<>();
@Inject
public SsmCommandService(
@@ -295,7 +296,7 @@ public class SsmCommandService implements Resettable {
}
}
// Remove any queued (not-yet-polled) messages for this instance
- Queue<PendingMessage> q = messageQueues.get(instanceId);
+ Queue<PendingMessage> q = messageQueues.get(messageOwner(region, instanceId));
if (q != null) {
q.removeIf(m -> m.commandId().equals(commandId));
}
@@ -344,16 +345,15 @@ public class SsmCommandService implements Resettable {
}
for (String instanceId : instanceIds) {
- Queue<PendingMessage> queue = messageQueues.remove(instanceId);
+ Queue<PendingMessage> queue = messageQueues.remove(messageOwner(region, instanceId));
if (queue != null) {
queue.clear();
}
}
messageIndex.entrySet().removeIf(entry -> {
- String[] meta = entry.getValue();
- return meta.length == 3
- && instanceIds.contains(meta[1])
- && region.equals(meta[2]);
+ MessageOwner owner = entry.getValue().owner();
+ return instanceIds.contains(owner.instanceId())
+ && owner.equals(messageOwner(region, owner.instanceId()));
});
commandIds.forEach(commandId -> updateCommandStatus(commandId, region));
return failed;
@@ -362,7 +362,8 @@ public class SsmCommandService implements Resettable {
// ── ec2messages agent protocol ──────────────────────────────────────────
public List<Map<String, Object>> getMessages(String instanceId, String messagesRequestId, int visibilityTimeout) {
- Queue<PendingMessage> queue = messageQueues.get(instanceId);
+ MessageOwner owner = messageOwner(regionResolver.getRegion(), instanceId);
+ Queue<PendingMessage> queue = messageQueues.get(owner);
List<Map<String, Object>> result = new ArrayList<>();
if (queue == null || queue.isEmpty()) {
return result;
@@ -374,7 +375,7 @@ public class SsmCommandService implements Resettable {
}
// Track for AcknowledgeMessage / SendReply correlation
- messageIndex.put(msg.messageId(), new String[]{msg.commandId(), instanceId, msg.region()});
+ messageIndex.put(msg.messageId(), new MessageCorrelation(owner, msg.commandId()));
result.add(Map.of(
"MessageId", msg.messageId(),
@@ -390,13 +391,13 @@ public class SsmCommandService implements Resettable {
public void acknowledgeMessage(String messageId) {
// Message was already removed from queue on GetMessages. Just update invocation to InProgress.
- String[] meta = messageIndex.get(messageId);
+ MessageCorrelation meta = ownedCorrelation(messageId, false);
if (meta == null) {
return;
}
- String commandId = meta[0];
- String instanceId = meta[1];
- String region = meta[2];
+ String commandId = meta.commandId();
+ String instanceId = meta.owner().instanceId();
+ String region = meta.owner().region();
String invKey = invocationKey(region, commandId, instanceId);
invocationStore.get(invKey).ifPresent(inv -> {
@@ -413,14 +414,14 @@ public class SsmCommandService implements Resettable {
}
public void sendReply(String messageId, String payloadBase64) {
- String[] meta = messageIndex.remove(messageId);
+ MessageCorrelation meta = ownedCorrelation(messageId, true);
if (meta == null) {
LOG.warnv("SendReply: unknown messageId={0}", messageId);
return;
}
- String commandId = meta[0];
- String instanceId = meta[1];
- String region = meta[2];
+ String commandId = meta.commandId();
+ String instanceId = meta.owner().instanceId();
+ String region = meta.owner().region();
try {
byte[] decoded = Base64.getDecoder().decode(payloadBase64);
@@ -477,13 +478,13 @@ public class SsmCommandService implements Resettable {
}
public void failMessage(String messageId, String failureType) {
- String[] meta = messageIndex.remove(messageId);
+ MessageCorrelation meta = ownedCorrelation(messageId, true);
if (meta == null) {
return;
}
- String commandId = meta[0];
- String instanceId = meta[1];
- String region = meta[2];
+ String commandId = meta.commandId();
+ String instanceId = meta.owner().instanceId();
+ String region = meta.owner().region();
String invKey = invocationKey(region, commandId, instanceId);
invocationStore.get(invKey).ifPresent(inv -> {
@@ -499,7 +500,20 @@ public class SsmCommandService implements Resettable {
}
public void deleteMessage(String messageId) {
- messageIndex.remove(messageId);
+ ownedCorrelation(messageId, true);
+ }
+
+ private MessageOwner messageOwner(String region, String instanceId) {
+ return new MessageOwner(regionResolver.getAccountId(), region, instanceId);
+ }
+
+ private MessageCorrelation ownedCorrelation(String messageId, boolean remove) {
+ MessageCorrelation correlation = messageIndex.get(messageId);
+ if (correlation == null || !correlation.owner().equals(
+ messageOwner(regionResolver.getRegion(), correlation.owner().instanceId()))) {
+ return null;
+ }
+ return !remove || messageIndex.remove(messageId, correlation) ? correlation : null;
}
// ── CodeDeploy integration helpers ─────────────────────────────────────
@@ -560,7 +574,7 @@ public class SsmCommandService implements Resettable {
Map<String, List<String>> parameters,
int timeoutSeconds,
String region) {
- CompletableFuture.runAsync(() -> {
+ CompletableFuture.runAsync(() -> RequestScopes.runAs(accountId, region, () -> {
String invKey = invocationKey(region, commandId, instanceId);
CommandInvocation invocation = invocationStore.get(invKey).orElse(null);
if (invocation == null || "Cancelled".equals(invocation.getStatus())) {
@@ -591,7 +605,7 @@ public class SsmCommandService implements Resettable {
invocationStore.put(invKey, invocation);
}
updateCommandStatus(commandId, region);
- }, directExecutionExecutor);
+ }), directExecutionExecutor);
}
private void applyDirectResult(CommandInvocation invocation, SsmDirectCommandExecutor.ExecutionResult result) {
@@ -619,7 +633,7 @@ public class SsmCommandService implements Resettable {
String messageId = UUID.randomUUID().toString();
String payload = buildCommandPayload(commandId, documentName, documentVersion(parameters), parameters, timeoutSeconds, region);
PendingMessage msg = new PendingMessage(messageId, commandId, region, Instant.now(), payload);
- messageQueues.computeIfAbsent(instanceId, k -> new ConcurrentLinkedQueue<>()).add(msg);
+ messageQueues.computeIfAbsent(messageOwner(region, instanceId), k -> new ConcurrentLinkedQueue<>()).add(msg);
}
private String buildCommandPayload(String commandId, String documentName, String docVersion,
@@ -846,5 +860,8 @@ public class SsmCommandService implements Resettable {
return region + "::" + commandId + "::" + instanceId;
}
+ private record MessageOwner(String accountId, String region, String instanceId) {}
+ private record MessageCorrelation(MessageOwner owner, String commandId) {}
+
record PendingMessage(String messageId, String commandId, String region, Instant createdDate, String payload) {}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java
index 305b33ed80..95aad47555 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java
@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.ssm.model.Command;
import io.github.hectorvent.floci.services.ssm.model.CommandInvocation;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Duration;
@@ -44,6 +45,12 @@ class SsmCommandServiceDirectExecutionTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private final RegionResolver regionResolver = mock(RegionResolver.class);
+ @BeforeEach
+ void requestRegion() {
+ when(regionResolver.getRegion()).thenReturn("us-west-2");
+ }
+
+
@Test
void agentRegistrationChecksEc2BackingOnFirstRegistrationOnly() throws Exception {
SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandAccountIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandAccountIntegrationTest.java
new file mode 100644
index 0000000000..75a65de285
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandAccountIntegrationTest.java
@@ -0,0 +1,395 @@
+package io.github.hectorvent.floci.services.ssm;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsRegions;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ssm.model.Command;
+import io.github.hectorvent.floci.services.ssm.model.CommandInvocation;
+import io.github.hectorvent.floci.testing.EnforcementFixtures;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.InjectMock;
+import io.quarkus.test.common.http.TestHTTPResource;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+@QuarkusTest
+class SsmDirectCommandAccountIntegrationTest {
+ @Inject StorageFactory storageFactory;
+ @Inject RegionResolver regionResolver;
+ @InjectMock SsmDirectCommandExecutor executor;
+ @Inject SsmCommandService commandService;
+ @TestHTTPResource URI endpoint;
+
+ @BeforeEach
+ void setup() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ reset(executor);
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "000000000000,us-east-1,false", "000000000000,us-east-1,true",
+ "222222222222,us-west-2,false", "222222222222,us-west-2,true",
+ "333333333333,cn-north-1,false", "333333333333,cn-north-1,true"
+ })
+ void directWorkerRetainsRequestOwnerForResultAndFallback(String account, String region, boolean fallback)
+ throws Exception {
+ String instance = "i-scope-" + UUID.randomUUID();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicReference<Thread> worker = new AtomicReference<>();
+ AtomicReference<String> workerAccount = new AtomicReference<>();
+ AtomicReference<String> workerRegion = new AtomicReference<>();
+ AtomicReference<String> workerPartition = new AtomicReference<>();
+ when(executor.supports(account, instance, "AWS-RunShellScript")).thenReturn(true);
+ when(executor.executeIfSupported(eq(account), eq(instance), eq("AWS-RunShellScript"), any(), eq(60)))
+ .thenAnswer(invocation -> {
+ worker.set(Thread.currentThread());
+ workerAccount.set(regionResolver.getAccountId());
+ workerRegion.set(regionResolver.getRegion());
+ workerPartition.set(regionResolver.getPartition());
+ entered.countDown();
+ assertTrue(release.await(10, TimeUnit.SECONDS));
+ return fallback ? Optional.empty() : Optional.of(new SsmDirectCommandExecutor.ExecutionResult(
+ "Success", "owned output", "owned stderr", 0, Instant.now(), Instant.now()));
+ });
+ AccountAwareStorageBackend<Command> commands = storageFactory.create(
+ "ssm", "ssm-commands.json", new TypeReference<>() {});
+ AccountAwareStorageBackend<CommandInvocation> invocations = storageFactory.create(
+ "ssm", "ssm-invocations.json", new TypeReference<>() {});
+ String commandId = call(account, region, "AmazonSSM.SendCommand", Map.of(
+ "DocumentName", "AWS-RunShellScript", "InstanceIds", List.of(instance),
+ "Parameters", Map.of("commands", List.of("echo owned")), "TimeoutSeconds", 60))
+ .jsonPath().getString("Command.CommandId");
+ assertNotNull(commandId);
+ String commandKey = region + "::" + commandId;
+ String invocationKey = commandKey + "::" + instance;
+ try {
+ assertTrue(commands.getForAccount(account, commandKey).isPresent());
+ assertTrue(invocations.getForAccount(account, invocationKey).isPresent());
+ assertEquals("InProgress", commands.getForAccount(account, commandKey).orElseThrow().getStatus());
+ assertTrue(entered.await(3, TimeUnit.SECONDS), "Worker did not resolve its issuing account's invocation");
+ assertEquals(account, workerAccount.get());
+ assertEquals(region, workerRegion.get());
+ assertEquals(AwsRegions.partitionFor(region), workerPartition.get());
+ release.countDown();
+ worker.get().join(5000);
+ assertFalse(worker.get().isAlive(), "Direct worker did not finish");
+ Response result = call(account, region, "AmazonSSM.GetCommandInvocation", Map.of(
+ "CommandId", commandId, "InstanceId", instance));
+ assertEquals(fallback ? "Pending" : "Success", result.jsonPath().getString("Status"));
+ if (fallback) {
+ Response poll = call(account, region, "AmazonSSMMessageDeliveryService.GetMessages",
+ Map.of("Destination", instance, "MessagesRequestId", "owned-request"));
+ assertEquals(1, poll.jsonPath().getList("Messages").size());
+ assertEquals("aws.ssm.sendCommand." + region, poll.jsonPath().getString("Messages[0].Topic"));
+ } else {
+ assertEquals("owned output", result.jsonPath().getString("StandardOutputContent"));
+ assertEquals("owned stderr", result.jsonPath().getString("StandardErrorContent"));
+ assertEquals("Success", commands.getForAccount(account, commandKey).orElseThrow().getStatus());
+ assertEquals(1, commands.getForAccount(account, commandKey).orElseThrow().getCompletedCount());
+ }
+ for (String other : List.of("000000000000", "222222222222", "333333333333")) {
+ if (!other.equals(account)) {
+ assertTrue(commands.getForAccount(other, commandKey).isEmpty());
+ assertTrue(invocations.getForAccount(other, invocationKey).isEmpty());
+ }
+ }
+ } finally {
+ release.countDown();
+ assertAll("release worker and remove every owned fixture",
+ () -> joinWorker(worker),
+ () -> deleteCommand(account, region, commandId, instance));
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"333333333333,us-west-2", "222222222222,cn-north-1"})
+ void blockedOwnerDoesNotDelayOtherScopeAndCancellationSurvivesCompletion(String otherAccount, String otherRegion)
+ throws Exception {
+ String account = "222222222222";
+ String region = "us-west-2";
+ String instance = "i-scope-" + UUID.randomUUID();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch otherEntered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicReference<Thread> blockedWorker = new AtomicReference<>();
+ AtomicReference<Thread> otherWorker = new AtomicReference<>();
+ when(executor.supports(anyString(), eq(instance), eq("AWS-RunShellScript"))).thenReturn(true);
+ when(executor.executeIfSupported(anyString(), eq(instance), eq("AWS-RunShellScript"), any(), eq(60)))
+ .thenAnswer(invocation -> {
+ assertEquals(invocation.getArgument(0), regionResolver.getAccountId());
+ if (account.equals(regionResolver.getAccountId()) && region.equals(regionResolver.getRegion())) {
+ blockedWorker.set(Thread.currentThread());
+ entered.countDown();
+ assertTrue(release.await(10, TimeUnit.SECONDS));
+ } else {
+ assertEquals(otherAccount, regionResolver.getAccountId());
+ assertEquals(otherRegion, regionResolver.getRegion());
+ otherWorker.set(Thread.currentThread());
+ otherEntered.countDown();
+ }
+ return Optional.of(new SsmDirectCommandExecutor.ExecutionResult(
+ "Success", "scope output", "", 0, Instant.now(), Instant.now()));
+ });
+ Map<String, Object> request = Map.of("DocumentName", "AWS-RunShellScript", "InstanceIds", List.of(instance),
+ "Parameters", Map.of("commands", List.of("echo scope")), "TimeoutSeconds", 60);
+ String firstId = call(account, region, "AmazonSSM.SendCommand", request)
+ .jsonPath().getString("Command.CommandId");
+ String secondId = null;
+ try {
+ assertOwnedInvocation(account, region, firstId, instance);
+ assertTrue(entered.await(3, TimeUnit.SECONDS), "Owner worker did not resolve its stored invocation");
+ secondId = call(otherAccount, otherRegion, "AmazonSSM.SendCommand", request)
+ .jsonPath().getString("Command.CommandId");
+ assertOwnedInvocation(otherAccount, otherRegion, secondId, instance);
+ assertTrue(otherEntered.await(3, TimeUnit.SECONDS), "Unrelated scope was blocked by direct execution");
+ otherWorker.get().join(3000);
+ assertFalse(otherWorker.get().isAlive());
+ assertEquals("Success", call(otherAccount, otherRegion, "AmazonSSM.GetCommandInvocation",
+ Map.of("CommandId", secondId, "InstanceId", instance)).jsonPath().getString("Status"));
+ call(account, region, "AmazonSSM.CancelCommand", Map.of("CommandId", firstId));
+ release.countDown();
+ blockedWorker.get().join(3000);
+ assertFalse(blockedWorker.get().isAlive());
+ assertEquals("Cancelled", call(account, region, "AmazonSSM.GetCommandInvocation",
+ Map.of("CommandId", firstId, "InstanceId", instance)).jsonPath().getString("Status"));
+ assertTrue(call(account, region, "AmazonSSMMessageDeliveryService.GetMessages",
+ Map.of("Destination", instance)).jsonPath().getList("Messages").isEmpty());
+ } finally {
+ release.countDown();
+ String otherId = secondId;
+ assertAll("release both workers and remove every owned fixture",
+ () -> joinWorker(blockedWorker),
+ () -> joinWorker(otherWorker),
+ () -> deleteCommand(account, region, firstId, instance),
+ () -> {
+ if (otherId != null) {
+ deleteCommand(otherAccount, otherRegion, otherId, instance);
+ }
+ });
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"333333333333,us-west-2", "222222222222,cn-north-1"})
+ void foreignScopeCannotPollDirectFallback(String foreignAccount, String foreignRegion) throws Exception {
+ String instance = "i-fallback-" + UUID.randomUUID();
+ String command = directFallback(instance);
+ try {
+ assertTrue(call(foreignAccount, foreignRegion, "AmazonSSMMessageDeliveryService.GetMessages",
+ Map.of("Destination", instance)).jsonPath().getList("Messages").isEmpty());
+ Response own = call("222222222222", "us-west-2", "AmazonSSMMessageDeliveryService.GetMessages",
+ Map.of("Destination", instance));
+ assertEquals(1, own.jsonPath().getList("Messages").size());
+ assertEquals(command, new ObjectMapper().readTree(Base64.getDecoder()
+ .decode(own.jsonPath().getString("Messages[0].Payload"))).path("CommandId").asText());
+ } finally {
+ deleteCommand("222222222222", "us-west-2", command, instance);
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"333333333333,us-west-2,AcknowledgeMessage", "333333333333,us-west-2,SendReply",
+ "333333333333,us-west-2,FailMessage", "333333333333,us-west-2,DeleteMessage",
+ "222222222222,cn-north-1,AcknowledgeMessage", "222222222222,cn-north-1,SendReply",
+ "222222222222,cn-north-1,FailMessage", "222222222222,cn-north-1,DeleteMessage"})
+ void foreignScopeCannotConsumeFallbackCorrelation(String foreignAccount, String foreignRegion, String operation)
+ throws Exception {
+ String instance = "i-fallback-" + UUID.randomUUID();
+ String command = directFallback(instance);
+ try {
+ String message = call("222222222222", "us-west-2", "AmazonSSMMessageDeliveryService.GetMessages",
+ Map.of("Destination", instance)).jsonPath().getString("Messages[0].MessageId");
+ assertNotNull(message);
+ Map<String, Object> body = Map.of("MessageId", message, "Payload", replyPayload(),
+ "FailureType", "foreign-failure");
+ call(foreignAccount, foreignRegion, "AmazonSSMMessageDeliveryService." + operation, body);
+ assertEquals("Pending", call("222222222222", "us-west-2", "AmazonSSM.GetCommandInvocation",
+ Map.of("CommandId", command, "InstanceId", instance)).jsonPath().getString("Status"));
+ call("222222222222", "us-west-2", "AmazonSSMMessageDeliveryService.AcknowledgeMessage", body);
+ assertEquals("InProgress", call("222222222222", "us-west-2", "AmazonSSM.GetCommandInvocation",
+ Map.of("CommandId", command, "InstanceId", instance)).jsonPath().getString("Status"));
+ call("222222222222", "us-west-2", "AmazonSSMMessageDeliveryService.SendReply", body);
+ Response result = call("222222222222", "us-west-2", "AmazonSSM.GetCommandInvocation",
+ Map.of("CommandId", command, "InstanceId", instance));
+ assertEquals("Success", result.jsonPath().getString("Status"));
+ assertEquals("owned agent output", result.jsonPath().getString("StandardOutputContent"));
+ } finally {
+ deleteCommand("222222222222", "us-west-2", command, instance);
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"333333333333,us-west-2,false,false", "333333333333,us-west-2,false,true",
+ "333333333333,us-west-2,true,false", "333333333333,us-west-2,true,true",
+ "222222222222,cn-north-1,false,false", "222222222222,cn-north-1,false,true",
+ "222222222222,cn-north-1,true,false", "222222222222,cn-north-1,true,true"})
+ void peerCleanupPreservesOwnerQueueAndCorrelation(String peerAccount, String peerRegion,
+ boolean polled, boolean cancel) throws Exception {
+ String instance = "i-fallback-" + UUID.randomUUID();
+ String command = directFallback(instance);
+ String peerCommand = null;
+ try {
+ peerCommand = directFallback(peerAccount, peerRegion, instance);
+ String message = polled ? call("222222222222", "us-west-2",
+ "AmazonSSMMessageDeliveryService.GetMessages", Map.of("Destination", instance))
+ .jsonPath().getString("Messages[0].MessageId") : null;
+ if (polled) {
+ assertNotNull(message);
+ }
+ if (cancel) {
+ call(peerAccount, peerRegion, "AmazonSSM.CancelCommand", Map.of("CommandId", peerCommand));
+ } else {
+ RequestScopes.runAs(peerAccount, peerRegion, () -> assertEquals(1,
+ commandService.failActiveInvocationsForInstance(peerRegion, instance, "peer stopped")));
+ }
+ assertTrue(call(peerAccount, peerRegion, "AmazonSSMMessageDeliveryService.GetMessages",
+ Map.of("Destination", instance)).jsonPath().getList("Messages").isEmpty());
+ if (!polled) {
+ message = call("222222222222", "us-west-2", "AmazonSSMMessageDeliveryService.GetMessages",
+ Map.of("Destination", instance)).jsonPath().getString("Messages[0].MessageId");
+ assertNotNull(message);
+ }
+ call("222222222222", "us-west-2", "AmazonSSMMessageDeliveryService.SendReply",
+ Map.of("MessageId", message, "Payload", replyPayload()));
+ assertEquals("Success", call("222222222222", "us-west-2", "AmazonSSM.GetCommandInvocation",
+ Map.of("CommandId", command, "InstanceId", instance)).jsonPath().getString("Status"));
+ } finally {
+ String peer = peerCommand;
+ assertAll(() -> deleteCommand("222222222222", "us-west-2", command, instance),
+ () -> { if (peer != null) { deleteCommand(peerAccount, peerRegion, peer, instance); } });
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"FailMessage,Failed", "DeleteMessage,Pending"})
+ void ownerCanFailOrDeleteFallbackCorrelation(String operation, String expected) throws Exception {
+ String instance = "i-fallback-" + UUID.randomUUID();
+ String command = directFallback(instance);
+ try {
+ String message = call("222222222222", "us-west-2", "AmazonSSMMessageDeliveryService.GetMessages",
+ Map.of("Destination", instance)).jsonPath().getString("Messages[0].MessageId");
+ assertNotNull(message);
+ Map<String, Object> body = Map.of("MessageId", message, "FailureType", "owner failure",
+ "Payload", replyPayload());
+ call("222222222222", "us-west-2", "AmazonSSMMessageDeliveryService." + operation, body);
+ call("222222222222", "us-west-2", "AmazonSSMMessageDeliveryService.AcknowledgeMessage", body);
+ call("222222222222", "us-west-2", "AmazonSSMMessageDeliveryService.SendReply", body);
+ Response result = call("222222222222", "us-west-2", "AmazonSSM.GetCommandInvocation",
+ Map.of("CommandId", command, "InstanceId", instance));
+ assertEquals(expected, result.jsonPath().getString("Status"));
+ if (operation.equals("FailMessage")) {
+ assertEquals("Failed: owner failure", result.jsonPath().getString("StatusDetails"));
+ }
+ } finally {
+ deleteCommand("222222222222", "us-west-2", command, instance);
+ }
+ }
+
+ private String directFallback(String instance) throws Exception {
+ return directFallback("222222222222", "us-west-2", instance);
+ }
+
+ private String directFallback(String account, String region, String instance) throws Exception {
+ CountDownLatch entered = new CountDownLatch(1);
+ AtomicReference<Thread> worker = new AtomicReference<>();
+ when(executor.supports(account, instance, "AWS-RunShellScript")).thenReturn(true);
+ when(executor.executeIfSupported(eq(account), eq(instance), eq("AWS-RunShellScript"), any(), eq(60)))
+ .thenAnswer(invocation -> {
+ worker.set(Thread.currentThread());
+ entered.countDown();
+ return Optional.empty();
+ });
+ String command = call(account, region, "AmazonSSM.SendCommand", Map.of("DocumentName", "AWS-RunShellScript",
+ "InstanceIds", List.of(instance), "Parameters", Map.of("commands", List.of("echo owned")),
+ "TimeoutSeconds", 60)).jsonPath().getString("Command.CommandId");
+ try {
+ assertNotNull(command);
+ assertOwnedInvocation(account, region, command, instance);
+ assertTrue(entered.await(3, TimeUnit.SECONDS));
+ joinWorker(worker);
+ assertEquals("Pending", call(account, region, "AmazonSSM.GetCommandInvocation",
+ Map.of("CommandId", command, "InstanceId", instance)).jsonPath().getString("Status"));
+ return command;
+ } catch (Throwable failure) {
+ assertAll(() -> { throw failure; }, () -> joinWorker(worker),
+ () -> deleteCommand(account, region, command, instance));
+ throw new AssertionError("unreachable");
+ }
+ }
+
+ private static String replyPayload() {
+ return Base64.getEncoder().encodeToString(("{\"runtimeStatus\":{\"aws:runShellScript\":{"
+ + "\"status\":\"Success\",\"code\":0,\"output\":\"owned agent output\"}}}")
+ .getBytes(StandardCharsets.UTF_8));
+ }
+
+ private void assertOwnedInvocation(String account, String region, String commandId, String instance) {
+ AccountAwareStorageBackend<Command> commands = storageFactory.create(
+ "ssm", "ssm-commands.json", new TypeReference<>() {});
+ AccountAwareStorageBackend<CommandInvocation> invocations = storageFactory.create(
+ "ssm", "ssm-invocations.json", new TypeReference<>() {});
+ String key = region + "::" + commandId;
+ assertTrue(commands.getForAccount(account, key).isPresent());
+ assertTrue(invocations.getForAccount(account, key + "::" + instance).isPresent());
+ }
+
+ private void deleteCommand(String account, String region, String commandId, String instance) {
+ AccountAwareStorageBackend<Command> commands = storageFactory.create(
+ "ssm", "ssm-commands.json", new TypeReference<>() {});
+ AccountAwareStorageBackend<CommandInvocation> invocations = storageFactory.create(
+ "ssm", "ssm-invocations.json", new TypeReference<>() {});
+ String key = region + "::" + commandId;
+ assertAll("remove owned queue, command and invocation",
+ () -> RequestScopes.runAs(account, region, () ->
+ commandService.failActiveInvocationsForInstance(region, instance, "fixture cleanup")),
+ () -> {
+ commands.deleteForAccount(account, key);
+ assertTrue(commands.getForAccount(account, key).isEmpty());
+ },
+ () -> {
+ invocations.deleteForAccount(account, key + "::" + instance);
+ assertTrue(invocations.getForAccount(account, key + "::" + instance).isEmpty());
+ });
+ }
+
+ private void joinWorker(AtomicReference<Thread> worker) throws InterruptedException {
+ if (worker.get() != null) {
+ worker.get().join(5000);
+ assertFalse(worker.get().isAlive(), "Owned direct worker did not finish");
+ }
+ }
+
+ private Response call(String account, String region, String action, Map<String, Object> body) {
+ return given().port(endpoint.getPort()).contentType("application/x-amz-json-1.1")
+ .header("Authorization", EnforcementFixtures.auth(account, region, "ssm"))
+ .header("X-Amz-Target", action).body(body).post("/")
+ .then().statusCode(200).extract().response();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/testing/EnforcementFixtures.java b/src/test/java/io/github/hectorvent/floci/testing/EnforcementFixtures.java
index b818fbb3fb..a1300cb361 100644
--- a/src/test/java/io/github/hectorvent/floci/testing/EnforcementFixtures.java
+++ b/src/test/java/io/github/hectorvent/floci/testing/EnforcementFixtures.java
@@ -111,7 +111,17 @@ public final class EnforcementFixtures {
* the request has to be signed for real instead.
*/
public static String auth(String accessKeyId, String service) {
- return "AWS4-HMAC-SHA256 Credential=" + accessKeyId + "/20260629/us-east-1/" + service
+ return auth(accessKeyId, "us-east-1", service);
+ }
+
+ /** A placeholder signature scoped to the caller's selected region and service. */
+ public static String auth(String accessKeyId, String region, String service) {
+ return auth(accessKeyId, "20260629", region, service);
+ }
+
+ /** A placeholder signature retaining the date selected by an existing request fixture. */
+ public static String auth(String accessKeyId, String date, String region, String service) {
+ return "AWS4-HMAC-SHA256 Credential=" + accessKeyId + "/" + date + "/" + region + "/" + service
+ "/aws4_request, SignedHeaders=host, Signature=abc";
}
diff --git a/src/test/java/io/github/hectorvent/floci/testing/LogsAuthorizationFixtures.java b/src/test/java/io/github/hectorvent/floci/testing/LogsAuthorizationFixtures.java
index 14d074fe45..330103ea23 100644
--- a/src/test/java/io/github/hectorvent/floci/testing/LogsAuthorizationFixtures.java
+++ b/src/test/java/io/github/hectorvent/floci/testing/LogsAuthorizationFixtures.java
@@ -31,8 +31,7 @@ public final class LogsAuthorizationFixtures {
}
public static Response call(int port, String caller, String date, String region, String action, Map<String, ?> body) {
- String authorization = "AWS4-HMAC-SHA256 Credential=" + caller + "/" + date + "/" + region
- + "/logs/aws4_request, SignedHeaders=host, Signature=abc";
+ String authorization = EnforcementFixtures.auth(caller, date, region, "logs");
return given().port(port).contentType("application/x-amz-json-1.1")
.header("Authorization", authorization).header("X-Amz-Target", "Logs_20140328." + action)
.body(new LinkedHashMap<>(body)).when().post("/");