Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ssm): stop re-reading shared mutable state in the command status rollup (#2477)

Olamide Kosile6f1ce9b23f68
diff --git a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
index 551a046d87..d7a3dde254 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
@@ -264,10 +264,16 @@ public class SsmCommandService implements Resettable {
for (String instanceId : targets) {
String invKey = invocationKey(region, commandId, instanceId);
invocationStore.get(invKey).ifPresent(inv -> {
- if ("Pending".equals(inv.getStatus()) || "InProgress".equals(inv.getStatus())) {
- inv.setStatus("Cancelled");
- inv.setStatusDetails("Cancelled");
- invocationStore.put(invKey, inv);
+ // Same lock discipline as the command-level fields: invocationStore returns the same
+ // shared CommandInvocation object on every get(), and multiple methods here (direct
+ // completion, agent SendReply/FailMessage, timeout sweeps) can race this cancellation
+ // on the same instance's invocation.
+ synchronized (inv) {
+ if ("Pending".equals(inv.getStatus()) || "InProgress".equals(inv.getStatus())) {
+ inv.setStatus("Cancelled");
+ inv.setStatusDetails("Cancelled");
+ invocationStore.put(invKey, inv);
+ }
}
// Remove any queued (not-yet-polled) messages for this instance
Queue<PendingMessage> q = messageQueues.get(instanceId);
@@ -277,9 +283,13 @@ public class SsmCommandService implements Resettable {
});
}
- command.setStatus("Cancelled");
- command.setStatusDetails("Cancelled");
- commandStore.put(commandKey(region, commandId), command);
+ // Same lock updateCommandStatus takes on this exact object, so a rollup completing this
+ // command concurrently can't have its own status+statusDetails write interleave with this one.
+ synchronized (command) {
+ command.setStatus("Cancelled");
+ command.setStatusDetails("Cancelled");
+ commandStore.put(commandKey(region, commandId), command);
+ }
LOG.infov("CancelCommand: commandId={0}", commandId);
}
@@ -303,11 +313,13 @@ public class SsmCommandService implements Resettable {
if (!isActiveInvocation(invocation.getStatus())) {
continue;
}
- invocation.setStatus("Failed");
- invocation.setStatusDetails(statusDetails);
- invocation.setResponseCode(-1);
- invocation.setExecutionEndDateTime(now);
- invocationStore.put(invocationKey(region, invocation.getCommandId(), invocation.getInstanceId()), invocation);
+ synchronized (invocation) {
+ invocation.setStatus("Failed");
+ invocation.setStatusDetails(statusDetails);
+ invocation.setResponseCode(-1);
+ invocation.setExecutionEndDateTime(now);
+ invocationStore.put(invocationKey(region, invocation.getCommandId(), invocation.getInstanceId()), invocation);
+ }
commandIds.add(invocation.getCommandId());
failed++;
}
@@ -369,11 +381,13 @@ public class SsmCommandService implements Resettable {
String invKey = invocationKey(region, commandId, instanceId);
invocationStore.get(invKey).ifPresent(inv -> {
- if ("Pending".equals(inv.getStatus())) {
- inv.setStatus("InProgress");
- inv.setStatusDetails(statusDetails("InProgress"));
- inv.setExecutionStartDateTime(Instant.now());
- invocationStore.put(invKey, inv);
+ synchronized (inv) {
+ if ("Pending".equals(inv.getStatus())) {
+ inv.setStatus("InProgress");
+ inv.setStatusDetails(statusDetails("InProgress"));
+ inv.setExecutionStartDateTime(Instant.now());
+ invocationStore.put(invKey, inv);
+ }
}
});
LOG.debugv("AcknowledgeMessage: messageId={0} commandId={1}", messageId, commandId);
@@ -424,13 +438,15 @@ public class SsmCommandService implements Resettable {
String invKey = invocationKey(region, commandId, instanceId);
CommandInvocation inv = invocationStore.get(invKey).orElse(null);
if (inv != null) {
- inv.setStatus(toInvocationStatus(status));
- inv.setStatusDetails(statusDetails(toInvocationStatus(status)));
- inv.setStandardOutputContent(stdout);
- inv.setStandardErrorContent(stderr);
- inv.setResponseCode(returnCode);
- inv.setExecutionEndDateTime(endTime);
- invocationStore.put(invKey, inv);
+ synchronized (inv) {
+ inv.setStatus(toInvocationStatus(status));
+ inv.setStatusDetails(statusDetails(toInvocationStatus(status)));
+ inv.setStandardOutputContent(stdout);
+ inv.setStandardErrorContent(stderr);
+ inv.setResponseCode(returnCode);
+ inv.setExecutionEndDateTime(endTime);
+ invocationStore.put(invKey, inv);
+ }
}
// Recalculate command status
@@ -452,10 +468,12 @@ public class SsmCommandService implements Resettable {
String invKey = invocationKey(region, commandId, instanceId);
invocationStore.get(invKey).ifPresent(inv -> {
- inv.setStatus("Failed");
- inv.setStatusDetails("Failed: " + failureType);
- inv.setExecutionEndDateTime(Instant.now());
- invocationStore.put(invKey, inv);
+ synchronized (inv) {
+ inv.setStatus("Failed");
+ inv.setStatusDetails("Failed: " + failureType);
+ inv.setExecutionEndDateTime(Instant.now());
+ invocationStore.put(invKey, inv);
+ }
});
updateCommandStatus(commandId, region);
LOG.warnv("FailMessage: commandId={0} instanceId={1} failureType={2}", commandId, instanceId, failureType);
@@ -533,15 +551,19 @@ public class SsmCommandService implements Resettable {
.executeIfSupported(instanceId, documentName, parameters, timeoutSeconds)
.orElse(null);
if (result == null) {
- invocation.setStatus("Pending");
- invocation.setStatusDetails(statusDetails("Pending"));
- invocationStore.put(invKey, invocation);
+ synchronized (invocation) {
+ invocation.setStatus("Pending");
+ invocation.setStatusDetails(statusDetails("Pending"));
+ invocationStore.put(invKey, invocation);
+ }
queueMessage(commandId, instanceId, documentName, parameters, timeoutSeconds, region);
updateCommandStatus(commandId, region);
return;
}
- applyDirectResult(invocation, result);
- invocationStore.put(invKey, invocation);
+ synchronized (invocation) {
+ applyDirectResult(invocation, result);
+ invocationStore.put(invKey, invocation);
+ }
updateCommandStatus(commandId, region);
}, directExecutionExecutor);
}
@@ -683,16 +705,30 @@ public class SsmCommandService implements Resettable {
}
}
- command.setCompletedCount(completed);
- command.setErrorCount(errors);
+ // Synchronized on the shared Command object itself (commandStore's get() always returns the
+ // same instance for a given key, not a defensive copy) so this status+statusDetails write
+ // can't interleave with cancelCommand's own status+statusDetails write on the same object -
+ // without this, cancelCommand's whole pair could land inside the gap between this method's
+ // two field writes, leaving status=Cancelled but statusDetails reflecting this call's stale
+ // completion status (or the reverse), even though each method's own two fields individually
+ // agree with each other when read right after being written.
+ synchronized (command) {
+ command.setCompletedCount(completed);
+ command.setErrorCount(errors);
+
+ if (!anyInProgress && completed == instanceIds.size()) {
+ // Use the local status, not a re-read of command.getStatus(): re-reading it here left
+ // a second, narrower window where a concurrent updateCommandStatus call for the same
+ // command (e.g. another instance's async completion racing this one) could mutate the
+ // shared object in between, so this call's own statusDetails ended up reflecting a
+ // DIFFERENT call's status than the one it had just written to status itself.
+ String status = commandStatus(errors, timedOut, instanceIds.size());
+ command.setStatus(status);
+ command.setStatusDetails(statusDetails(status));
+ }
- if (!anyInProgress && completed == instanceIds.size()) {
- String status = commandStatus(errors, timedOut, instanceIds.size());
- command.setStatus(status);
- command.setStatusDetails(statusDetails(command.getStatus()));
+ commandStore.put(commandKey(region, commandId), command);
}
-
- commandStore.put(commandKey(region, commandId), command);
}
private static String commandStatus(int errors, int timedOut, int targetCount) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmJsonHandler.java
index cdd6a348b1..bd2057c70e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmJsonHandler.java
@@ -423,8 +423,15 @@ public class SsmJsonHandler {
if (c.getComment() != null) node.put("Comment", c.getComment());
if (c.getRequestedDateTime() != null) node.put("RequestedDateTime", c.getRequestedDateTime().toEpochMilli() / 1000.0);
if (c.getExpiresAfter() != null) node.put("ExpiresAfter", c.getExpiresAfter().toEpochMilli() / 1000.0);
- node.put("Status", c.getStatus());
- node.put("StatusDetails", c.getStatusDetails());
+ // c is the same live, shared Command object SsmCommandService's writers (updateCommandStatus,
+ // cancelCommand) synchronize on before touching Status/StatusDetails together. Reading the
+ // pair here without the same lock would let this read straddle a concurrent writer's update -
+ // one field read before it, the other after - producing a Status/StatusDetails combination
+ // that was never actually true at any single instant.
+ synchronized (c) {
+ node.put("Status", c.getStatus());
+ node.put("StatusDetails", c.getStatusDetails());
+ }
node.put("TargetCount", c.getTargetCount());
node.put("CompletedCount", c.getCompletedCount());
node.put("ErrorCount", c.getErrorCount());
@@ -454,8 +461,12 @@ public class SsmJsonHandler {
node.put("DocumentName", inv.getDocumentName());
if (inv.getDocumentVersion() != null) node.put("DocumentVersion", inv.getDocumentVersion());
if (inv.getRequestedDateTime() != null) node.put("RequestedDateTime", inv.getRequestedDateTime().toEpochMilli() / 1000.0);
- node.put("Status", inv.getStatus());
- node.put("StatusDetails", inv.getStatusDetails());
+ // Same reasoning as commandToNode: inv is the same live, shared CommandInvocation object its
+ // writers (cancelCommand, direct/agent completion, timeout sweeps) now synchronize on.
+ synchronized (inv) {
+ node.put("Status", inv.getStatus());
+ node.put("StatusDetails", inv.getStatusDetails());
+ }
return node;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java
index d9a6be623a..b2da0267f9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmCommandServiceDirectExecutionTest.java
@@ -2,6 +2,8 @@ package io.github.hectorvent.floci.services.ssm;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.core.storage.StorageBackend;
@@ -12,17 +14,24 @@ import io.github.hectorvent.floci.services.ssm.model.CommandInvocation;
import org.junit.jupiter.api.Test;
import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
class SsmCommandServiceDirectExecutionTest {
@@ -203,6 +212,221 @@ class SsmCommandServiceDirectExecutionTest {
assertEquals("Timed out after 30s", invocation.getStandardErrorContent());
}
+ @Test
+ void cancelAndConcurrentRollup_areMutuallyExclusiveForTheSameCommand() throws Exception {
+ // Regression: even with the self-consistency fix above, cancelCommand's own status +
+ // statusDetails write could still land entirely inside updateCommandStatus's window between
+ // ITS two field writes - leaving status=Cancelled but statusDetails reflecting the rollup's
+ // stale completion status, or the reverse. Both methods now synchronize on the shared Command
+ // object (commandStore's get() always returns the same instance for a key) before touching
+ // its status fields, so the two calls can't interleave with each other at all, closing this
+ // regardless of which field either one happens to write first.
+ //
+ // Proving this directly rather than by timing: commandStore.put() is called from inside the
+ // synchronized block in both methods and is a real interception point (unlike the two setter
+ // calls themselves), so a spy can force updateCommandStatus's finalize write to block while
+ // it still holds the lock, and assert a concurrent cancelCommand call blocks too rather than
+ // racing past it - the same style already used to prove the code-volume lock in
+ // ContainerLauncherTest's cleanupAndAConcurrentRollback... test.
+ SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);
+ Instant start = Instant.parse("2026-06-07T00:00:00Z");
+ Instant end = Instant.parse("2026-06-07T00:00:01Z");
+ when(executor.supports(eq("i-race"), eq("AWS-RunShellScript"))).thenReturn(true);
+ when(executor.executeIfSupported(eq("i-race"), eq("AWS-RunShellScript"), any(), eq(60)))
+ .thenReturn(Optional.of(new SsmDirectCommandExecutor.ExecutionResult(
+ "Success", "done\n", "", 0, start, end)));
+
+ CountDownLatch rollupHoldingLock = new CountDownLatch(1);
+ CountDownLatch releaseRollup = new CountDownLatch(1);
+ AccountAwareStorageBackend<Command> realCommandStore = AccountAwareStorageBackend.inMemory("000000000000");
+ AccountAwareStorageBackend<Command> commandStore = spy(realCommandStore);
+ doAnswer(invocation -> {
+ Command value = invocation.getArgument(1);
+ // Only the finalize write (status=Success) blocks - the initial creation put (status=
+ // InProgress) must go through immediately, or sendCommand itself would never return.
+ if ("Success".equals(value.getStatus())) {
+ rollupHoldingLock.countDown();
+ assertTrue(releaseRollup.await(5, TimeUnit.SECONDS),
+ "test did not release the rollup in time");
+ }
+ return invocation.callRealMethod();
+ }).when(commandStore).put(any(), any());
+
+ SsmCommandService service = new SsmCommandService(
+ new SingleCommandStoreFactory(commandStore), objectMapper, regionResolver, executor);
+
+ Command command = service.sendCommand(objectMapper.readTree("""
+ {
+ "InstanceIds": ["i-race"],
+ "DocumentName": "AWS-RunShellScript",
+ "Parameters": {
+ "commands": ["echo hi"]
+ },
+ "TimeoutSeconds": 60
+ }
+ """), "us-west-2");
+
+ assertTrue(rollupHoldingLock.await(5, TimeUnit.SECONDS), "rollup never reached its blocking put()");
+
+ AtomicBoolean cancelReturned = new AtomicBoolean(false);
+ Thread cancelThread = new Thread(() -> {
+ service.cancelCommand(command.getCommandId(), null, "us-west-2");
+ cancelReturned.set(true);
+ });
+ cancelThread.start();
+
+ // Give cancelCommand every chance to (wrongly) proceed if the lock weren't shared.
+ Thread.sleep(200);
+ assertFalse(cancelReturned.get(),
+ "cancelCommand must block while the rollup holds the command's lock, not race past it");
+
+ releaseRollup.countDown();
+ cancelThread.join(5000);
+ assertTrue(cancelReturned.get(), "cancelCommand should complete once the rollup releases the lock");
+
+ Command finalCommand = service.listCommands(command.getCommandId(), null, "us-west-2").getFirst();
+ assertEquals("Cancelled", finalCommand.getStatus());
+ assertEquals("Cancelled", finalCommand.getStatusDetails());
+ }
+
+ @Test
+ void listCommandsBlocksWhileARollupHoldsTheCommandLock() throws Exception {
+ // Regression flagged in review on PR #2477: even with writers now synchronized against each
+ // other, SsmJsonHandler.commandToNode() (the ListCommands serializer) used to read Status and
+ // StatusDetails from the same live, shared Command object without taking that same lock. A
+ // ListCommands call whose two field reads straddled a concurrent writer's update could observe
+ // a torn pair - e.g. Status read before the writer's change, StatusDetails read after it
+ // completes - reporting a combination that was never actually true at any single instant. This
+ // is plausibly the actual path #2264 was originally observed through, since a customer polls
+ // status via ListCommands, not the internal rollup directly.
+ //
+ // Proven the same way as the writer-vs-writer case: commandStore.put() is called from inside
+ // the synchronized block that guards the write, so a spy can force the rollup to block while
+ // still holding the lock, and this asserts a concurrent ListCommands call (which now also
+ // synchronizes on the same object before reading) blocks too rather than observing a torn read.
+ SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);
+ Instant start = Instant.parse("2026-06-07T00:00:00Z");
+ Instant end = Instant.parse("2026-06-07T00:00:01Z");
+ when(executor.supports(eq("i-race"), eq("AWS-RunShellScript"))).thenReturn(true);
+ when(executor.executeIfSupported(eq("i-race"), eq("AWS-RunShellScript"), any(), eq(60)))
+ .thenReturn(Optional.of(new SsmDirectCommandExecutor.ExecutionResult(
+ "Success", "done\n", "", 0, start, end)));
+
+ CountDownLatch rollupHoldingLock = new CountDownLatch(1);
+ CountDownLatch releaseRollup = new CountDownLatch(1);
+ AccountAwareStorageBackend<Command> realCommandStore = AccountAwareStorageBackend.inMemory("000000000000");
+ AccountAwareStorageBackend<Command> commandStore = spy(realCommandStore);
+ doAnswer(invocation -> {
+ Command value = invocation.getArgument(1);
+ if ("Success".equals(value.getStatus())) {
+ rollupHoldingLock.countDown();
+ assertTrue(releaseRollup.await(5, TimeUnit.SECONDS),
+ "test did not release the rollup in time");
+ }
+ return invocation.callRealMethod();
+ }).when(commandStore).put(any(), any());
+
+ SsmCommandService commandService = new SsmCommandService(
+ new SingleCommandStoreFactory(commandStore), objectMapper, regionResolver, executor);
+ SsmJsonHandler jsonHandler = new SsmJsonHandler(mock(SsmService.class), commandService, objectMapper);
+
+ Command command = commandService.sendCommand(objectMapper.readTree("""
+ {
+ "InstanceIds": ["i-race"],
+ "DocumentName": "AWS-RunShellScript",
+ "Parameters": {
+ "commands": ["echo hi"]
+ },
+ "TimeoutSeconds": 60
+ }
+ """), "us-west-2");
+
+ assertTrue(rollupHoldingLock.await(5, TimeUnit.SECONDS), "rollup never reached its blocking put()");
+
+ ObjectNode listRequest = objectMapper.createObjectNode().put("CommandId", command.getCommandId());
+ AtomicBoolean listReturned = new AtomicBoolean(false);
+ Thread listThread = new Thread(() -> {
+ jsonHandler.handle("ListCommands", listRequest, "us-west-2");
+ listReturned.set(true);
+ });
+ listThread.start();
+
+ // Give ListCommands every chance to (wrongly) proceed if the read weren't guarded by the
+ // same lock.
+ Thread.sleep(200);
+ assertFalse(listReturned.get(),
+ "ListCommands must block while the rollup holds the command's lock, not observe a torn read");
+
+ releaseRollup.countDown();
+ listThread.join(5000);
+ assertTrue(listReturned.get(), "ListCommands should complete once the rollup releases the lock");
+ }
+
+ @Test
+ void concurrentInstanceCompletions_neverLeaveStatusAndStatusDetailsInconsistent() throws Exception {
+ // Regression for #2264: commandStore is a plain ConcurrentHashMap-backed InMemoryStorage
+ // whose get() returns the SAME shared Command object on every call, not a defensive copy. A
+ // multi-instance command's last two instances completing at (nearly) the same moment can both
+ // observe "all done" and both enter updateCommandStatus's finalize branch concurrently on that
+ // one shared object. The finalize branch used to re-read command.getStatus() instead of the
+ // local variable it had just computed, so whichever thread's write landed last could leave
+ // statusDetails reflecting a DIFFERENT status than the one actually stored - exactly the
+ // reported symptom (status=TimedOut, statusDetails=In Progress).
+ //
+ // A CyclicBarrier forces every instance's mocked completion to release as close to
+ // simultaneously as Java threads allow, with many instances (not just two) and many trials
+ // to make several concurrently reaching the finalize branch as likely as this test can make
+ // it. In practice even 1200 such attempts (12 instances x 100 trials) never landed a thread
+ // exactly between the old code's setStatus() and its re-read of getStatus() - that gap is a
+ // few nanoseconds of adjacent bytecode, far narrower than normal thread-scheduling
+ // granularity in a clean, uncontended test JVM. That matches the issue's own report (passes
+ // locally, only flakes under real CI load, where GC pauses and dozens of other tests
+ // competing for CPU create far more scheduling noise than this test can reproduce alone).
+ // This test therefore isn't a fails-before/passes-after reproduction of the historical bug;
+ // it's a standing invariant check that concurrent completions never leave status and
+ // statusDetails disagreeing, which the fix satisfies by construction: statusDetails is now
+ // computed from a local variable no other thread can touch, so the specific TOCTOU window
+ // is closed regardless of scheduling, not merely made statistically unlikely to hit.
+ int instanceCount = 12;
+ int trials = 100;
+ for (int trial = 0; trial < trials; trial++) {
+ SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);
+ Instant start = Instant.parse("2026-06-07T00:00:00Z");
+ Instant end = Instant.parse("2026-06-07T00:00:01Z");
+ CyclicBarrier barrier = new CyclicBarrier(instanceCount);
+ List<String> instanceIds = new ArrayList<>();
+ for (int i = 0; i < instanceCount; i++) {
+ String instanceId = "i-" + i;
+ instanceIds.add(instanceId);
+ when(executor.supports(eq(instanceId), eq("AWS-RunShellScript"))).thenReturn(true);
+ when(executor.executeIfSupported(eq(instanceId), eq("AWS-RunShellScript"), any(), eq(60)))
+ .thenAnswer(invocation -> {
+ barrier.await(5, TimeUnit.SECONDS);
+ return Optional.of(new SsmDirectCommandExecutor.ExecutionResult(
+ "Success", "done\n", "", 0, start, end));
+ });
+ }
+
+ SsmCommandService service = new SsmCommandService(
+ new InMemoryStorageFactory(), objectMapper, regionResolver, executor);
+
+ ObjectNode request = objectMapper.createObjectNode();
+ ArrayNode instanceIdsNode = request.putArray("InstanceIds");
+ instanceIds.forEach(instanceIdsNode::add);
+ request.put("DocumentName", "AWS-RunShellScript");
+ request.putObject("Parameters").putArray("commands").add("echo hi");
+ request.put("TimeoutSeconds", 60);
+
+ Command command = service.sendCommand(request, "us-west-2");
+
+ String finalStatus = waitForCommandStatus(service, command.getCommandId(), "us-west-2");
+ Command updated = service.listCommands(command.getCommandId(), null, "us-west-2").getFirst();
+ assertEquals("Success", finalStatus, "trial " + trial + ": command status");
+ assertEquals("Success", updated.getStatus(), "trial " + trial + ": command status on re-fetch");
+ assertEquals("Success", updated.getStatusDetails(), "trial " + trial + ": statusDetails must match status");
+ }
+ }
+
@Test
void mixedDirectAndQueuedTargetsRemainInProgress() throws Exception {
SsmDirectCommandExecutor executor = mock(SsmDirectCommandExecutor.class);
@@ -396,4 +620,25 @@ class SsmCommandServiceDirectExecutionTest {
return AccountAwareStorageBackend.inMemory("000000000000");
}
}
+
+ /** Like InMemoryStorageFactory, but hands out a caller-supplied backend for ssm-commands.json
+ * specifically, so a test can wrap it (e.g. via Mockito spy) to control its timing. */
+ private static final class SingleCommandStoreFactory extends StorageFactory {
+ private final AccountAwareStorageBackend<Command> commandStore;
+
+ private SingleCommandStoreFactory(AccountAwareStorageBackend<Command> commandStore) {
+ super(null, null);
+ this.commandStore = commandStore;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ if ("ssm-commands.json".equals(fileName)) {
+ return (AccountAwareStorageBackend<V>) commandStore;
+ }
+ return AccountAwareStorageBackend.inMemory("000000000000");
+ }
+ }
}