Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sqs): bound message move task history (#3292)

Niels Kootstra520568ef9365
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
index bf16a2fdb9..a2846696e9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
@@ -25,6 +25,8 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.time.Clock;
+import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@@ -36,6 +38,8 @@ public class SqsService implements Resettable, ResourceProvider {
private static final Logger LOG = Logger.getLogger(SqsService.class);
private static final int DEDUP_WINDOW_SECONDS = 300; // 5 minutes
private static final int MAX_RECEIVE_WAIT_TIME_SECONDS = 20;
+ private static final int MAX_TERMINAL_MOVE_TASKS = 10;
+ private static final Duration TERMINAL_MOVE_TASK_TTL = Duration.ofHours(1);
private final StorageBackend<String, Queue> queueStore;
private final StorageBackend<String, List<Message>> messageStore;
@@ -45,7 +49,7 @@ public class SqsService implements Resettable, ResourceProvider {
private final ConcurrentHashMap<String, RedrivePolicy> redrivePolicyCache = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, ConcurrentHashMap<String, Instant>> deduplicationCache = new ConcurrentHashMap<>();
/** Move tasks keyed by opaque task handle. */
- private final ConcurrentHashMap<String, MoveTask> moveTasksByHandle = new ConcurrentHashMap<>();
+ private final MoveTaskStore moveTasksByHandle;
/** Per-task cancellation flag the move worker polls between iterations. */
private final ConcurrentHashMap<String, java.util.concurrent.atomic.AtomicBoolean> moveTaskCancellation =
new ConcurrentHashMap<>();
@@ -70,16 +74,98 @@ public class SqsService implements Resettable, ResourceProvider {
long startedTimestampMillis, String failureReason) {
}
+ private static final class MoveTaskStore {
+ private final Map<String, Entry> entries = new HashMap<>();
+ private final Clock clock;
+ private long terminalSequence;
+
+ private MoveTaskStore(Clock clock) {
+ this.clock = clock;
+ }
+
+ private synchronized MoveTask get(String taskHandle) {
+ cleanup();
+ Entry entry = entries.get(taskHandle);
+ return entry == null ? null : entry.task();
+ }
+
+ private synchronized MoveTask getCurrentOrDefault(String taskHandle, MoveTask fallback) {
+ Entry entry = entries.get(taskHandle);
+ MoveTask task = entry == null ? null : entry.task();
+ return task == null ? fallback : task;
+ }
+
+ private synchronized Collection<MoveTask> values() {
+ cleanup();
+ return entries.values().stream().map(Entry::task).toList();
+ }
+
+ private synchronized void put(String taskHandle, MoveTask task) {
+ boolean terminal = !"RUNNING".equals(task.status());
+ Long terminalAtMillis = terminal ? clock.millis() : null;
+ long sequence = terminal ? ++terminalSequence : 0;
+ entries.put(taskHandle, new Entry(task, terminalAtMillis, sequence));
+ if (terminal) {
+ cleanup();
+ evictOldestTerminalTasks();
+ }
+ }
+
+ private synchronized void clear() {
+ entries.clear();
+ }
+
+ private void cleanup() {
+ long now = clock.millis();
+ entries.entrySet().removeIf(entry -> {
+ Long terminalAtMillis = entry.getValue().terminalAtMillis();
+ return terminalAtMillis != null
+ && now >= terminalAtMillis
+ && now - terminalAtMillis >= TERMINAL_MOVE_TASK_TTL.toMillis();
+ });
+ }
+
+ private void evictOldestTerminalTasks() {
+ Set<String> sourceArns = entries.values().stream()
+ .filter(Entry::terminal)
+ .map(entry -> entry.task().sourceArn())
+ .collect(java.util.stream.Collectors.toSet());
+ for (String sourceArn : sourceArns) {
+ List<Map.Entry<String, Entry>> terminalEntries = entries.entrySet().stream()
+ .filter(entry -> entry.getValue().terminal()
+ && Objects.equals(sourceArn, entry.getValue().task().sourceArn()))
+ .sorted(Map.Entry.comparingByValue(Comparator.comparingLong(Entry::terminalSequence)))
+ .toList();
+ int excess = terminalEntries.size() - MAX_TERMINAL_MOVE_TASKS;
+ for (int i = 0; i < excess; i++) {
+ entries.remove(terminalEntries.get(i).getKey());
+ }
+ }
+ }
+
+ private record Entry(MoveTask task, Long terminalAtMillis, long terminalSequence) {
+ private boolean terminal() {
+ return terminalAtMillis != null;
+ }
+ }
+ }
+
private final int defaultVisibilityTimeout;
private final int maxMessageSize;
private final String baseUrl;
private final RegionResolver regionResolver;
private final boolean clearFifoDeduplicationCacheOnPurge;
private final SnsService snsService;
+ private final Clock clock;
- @Inject
public SqsService(StorageFactory storageFactory, EmulatorConfig config, RegionResolver regionResolver,
SnsService snsService) {
+ this(storageFactory, config, regionResolver, snsService, Clock.systemUTC());
+ }
+
+ @Inject
+ public SqsService(StorageFactory storageFactory, EmulatorConfig config, RegionResolver regionResolver,
+ SnsService snsService, Clock clock) {
this(
storageFactory.create("sqs", "sqs-queues.json",
new TypeReference<Map<String, Queue>>() {
@@ -95,7 +181,8 @@ public class SqsService implements Resettable, ResourceProvider {
config.effectiveBaseUrl(),
regionResolver,
config.services().sqs().clearFifoDeduplicationCacheOnPurge(),
- snsService
+ snsService,
+ clock
);
}
@@ -108,6 +195,12 @@ public class SqsService implements Resettable, ResourceProvider {
new RegionResolver("us-east-1", "000000000000"), false, null);
}
+ SqsService(StorageBackend<String, Queue> queueStore,
+ int defaultVisibilityTimeout, int maxMessageSize, String baseUrl, Clock clock) {
+ this(queueStore, null, null, defaultVisibilityTimeout, maxMessageSize, baseUrl,
+ new RegionResolver("us-east-1", "000000000000"), false, null, clock);
+ }
+
SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,
StorageBackend<String, Map<String, Long>> dedupStore,
int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
@@ -121,6 +214,15 @@ public class SqsService implements Resettable, ResourceProvider {
int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,
SnsService snsService) {
+ this(queueStore, messageStore, dedupStore, defaultVisibilityTimeout, maxMessageSize, baseUrl,
+ regionResolver, clearFifoDeduplicationCacheOnPurge, snsService, Clock.systemUTC());
+ }
+
+ SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,
+ StorageBackend<String, Map<String, Long>> dedupStore,
+ int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
+ RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,
+ SnsService snsService, Clock clock) {
this.queueStore = queueStore;
this.messageStore = messageStore;
this.dedupStore = dedupStore;
@@ -130,6 +232,8 @@ public class SqsService implements Resettable, ResourceProvider {
this.regionResolver = regionResolver;
this.clearFifoDeduplicationCacheOnPurge = clearFifoDeduplicationCacheOnPurge;
this.snsService = snsService;
+ this.clock = clock;
+ this.moveTasksByHandle = new MoveTaskStore(clock);
loadPersistedMessages();
loadPersistedDedup();
}
@@ -893,7 +997,7 @@ public class SqsService implements Resettable, ResourceProvider {
// the additional 1-second window covers the small gap between StartMessageMoveTask
// returning and the worker flipping the task to RUNNING (so back-to-back start
// calls that AWS would reject can't slip through).
- long now = System.currentTimeMillis();
+ long now = clock.millis();
for (MoveTask existing : moveTasksByHandle.values()) {
if (!sourceArn.equals(existing.sourceArn())) {
continue;
@@ -928,7 +1032,7 @@ public class SqsService implements Resettable, ResourceProvider {
moveTasksByHandle.put(taskHandle, new MoveTask(
taskHandle, sourceArn, destinationArn,
maxNumberOfMessagesPerSecond, "RUNNING",
- 0L, toMove, System.currentTimeMillis(), null));
+ 0L, toMove, clock.millis(), null));
var cancelled = new java.util.concurrent.atomic.AtomicBoolean(false);
moveTaskCancellation.put(taskHandle, cancelled);
@@ -1025,6 +1129,7 @@ public class SqsService implements Resettable, ResourceProvider {
moved, cur.approximateNumberOfMessagesToMove(),
cur.startedTimestampMillis(), cur.failureReason()));
}
+ moveTaskCancellation.remove(taskHandle, cancelled);
LOG.infov("Move task {0} {1}: moved {2} messages from {3} to {4}", taskHandle,
cancelled.get() ? "cancelled" : "completed", moved, sourceArn,
destinationArn != null ? destinationArn : "original source");
@@ -1072,7 +1177,7 @@ public class SqsService implements Resettable, ResourceProvider {
if (flag != null) {
flag.set(true);
}
- return moveTasksByHandle.getOrDefault(taskHandle, task).approximateNumberOfMessagesMoved();
+ return moveTasksByHandle.getCurrentOrDefault(taskHandle, task).approximateNumberOfMessagesMoved();
}
private boolean isDeadLetterQueue(String queueArn, String region) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailAdvancedSelectorsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailAdvancedSelectorsIntegrationTest.java
index 6cc9e04bbe..59248fad59 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailAdvancedSelectorsIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailAdvancedSelectorsIntegrationTest.java
@@ -2,10 +2,13 @@ package io.github.hectorvent.floci.services.cloudtrail;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
import jakarta.inject.Inject;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import java.util.Map;
import java.util.UUID;
import static io.restassured.RestAssured.given;
@@ -21,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
* recording its own delivery writes.
*/
@QuarkusTest
+@TestProfile(CloudTrailAdvancedSelectorsIntegrationTest.IsolatedProfile.class)
class CloudTrailAdvancedSelectorsIntegrationTest {
private static final String CT_TARGET = "CloudTrail_20131101.";
@@ -29,6 +33,13 @@ class CloudTrailAdvancedSelectorsIntegrationTest {
@Inject
CloudTrailLogWriter writer;
+ public static final class IsolatedProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of("floci.services.cloudtrail.flush-interval-seconds", "3600");
+ }
+ }
+
@BeforeAll
static void configureRestAssured() {
RestAssuredJsonUtils.configureAwsContentTypes();
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
index 1e88fe621b..95a1cdbae1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
@@ -7,9 +7,12 @@ import io.github.hectorvent.floci.services.sns.SnsService;
import io.github.hectorvent.floci.services.sqs.model.Message;
import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;
import io.github.hectorvent.floci.services.sqs.model.Queue;
+import io.github.hectorvent.floci.testing.MutableClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.time.Duration;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
@@ -23,11 +26,13 @@ import static org.mockito.Mockito.verify;
class SqsServiceTest {
private SqsService sqsService;
+ private MutableClock clock;
private static final String BASE_URL = "http://localhost:4566";
@BeforeEach
void setUp() {
- sqsService = new SqsService(new InMemoryStorage<>(), 30, 1048576, BASE_URL);
+ clock = new MutableClock();
+ sqsService = new SqsService(new InMemoryStorage<>(), 30, 1048576, BASE_URL, clock);
}
@Test
@@ -870,17 +875,132 @@ class SqsServiceTest {
}
String taskHandle = sqsService.startMessageMoveTask(dlqArn, destArn, 1, "us-east-1");
- sqsService.cancelMessageMoveTask(taskHandle, "us-east-1");
+ assertEquals(1, sqsService.cancelMessageMoveTask(taskHandle, "us-east-1"));
- // Give the worker a moment to observe the cancel and write the terminal status.
+ awaitMoveTaskStatus(dlqArn, taskHandle, "CANCELLED");
+ }
+
+ @Test
+ void completedMessageMoveTasks_areBoundedToRecentSummaries() throws Exception {
+ sqsService.createQueue("terminal-cap-dlq", null, "us-east-1");
+ String dlqArn = queueArn("terminal-cap-dlq");
+ sqsService.createQueue("terminal-cap-source",
+ Map.of("RedrivePolicy",
+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\",\"maxReceiveCount\":\"1\"}"),
+ "us-east-1");
+ sqsService.createQueue("terminal-cap-destination", null, "us-east-1");
+ String destinationArn = queueArn("terminal-cap-destination");
+
+ List<String> taskHandles = new ArrayList<>();
+ for (int i = 0; i < 11; i++) {
+ String taskHandle = sqsService.startMessageMoveTask(dlqArn, destinationArn, 0, "us-east-1");
+ taskHandles.add(taskHandle);
+ for (int attempt = 0; attempt < 50; attempt++) {
+ if (sqsService.listMessageMoveTasks(dlqArn, "us-east-1").stream()
+ .anyMatch(task -> task.taskHandle().equals(taskHandle) && "COMPLETED".equals(task.status()))) {
+ break;
+ }
+ Thread.sleep(10);
+ }
+ assertTrue(sqsService.listMessageMoveTasks(dlqArn, "us-east-1").stream()
+ .anyMatch(task -> task.taskHandle().equals(taskHandle) && "COMPLETED".equals(task.status())));
+ if (i < 10) {
+ clock.advance(Duration.ofSeconds(2));
+ }
+ }
+
+ List<SqsService.MoveTask> recentTasks = sqsService.listMessageMoveTasks(dlqArn, "us-east-1");
+ assertEquals(10, recentTasks.size());
+ assertFalse(recentTasks.stream().anyMatch(task -> task.taskHandle().equals(taskHandles.getFirst())));
+
+ clock.advance(Duration.ofHours(1).plusMillis(1));
+ assertTrue(sqsService.listMessageMoveTasks(dlqArn, "us-east-1").isEmpty());
+ AwsException ex = assertThrows(AwsException.class, () ->
+ sqsService.cancelMessageMoveTask(taskHandles.getLast(), "us-east-1"));
+ assertEquals("ResourceNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void completedMessageMoveTasks_areBoundedPerSourceQueue() throws Exception {
+ sqsService.createQueue("per-source-dlq-a", null, "us-east-1");
+ String sourceArnA = queueArn("per-source-dlq-a");
+ sqsService.createQueue("per-source-source-a",
+ Map.of("RedrivePolicy",
+ "{\"deadLetterTargetArn\":\"" + sourceArnA + "\",\"maxReceiveCount\":\"1\"}"),
+ "us-east-1");
+ sqsService.createQueue("per-source-destination-a", null, "us-east-1");
+ String destinationArnA = queueArn("per-source-destination-a");
+
+ sqsService.createQueue("per-source-dlq-b", null, "us-east-1");
+ String sourceArnB = queueArn("per-source-dlq-b");
+ sqsService.createQueue("per-source-source-b",
+ Map.of("RedrivePolicy",
+ "{\"deadLetterTargetArn\":\"" + sourceArnB + "\",\"maxReceiveCount\":\"1\"}"),
+ "us-east-1");
+ sqsService.createQueue("per-source-destination-b", null, "us-east-1");
+ String destinationArnB = queueArn("per-source-destination-b");
+
+ for (int i = 0; i < 10; i++) {
+ String taskHandle = sqsService.startMessageMoveTask(sourceArnA, destinationArnA, 0, "us-east-1");
+ awaitMoveTaskStatus(sourceArnA, taskHandle, "COMPLETED");
+ clock.advance(Duration.ofSeconds(2));
+ }
+ String taskHandleB = sqsService.startMessageMoveTask(sourceArnB, destinationArnB, 0, "us-east-1");
+ awaitMoveTaskStatus(sourceArnB, taskHandleB, "COMPLETED");
+
+ assertEquals(10, sqsService.listMessageMoveTasks(sourceArnA, "us-east-1").size());
+ assertEquals(1, sqsService.listMessageMoveTasks(sourceArnB, "us-east-1").size());
+ }
+
+ @Test
+ void cancelledMessageMoveTasks_areBoundedAndAllowTheNextTask() throws Exception {
+ Queue dlq = sqsService.createQueue("cancel-cap-dlq", null, "us-east-1");
+ String dlqArn = queueArn("cancel-cap-dlq");
+ sqsService.createQueue("cancel-cap-source",
+ Map.of("RedrivePolicy",
+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\",\"maxReceiveCount\":\"1\"}"),
+ "us-east-1");
+ sqsService.createQueue("cancel-cap-destination", null, "us-east-1");
+ String destinationArn = queueArn("cancel-cap-destination");
for (int i = 0; i < 50; i++) {
- var status = sqsService.listMessageMoveTasks(dlqArn, "us-east-1").get(0).status();
- if ("CANCELLED".equals(status)) {
+ sqsService.sendMessage(dlq.getQueueUrl(), "msg-" + i, 0, null, null, "us-east-1");
+ }
+
+ List<String> taskHandles = new ArrayList<>();
+ for (int i = 0; i < 11; i++) {
+ String taskHandle = sqsService.startMessageMoveTask(dlqArn, destinationArn, 1, "us-east-1");
+ taskHandles.add(taskHandle);
+ sqsService.cancelMessageMoveTask(taskHandle, "us-east-1");
+ awaitMoveTaskStatus(dlqArn, taskHandle, "CANCELLED");
+ if (i < 10) {
+ clock.advance(Duration.ofSeconds(2));
+ }
+ }
+
+ List<SqsService.MoveTask> recentTasks = sqsService.listMessageMoveTasks(dlqArn, "us-east-1");
+ assertEquals(10, recentTasks.size());
+ assertFalse(recentTasks.stream().anyMatch(task -> task.taskHandle().equals(taskHandles.getFirst())));
+
+ clock.advance(Duration.ofSeconds(2));
+ String nextTask = sqsService.startMessageMoveTask(dlqArn, destinationArn, 1, "us-east-1");
+ sqsService.cancelMessageMoveTask(nextTask, "us-east-1");
+ awaitMoveTaskStatus(dlqArn, nextTask, "CANCELLED");
+
+ clock.advance(Duration.ofHours(1).plusMillis(1));
+ assertTrue(sqsService.listMessageMoveTasks(dlqArn, "us-east-1").isEmpty());
+ }
+
+ private void awaitMoveTaskStatus(String sourceArn, String taskHandle, String expectedStatus) throws Exception {
+ for (int attempt = 0; attempt < 100; attempt++) {
+ boolean reachedStatus = sqsService.listMessageMoveTasks(sourceArn, "us-east-1").stream()
+ .anyMatch(task -> task.taskHandle().equals(taskHandle)
+ && expectedStatus.equals(task.status()));
+ if (reachedStatus) {
return;
}
- Thread.sleep(50);
+ Thread.sleep(10);
}
- fail("Move task did not transition to CANCELLED within timeout");
+ fail("Move task did not transition to " + expectedStatus + " within timeout");
}
@Test