Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sqs): reflect CancelMessageMoveTask at once instead of after the rate interval (#3757)

David8d755c20d646
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
index 3cee5ca85c..56f3a1d61a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
@@ -30,6 +30,8 @@ import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@ApplicationScoped
@@ -54,8 +56,9 @@ public class SqsService implements Resettable, ResourceProvider {
private final ConcurrentHashMap<String, ConcurrentHashMap<String, Instant>> deduplicationCache = new ConcurrentHashMap<>();
/** Move tasks keyed by opaque task handle. */
private final MoveTaskStore moveTasksByHandle;
- /** Per-task cancellation flag the move worker polls between iterations. */
- private final ConcurrentHashMap<String, java.util.concurrent.atomic.AtomicBoolean> moveTaskCancellation =
+ /** Per-task cancellation signal: the move worker waits on it between moves, so a cancel wakes
+ * the worker instead of being noticed only after the rate interval has elapsed. */
+ private final ConcurrentHashMap<String, MoveTaskCancellation> moveTaskCancellation =
new ConcurrentHashMap<>();
/** Move tasks execute on a background thread so MaxNumberOfMessagesPerSecond can throttle
* and CancelMessageMoveTask has something to interrupt. One thread per task is sufficient. */
@@ -78,6 +81,26 @@ public class SqsService implements Resettable, ResourceProvider {
long startedTimestampMillis, String failureReason) {
}
+ /**
+ * A cancel request the worker can wait on. {@link #awaitRequested} is the worker's throttle
+ * sleep: it returns early, and true, as soon as the task is cancelled.
+ */
+ private static final class MoveTaskCancellation {
+ private final CountDownLatch requested = new CountDownLatch(1);
+
+ void request() {
+ requested.countDown();
+ }
+
+ boolean isRequested() {
+ return requested.getCount() == 0;
+ }
+
+ boolean awaitRequested(long millis) throws InterruptedException {
+ return requested.await(millis, TimeUnit.MILLISECONDS);
+ }
+ }
+
private static final class MoveTaskStore {
private final Map<String, Entry> entries = new HashMap<>();
private final Clock clock;
@@ -105,7 +128,7 @@ public class SqsService implements Resettable, ResourceProvider {
}
private synchronized void put(String taskHandle, MoveTask task) {
- boolean terminal = !"RUNNING".equals(task.status());
+ boolean terminal = !isActive(task.status());
Long terminalAtMillis = terminal ? clock.millis() : null;
long sequence = terminal ? ++terminalSequence : 0;
entries.put(taskHandle, new Entry(task, terminalAtMillis, sequence));
@@ -147,6 +170,60 @@ public class SqsService implements Resettable, ResourceProvider {
}
}
+ private static boolean isActive(String status) {
+ return "RUNNING".equals(status) || "CANCELLING".equals(status);
+ }
+
+ /**
+ * Records the worker's running count against whatever status the task holds right now.
+ * The worker and {@code cancelMessageMoveTask} write the same record from two threads,
+ * so the read of the current status and the write back happen under the store's lock:
+ * a cancel that lands in between is never stomped back to RUNNING.
+ */
+ private synchronized void recordMoved(String taskHandle, long moved) {
+ Entry entry = entries.get(taskHandle);
+ if (entry == null) {
+ return;
+ }
+ MoveTask cur = entry.task();
+ put(taskHandle, new MoveTask(cur.taskHandle(), cur.sourceArn(), cur.destinationArn(),
+ cur.maxNumberOfMessagesPerSecond(), cur.status(), moved,
+ cur.approximateNumberOfMessagesToMove(), cur.startedTimestampMillis(), cur.failureReason()));
+ }
+
+ /**
+ * The worker's terminal write, resolved against the status the task holds at that moment.
+ * A cancel accepted while the worker was deciding has already moved the task to
+ * CANCELLING, so it ends CANCELLED whatever the worker concluded from its own signal;
+ * otherwise the worker's view (COMPLETED, or CANCELLED when it saw the cancel) stands.
+ * Reading the status and writing the outcome happen under one lock, so the two writers
+ * cannot interleave between them.
+ */
+ private synchronized void finish(String taskHandle, long moved, boolean cancellationRequested) {
+ Entry entry = entries.get(taskHandle);
+ if (entry == null) {
+ return;
+ }
+ MoveTask cur = entry.task();
+ String status = "CANCELLING".equals(cur.status()) || cancellationRequested ? "CANCELLED" : "COMPLETED";
+ put(taskHandle, new MoveTask(cur.taskHandle(), cur.sourceArn(), cur.destinationArn(),
+ cur.maxNumberOfMessagesPerSecond(), status, moved,
+ cur.approximateNumberOfMessagesToMove(), cur.startedTimestampMillis(), cur.failureReason()));
+ }
+
+ /** Moves a task from one status to another only if it still holds the expected one. */
+ private synchronized boolean transition(String taskHandle, String from, String to) {
+ Entry entry = entries.get(taskHandle);
+ if (entry == null || !from.equals(entry.task().status())) {
+ return false;
+ }
+ MoveTask cur = entry.task();
+ put(taskHandle, new MoveTask(cur.taskHandle(), cur.sourceArn(), cur.destinationArn(),
+ cur.maxNumberOfMessagesPerSecond(), to, cur.approximateNumberOfMessagesMoved(),
+ cur.approximateNumberOfMessagesToMove(), cur.startedTimestampMillis(), cur.failureReason()));
+ return true;
+ }
+
private record Entry(MoveTask task, Long terminalAtMillis, long terminalSequence) {
private boolean terminal() {
return terminalAtMillis != null;
@@ -257,7 +334,7 @@ public class SqsService implements Resettable, ResourceProvider {
queueLocks.clear();
redrivePolicyCache.clear();
deduplicationCache.clear();
- moveTaskCancellation.values().forEach(flag -> flag.set(true));
+ moveTaskCancellation.values().forEach(MoveTaskCancellation::request);
moveTaskCancellation.clear();
moveTasksByHandle.clear();
}
@@ -1068,7 +1145,7 @@ public class SqsService implements Resettable, ResourceProvider {
if (!sourceArn.equals(existing.sourceArn())) {
continue;
}
- if ("RUNNING".equals(existing.status())
+ if (MoveTaskStore.isActive(existing.status())
|| (now - existing.startedTimestampMillis()) < 1_000) {
throw new AwsException("InvalidParameterValue",
"There is already a task running. Only one active task is allowed for a source queue arn at a given time.",
@@ -1099,7 +1176,7 @@ public class SqsService implements Resettable, ResourceProvider {
taskHandle, sourceArn, destinationArn,
maxNumberOfMessagesPerSecond, "RUNNING",
0L, toMove, clock.millis(), null));
- var cancelled = new java.util.concurrent.atomic.AtomicBoolean(false);
+ MoveTaskCancellation cancelled = new MoveTaskCancellation();
moveTaskCancellation.put(taskHandle, cancelled);
// Move the first message synchronously inside the request scope so callers
@@ -1181,23 +1258,22 @@ public class SqsService implements Resettable, ResourceProvider {
private void runMoveTask(String taskHandle, String srcKey, String destUrl,
int maxRate, String region, String sourceArn,
String destinationArn,
- java.util.concurrent.atomic.AtomicBoolean cancelled,
+ MoveTaskCancellation cancelled,
long initialMoved) {
long intervalMillis = maxRate > 0 ? Math.max(1L, 1000L / maxRate) : 0L;
long moved = initialMoved;
try {
var srcQueue = getOrCreateQueue(srcKey);
- while (!cancelled.get()) {
+ while (!cancelled.isRequested()) {
if (intervalMillis > 0) {
try {
- Thread.sleep(intervalMillis);
+ if (cancelled.awaitRequested(intervalMillis)) {
+ break;
+ }
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
return;
}
- if (cancelled.get()) {
- break;
- }
}
if (!moveOneMessage(srcQueue, destUrl, region)) {
break;
@@ -1206,32 +1282,26 @@ public class SqsService implements Resettable, ResourceProvider {
updateMoveTaskCounter(taskHandle, moved);
}
} finally {
- MoveTask cur = moveTasksByHandle.get(taskHandle);
- if (cur != null) {
- String status = cancelled.get() ? "CANCELLED" : "COMPLETED";
- moveTasksByHandle.put(taskHandle, new MoveTask(
- cur.taskHandle(), cur.sourceArn(), cur.destinationArn(),
- cur.maxNumberOfMessagesPerSecond(), status,
- moved, cur.approximateNumberOfMessagesToMove(),
- cur.startedTimestampMillis(), cur.failureReason()));
- }
+ finishMoveTask(taskHandle, moved, cancelled.isRequested());
moveTaskCancellation.remove(taskHandle, cancelled);
LOG.infov("Move task {0} {1}: moved {2} messages from {3} to {4}", taskHandle,
- cancelled.get() ? "cancelled" : "completed", moved, sourceArn,
+ cancelled.isRequested() ? "cancelled" : "completed", moved, sourceArn,
destinationArn != null ? destinationArn : "original source");
}
}
private void updateMoveTaskCounter(String taskHandle, long moved) {
- MoveTask cur = moveTasksByHandle.get(taskHandle);
- if (cur == null) {
- return;
- }
- moveTasksByHandle.put(taskHandle, new MoveTask(
- cur.taskHandle(), cur.sourceArn(), cur.destinationArn(),
- cur.maxNumberOfMessagesPerSecond(), cur.status(),
- moved, cur.approximateNumberOfMessagesToMove(),
- cur.startedTimestampMillis(), cur.failureReason()));
+ moveTasksByHandle.recordMoved(taskHandle, moved);
+ }
+
+ /**
+ * Records a move task's outcome. {@code cancellationRequested} is what the worker saw on its
+ * own signal; the store still resolves a task that a cancel has already moved to CANCELLING
+ * as CANCELLED. Package-private so a test can replay the worker's terminal write with a stale
+ * decision after a cancel has been accepted.
+ */
+ void finishMoveTask(String taskHandle, long moved, boolean cancellationRequested) {
+ moveTasksByHandle.finish(taskHandle, moved, cancellationRequested);
}
public List<MoveTask> listMessageMoveTasks(String sourceArn, String region) {
@@ -1257,11 +1327,13 @@ public class SqsService implements Resettable, ResourceProvider {
throw new AwsException("ResourceNotFoundException",
"The task you specified does not exist.", 404);
}
- // Signal the background worker to stop. The worker flips status to CANCELLED in
- // its finally block and updates the moved counter; read both back here.
- var flag = moveTaskCancellation.get(taskHandle);
- if (flag != null) {
- flag.set(true);
+ // As on AWS the task reports CANCELLING at once and CANCELLED when the worker has
+ // stopped: the worker is woken out of its throttle wait rather than left to notice the
+ // cancel after the interval, and its finally block writes CANCELLED and the final count.
+ moveTasksByHandle.transition(taskHandle, "RUNNING", "CANCELLING");
+ MoveTaskCancellation cancellation = moveTaskCancellation.get(taskHandle);
+ if (cancellation != null) {
+ cancellation.request();
}
return moveTasksByHandle.getCurrentOrDefault(taskHandle, task).approximateNumberOfMessagesMoved();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
index a7fc809f47..2e32fba00d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
@@ -1309,6 +1309,102 @@ class SqsServiceTest {
awaitMoveTaskStatus(dlqArn, taskHandle, "CANCELLED");
}
+ @Test
+ void cancelMessageMoveTask_takesEffectWithoutWaitingOutTheRateInterval() throws Exception {
+ Queue dlq = sqsService.createQueue("cancel-latency-dlq", null, "us-east-1");
+ String dlqArn = queueArn("cancel-latency-dlq");
+ sqsService.createQueue("cancel-latency-source",
+ Map.of("RedrivePolicy",
+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\",\"maxReceiveCount\":\"1\"}"),
+ "us-east-1");
+ sqsService.createQueue("cancel-latency-destination", null, "us-east-1");
+ String destinationArn = queueArn("cancel-latency-destination");
+ for (int i = 0; i < 50; i++) {
+ sqsService.sendMessage(dlq.getQueueUrl(), "msg-" + i, 0, null, null, "us-east-1");
+ }
+
+ // At one message per second the worker sleeps a full second between moves. AWS
+ // reflects a cancel immediately (CANCELLING, then CANCELLED once nothing is in
+ // flight); the worker must be woken, not left to sleep out its interval.
+ String taskHandle = sqsService.startMessageMoveTask(dlqArn, destinationArn, 1, "us-east-1");
+ sqsService.cancelMessageMoveTask(taskHandle, "us-east-1");
+
+ String statusRightAfterCancel = moveTaskStatus(dlqArn, taskHandle);
+ assertTrue("CANCELLING".equals(statusRightAfterCancel) || "CANCELLED".equals(statusRightAfterCancel),
+ "expected CANCELLING or CANCELLED right after CancelMessageMoveTask returned, was " + statusRightAfterCancel);
+
+ long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(300);
+ while (!"CANCELLED".equals(moveTaskStatus(dlqArn, taskHandle))) {
+ assertTrue(System.nanoTime() < deadline,
+ "move task still " + moveTaskStatus(dlqArn, taskHandle) + " 300 ms after the cancel");
+ Thread.sleep(5);
+ }
+ assertEquals(1, sqsService.listMessageMoveTasks(dlqArn, "us-east-1").size());
+ }
+
+ @Test
+ void cancelMessageMoveTask_statusNeverRegressesToRunningWhileTheWorkerRecordsProgress() throws Exception {
+ Queue dlq = sqsService.createQueue("cancel-stomp-dlq", null, "us-east-1");
+ String dlqArn = queueArn("cancel-stomp-dlq");
+ sqsService.createQueue("cancel-stomp-source",
+ Map.of("RedrivePolicy",
+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\",\"maxReceiveCount\":\"1\"}"),
+ "us-east-1");
+ sqsService.createQueue("cancel-stomp-destination", null, "us-east-1");
+ String destinationArn = queueArn("cancel-stomp-destination");
+ for (int i = 0; i < 3000; i++) {
+ sqsService.sendMessage(dlq.getQueueUrl(), "msg-" + i, 0, null, null, "us-east-1");
+ }
+
+ // Unthrottled, so the worker records its count after every message while the cancel
+ // lands: the two writers share the store's lock, and once the cancel has returned the
+ // task must only ever read CANCELLING or CANCELLED.
+ String taskHandle = sqsService.startMessageMoveTask(dlqArn, destinationArn, "us-east-1");
+ sqsService.cancelMessageMoveTask(taskHandle, "us-east-1");
+
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ String status;
+ while (!"CANCELLED".equals(status = moveTaskStatus(dlqArn, taskHandle))) {
+ assertNotEquals("RUNNING", status, "a cancelled task reported RUNNING again");
+ assertTrue(System.nanoTime() < deadline, "move task never reached CANCELLED, last seen " + status);
+ }
+ }
+
+ @Test
+ void cancelMessageMoveTask_acceptedCancelIsNotOverwrittenByTheWorkersTerminalWrite() throws Exception {
+ Queue dlq = sqsService.createQueue("cancel-terminal-dlq", null, "us-east-1");
+ String dlqArn = queueArn("cancel-terminal-dlq");
+ sqsService.createQueue("cancel-terminal-source",
+ Map.of("RedrivePolicy",
+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\",\"maxReceiveCount\":\"1\"}"),
+ "us-east-1");
+ sqsService.createQueue("cancel-terminal-destination", null, "us-east-1");
+ String destinationArn = queueArn("cancel-terminal-destination");
+ for (int i = 0; i < 50; i++) {
+ sqsService.sendMessage(dlq.getQueueUrl(), "msg-" + i, 0, null, null, "us-east-1");
+ }
+
+ // The interleaving: the worker reads its own signal and concludes COMPLETED, the cancel is
+ // accepted (the task reads CANCELLING), and only then does the worker's terminal write
+ // land. Replaying that write with the stale decision must not turn CANCELLING back into
+ // COMPLETED.
+ String taskHandle = sqsService.startMessageMoveTask(dlqArn, destinationArn, 1, "us-east-1");
+ sqsService.cancelMessageMoveTask(taskHandle, "us-east-1");
+ sqsService.finishMoveTask(taskHandle, 1, false);
+
+ assertEquals("CANCELLED", moveTaskStatus(dlqArn, taskHandle));
+ awaitMoveTaskStatus(dlqArn, taskHandle, "CANCELLED");
+ assertEquals("CANCELLED", moveTaskStatus(dlqArn, taskHandle));
+ }
+
+ private String moveTaskStatus(String sourceArn, String taskHandle) {
+ return sqsService.listMessageMoveTasks(sourceArn, "us-east-1").stream()
+ .filter(task -> task.taskHandle().equals(taskHandle))
+ .map(SqsService.MoveTask::status)
+ .findFirst()
+ .orElse("<absent>");
+ }
+
@Test
void completedMessageMoveTasks_areBoundedToRecentSummaries() throws Exception {
sqsService.createQueue("terminal-cap-dlq", null, "us-east-1");