Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): redrive failed SQS ESM messages to the DLQ (#1419)

Stuart Langdba23bf7e2c3
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
index 3c1f424afe..6061706840 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
@@ -35,6 +35,9 @@ public class SqsEventSourcePoller {
private static final Logger LOG = Logger.getLogger(SqsEventSourcePoller.class);
+ /** AWS SQS default visibility timeout, used as the retry backoff when a queue has none configured. */
+ private static final int DEFAULT_RETRY_VISIBILITY_SECONDS = 30;
+
private final Vertx vertx;
private final SqsService sqsService;
private final LambdaExecutorService executorService;
@@ -114,7 +117,8 @@ public class SqsEventSourcePoller {
}
}
- private void pollAndInvoke(EventSourceMapping esm) {
+ /** Package-private (not private) only so unit tests can drive a single poll directly. */
+ void pollAndInvoke(EventSourceMapping esm) {
// Skip this tick if a previous poll for this ESM is still in progress.
// This prevents concurrent deliveries of the same message when the Lambda
// cold-start / execution time exceeds the SQS visibility timeout.
@@ -176,9 +180,18 @@ public class SqsEventSourcePoller {
msg.getMessageId(), e.getMessage());
}
}
+ // Reported partial-batch failures are not deleted; return them to the
+ // queue immediately so they can be retried/redriven rather than sitting
+ // in-flight for the full execution-cover visibility window.
+ if (!failedIds.isEmpty()) {
+ List<Message> toReturn = messages.stream()
+ .filter(m -> failedIds.contains(m.getMessageId())).toList();
+ returnMessagesToQueue(esm, toReturn);
+ }
} else {
- LOG.warnv("ESM {0}: Lambda returned error [{1}], messages will return to queue",
- esm.getUuid(), result.getFunctionError());
+ LOG.warnv("ESM {0}: Lambda returned error [{1}], returning {2} message(s) to queue for retry/redrive",
+ esm.getUuid(), result.getFunctionError(), messages.size());
+ returnMessagesToQueue(esm, messages);
}
} catch (Exception e) {
LOG.warnv("ESM {0}: poll/invoke error: {1} ({2})",
@@ -189,6 +202,51 @@ public class SqsEventSourcePoller {
});
}
+ /**
+ * Returns failed messages to the source queue by resetting their visibility timeout
+ * to the queue's own {@code VisibilityTimeout}. The poller hides messages for
+ * {@code fn.timeout + 30s} to cover execution time, but on failure that long window
+ * would keep the message in-flight (and therefore not redelivered nor redriven) far
+ * longer than the queue's own visibility/redrive policy intends. Shrinking the window
+ * back to the queue's visibility timeout lets the next poll re-receive them — matching
+ * AWS's redelivery cadence rather than spinning a tight retry loop (which resetting to
+ * 0 would cause for a persistently failing function) — so ApproximateReceiveCount
+ * climbs and the queue's RedrivePolicy moves them to the DLQ once
+ * {@code maxReceiveCount} is exceeded.
+ */
+ private void returnMessagesToQueue(EventSourceMapping esm, List<Message> messages) {
+ int retryVisibility = retryVisibilityTimeout(esm);
+ for (Message msg : messages) {
+ try {
+ sqsService.changeMessageVisibility(
+ esm.getQueueUrl(), msg.getReceiptHandle(), retryVisibility, esm.getRegion());
+ } catch (Exception e) {
+ LOG.warnv("ESM {0}: failed to return message {1} to queue: {2}",
+ esm.getUuid(), msg.getMessageId(), e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * The visibility timeout to apply when returning a failed message to the queue: the
+ * queue's configured {@code VisibilityTimeout}, or the AWS default of 30s when unset
+ * or unreadable. This governs how soon the message is retried/redriven.
+ */
+ private int retryVisibilityTimeout(EventSourceMapping esm) {
+ try {
+ String vt = sqsService.getQueueAttributes(
+ esm.getQueueUrl(), List.of("VisibilityTimeout"), esm.getRegion())
+ .get("VisibilityTimeout");
+ if (vt != null) {
+ return Math.max(0, Integer.parseInt(vt));
+ }
+ } catch (Exception e) {
+ LOG.debugv("ESM {0}: could not read VisibilityTimeout, using default {1}s backoff: {2}",
+ esm.getUuid(), DEFAULT_RETRY_VISIBILITY_SECONDS, e.getMessage());
+ }
+ return DEFAULT_RETRY_VISIBILITY_SECONDS;
+ }
+
private Set<String> extractBatchItemFailures(EventSourceMapping esm, InvokeResult result) {
if (!esm.isReportBatchItemFailures() || result.getPayload() == null || result.getPayload().length == 0) {
return Set.of();
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java
index bf2ba9fbd5..96559e5fbe 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java
@@ -4,6 +4,9 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
import io.github.hectorvent.floci.services.sqs.SqsService;
import io.github.hectorvent.floci.services.sqs.model.Message;
import io.vertx.core.Vertx;
@@ -12,15 +15,26 @@ import org.junit.jupiter.api.Test;
import java.time.Instant;
import java.util.List;
+import java.util.Map;
+import java.util.Optional;
import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class SqsEventSourcePollerTest {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private SqsEventSourcePoller poller;
+ private SqsService sqsService;
+ private LambdaExecutorService executorService;
+ private LambdaFunctionStore functionStore;
@BeforeEach
void setUp() {
@@ -32,11 +46,15 @@ class SqsEventSourcePollerTest {
when(lambdaConfig.pollIntervalMs()).thenReturn(1000L);
when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ sqsService = mock(SqsService.class);
+ executorService = mock(LambdaExecutorService.class);
+ functionStore = mock(LambdaFunctionStore.class);
+
poller = new SqsEventSourcePoller(
mock(Vertx.class),
- mock(SqsService.class),
- mock(LambdaExecutorService.class),
- mock(LambdaFunctionStore.class),
+ sqsService,
+ executorService,
+ functionStore,
mock(EsmStore.class),
config,
OBJECT_MAPPER
@@ -87,4 +105,105 @@ class SqsEventSourcePollerTest {
assertEquals("000000000000", attrs.get("SenderId").asText());
}
+
+ private EventSourceMapping esm() {
+ EventSourceMapping esm = new EventSourceMapping();
+ esm.setUuid("esm-uuid");
+ esm.setAccountId("000000000000");
+ esm.setRegion("us-east-1");
+ esm.setFunctionName("throwfn");
+ esm.setEventSourceArn("arn:aws:sqs:us-east-1:000000000000:esm-src");
+ esm.setQueueUrl("http://localhost:4566/000000000000/esm-src");
+ esm.setBatchSize(1);
+ return esm;
+ }
+
+ private Message message(String id) {
+ Message msg = new Message();
+ msg.setMessageId(id);
+ msg.setReceiptHandle("rh-" + id);
+ msg.setBody("body-" + id);
+ msg.setSentTimestamp(Instant.now());
+ return msg;
+ }
+
+ @Test
+ void failedInvocationReturnsMessagesToQueueUsingQueueVisibilityTimeout() {
+ EventSourceMapping esm = esm();
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("throwfn");
+ fn.setTimeout(10);
+ when(functionStore.getForAccount("000000000000", "us-east-1", "throwfn"))
+ .thenReturn(Optional.of(fn));
+
+ Message msg = message("m1");
+ when(sqsService.receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1")))
+ .thenReturn(List.of(msg));
+ when(sqsService.getQueueAttributes(eq(esm.getQueueUrl()), any(), eq("us-east-1")))
+ .thenReturn(Map.of("VisibilityTimeout", "2"));
+
+ InvokeResult failure = new InvokeResult();
+ failure.setFunctionError("Handled");
+ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(failure);
+
+ poller.pollAndInvoke(esm);
+
+ // The failed message must be made visible again after the queue's own visibility
+ // timeout (2s here) so the next poll re-receives it and the queue's RedrivePolicy
+ // can move it to the DLQ — rather than staying in-flight for fn.timeout + 30s.
+ verify(sqsService, timeout(2000)).changeMessageVisibility(
+ esm.getQueueUrl(), "rh-m1", 2, "us-east-1");
+ verify(sqsService, never()).deleteMessage(any(), any(), any());
+ }
+
+ @Test
+ void failedInvocationFallsBackToDefaultVisibilityWhenQueueHasNone() {
+ EventSourceMapping esm = esm();
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("throwfn");
+ fn.setTimeout(10);
+ when(functionStore.getForAccount("000000000000", "us-east-1", "throwfn"))
+ .thenReturn(Optional.of(fn));
+
+ Message msg = message("m1");
+ when(sqsService.receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1")))
+ .thenReturn(List.of(msg));
+ // Queue reports no VisibilityTimeout attribute.
+ when(sqsService.getQueueAttributes(eq(esm.getQueueUrl()), any(), eq("us-east-1")))
+ .thenReturn(Map.of());
+
+ InvokeResult failure = new InvokeResult();
+ failure.setFunctionError("Handled");
+ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(failure);
+
+ poller.pollAndInvoke(esm);
+
+ // Falls back to the AWS default of 30s rather than 0 (which would spin a tight retry loop).
+ verify(sqsService, timeout(2000)).changeMessageVisibility(
+ esm.getQueueUrl(), "rh-m1", 30, "us-east-1");
+ }
+
+ @Test
+ void successfulInvocationDeletesMessagesAndDoesNotResetVisibility() {
+ EventSourceMapping esm = esm();
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("throwfn");
+ fn.setTimeout(10);
+ when(functionStore.getForAccount("000000000000", "us-east-1", "throwfn"))
+ .thenReturn(Optional.of(fn));
+
+ Message msg = message("m1");
+ when(sqsService.receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1")))
+ .thenReturn(List.of(msg));
+
+ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult());
+
+ poller.pollAndInvoke(esm);
+
+ verify(sqsService, timeout(2000)).deleteMessage(esm.getQueueUrl(), "rh-m1", "us-east-1");
+ verify(sqsService, never()).changeMessageVisibility(any(), any(), anyInt(), any());
+ }
}