Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): redrive failed SQS ESM messages to the DLQ (#1419)
Stuart Langdba23bf7e2c3
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.javaindex 3c1f424afe..6061706840 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java@@ -35,6 +35,9 @@ public class SqsEventSourcePoller {private static final Logger LOG = Logger.getLogger(SqsEventSourcePoller.class);+ /** AWS SQS default visibility timeout, used as the retry backoff when a queue has none configured. */+ private static final int DEFAULT_RETRY_VISIBILITY_SECONDS = 30;+private final Vertx vertx;private final SqsService sqsService;private final LambdaExecutorService executorService;@@ -114,7 +117,8 @@ public class SqsEventSourcePoller {}}- private void pollAndInvoke(EventSourceMapping esm) {+ /** Package-private (not private) only so unit tests can drive a single poll directly. */+ void pollAndInvoke(EventSourceMapping esm) {// Skip this tick if a previous poll for this ESM is still in progress.// This prevents concurrent deliveries of the same message when the Lambda// cold-start / execution time exceeds the SQS visibility timeout.@@ -176,9 +180,18 @@ public class SqsEventSourcePoller {msg.getMessageId(), e.getMessage());}}+ // Reported partial-batch failures are not deleted; return them to the+ // queue immediately so they can be retried/redriven rather than sitting+ // in-flight for the full execution-cover visibility window.+ if (!failedIds.isEmpty()) {+ List<Message> toReturn = messages.stream()+ .filter(m -> failedIds.contains(m.getMessageId())).toList();+ returnMessagesToQueue(esm, toReturn);+ }} else {- LOG.warnv("ESM {0}: Lambda returned error [{1}], messages will return to queue",- esm.getUuid(), result.getFunctionError());+ LOG.warnv("ESM {0}: Lambda returned error [{1}], returning {2} message(s) to queue for retry/redrive",+ esm.getUuid(), result.getFunctionError(), messages.size());+ returnMessagesToQueue(esm, messages);}} catch (Exception e) {LOG.warnv("ESM {0}: poll/invoke error: {1} ({2})",@@ -189,6 +202,51 @@ public class SqsEventSourcePoller {});}+ /**+ * Returns failed messages to the source queue by resetting their visibility timeout+ * to the queue's own {@code VisibilityTimeout}. The poller hides messages for+ * {@code fn.timeout + 30s} to cover execution time, but on failure that long window+ * would keep the message in-flight (and therefore not redelivered nor redriven) far+ * longer than the queue's own visibility/redrive policy intends. Shrinking the window+ * back to the queue's visibility timeout lets the next poll re-receive them — matching+ * AWS's redelivery cadence rather than spinning a tight retry loop (which resetting to+ * 0 would cause for a persistently failing function) — so ApproximateReceiveCount+ * climbs and the queue's RedrivePolicy moves them to the DLQ once+ * {@code maxReceiveCount} is exceeded.+ */+ private void returnMessagesToQueue(EventSourceMapping esm, List<Message> messages) {+ int retryVisibility = retryVisibilityTimeout(esm);+ for (Message msg : messages) {+ try {+ sqsService.changeMessageVisibility(+ esm.getQueueUrl(), msg.getReceiptHandle(), retryVisibility, esm.getRegion());+ } catch (Exception e) {+ LOG.warnv("ESM {0}: failed to return message {1} to queue: {2}",+ esm.getUuid(), msg.getMessageId(), e.getMessage());+ }+ }+ }++ /**+ * The visibility timeout to apply when returning a failed message to the queue: the+ * queue's configured {@code VisibilityTimeout}, or the AWS default of 30s when unset+ * or unreadable. This governs how soon the message is retried/redriven.+ */+ private int retryVisibilityTimeout(EventSourceMapping esm) {+ try {+ String vt = sqsService.getQueueAttributes(+ esm.getQueueUrl(), List.of("VisibilityTimeout"), esm.getRegion())+ .get("VisibilityTimeout");+ if (vt != null) {+ return Math.max(0, Integer.parseInt(vt));+ }+ } catch (Exception e) {+ LOG.debugv("ESM {0}: could not read VisibilityTimeout, using default {1}s backoff: {2}",+ esm.getUuid(), DEFAULT_RETRY_VISIBILITY_SECONDS, e.getMessage());+ }+ return DEFAULT_RETRY_VISIBILITY_SECONDS;+ }+private Set<String> extractBatchItemFailures(EventSourceMapping esm, InvokeResult result) {if (!esm.isReportBatchItemFailures() || result.getPayload() == null || result.getPayload().length == 0) {return Set.of();diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.javaindex bf2ba9fbd5..96559e5fbe 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java@@ -4,6 +4,9 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;+import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.sqs.SqsService;import io.github.hectorvent.floci.services.sqs.model.Message;import io.vertx.core.Vertx;@@ -12,15 +15,26 @@ import org.junit.jupiter.api.Test;import java.time.Instant;import java.util.List;+import java.util.Map;+import java.util.Optional;import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyInt;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.timeout;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;class SqsEventSourcePollerTest {private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();private SqsEventSourcePoller poller;+ private SqsService sqsService;+ private LambdaExecutorService executorService;+ private LambdaFunctionStore functionStore;@BeforeEachvoid setUp() {@@ -32,11 +46,15 @@ class SqsEventSourcePollerTest {when(lambdaConfig.pollIntervalMs()).thenReturn(1000L);when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ sqsService = mock(SqsService.class);+ executorService = mock(LambdaExecutorService.class);+ functionStore = mock(LambdaFunctionStore.class);+poller = new SqsEventSourcePoller(mock(Vertx.class),- mock(SqsService.class),- mock(LambdaExecutorService.class),- mock(LambdaFunctionStore.class),+ sqsService,+ executorService,+ functionStore,mock(EsmStore.class),config,OBJECT_MAPPER@@ -87,4 +105,105 @@ class SqsEventSourcePollerTest {assertEquals("000000000000", attrs.get("SenderId").asText());}++ private EventSourceMapping esm() {+ EventSourceMapping esm = new EventSourceMapping();+ esm.setUuid("esm-uuid");+ esm.setAccountId("000000000000");+ esm.setRegion("us-east-1");+ esm.setFunctionName("throwfn");+ esm.setEventSourceArn("arn:aws:sqs:us-east-1:000000000000:esm-src");+ esm.setQueueUrl("http://localhost:4566/000000000000/esm-src");+ esm.setBatchSize(1);+ return esm;+ }++ private Message message(String id) {+ Message msg = new Message();+ msg.setMessageId(id);+ msg.setReceiptHandle("rh-" + id);+ msg.setBody("body-" + id);+ msg.setSentTimestamp(Instant.now());+ return msg;+ }++ @Test+ void failedInvocationReturnsMessagesToQueueUsingQueueVisibilityTimeout() {+ EventSourceMapping esm = esm();+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("throwfn");+ fn.setTimeout(10);+ when(functionStore.getForAccount("000000000000", "us-east-1", "throwfn"))+ .thenReturn(Optional.of(fn));++ Message msg = message("m1");+ when(sqsService.receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1")))+ .thenReturn(List.of(msg));+ when(sqsService.getQueueAttributes(eq(esm.getQueueUrl()), any(), eq("us-east-1")))+ .thenReturn(Map.of("VisibilityTimeout", "2"));++ InvokeResult failure = new InvokeResult();+ failure.setFunctionError("Handled");+ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(failure);++ poller.pollAndInvoke(esm);++ // The failed message must be made visible again after the queue's own visibility+ // timeout (2s here) so the next poll re-receives it and the queue's RedrivePolicy+ // can move it to the DLQ — rather than staying in-flight for fn.timeout + 30s.+ verify(sqsService, timeout(2000)).changeMessageVisibility(+ esm.getQueueUrl(), "rh-m1", 2, "us-east-1");+ verify(sqsService, never()).deleteMessage(any(), any(), any());+ }++ @Test+ void failedInvocationFallsBackToDefaultVisibilityWhenQueueHasNone() {+ EventSourceMapping esm = esm();+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("throwfn");+ fn.setTimeout(10);+ when(functionStore.getForAccount("000000000000", "us-east-1", "throwfn"))+ .thenReturn(Optional.of(fn));++ Message msg = message("m1");+ when(sqsService.receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1")))+ .thenReturn(List.of(msg));+ // Queue reports no VisibilityTimeout attribute.+ when(sqsService.getQueueAttributes(eq(esm.getQueueUrl()), any(), eq("us-east-1")))+ .thenReturn(Map.of());++ InvokeResult failure = new InvokeResult();+ failure.setFunctionError("Handled");+ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(failure);++ poller.pollAndInvoke(esm);++ // Falls back to the AWS default of 30s rather than 0 (which would spin a tight retry loop).+ verify(sqsService, timeout(2000)).changeMessageVisibility(+ esm.getQueueUrl(), "rh-m1", 30, "us-east-1");+ }++ @Test+ void successfulInvocationDeletesMessagesAndDoesNotResetVisibility() {+ EventSourceMapping esm = esm();+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("throwfn");+ fn.setTimeout(10);+ when(functionStore.getForAccount("000000000000", "us-east-1", "throwfn"))+ .thenReturn(Optional.of(fn));++ Message msg = message("m1");+ when(sqsService.receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1")))+ .thenReturn(List.of(msg));++ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult());++ poller.pollAndInvoke(esm);++ verify(sqsService, timeout(2000)).deleteMessage(esm.getQueueUrl(), "rh-m1", "us-east-1");+ verify(sqsService, never()).changeMessageVisibility(any(), any(), anyInt(), any());+ }}