Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): fail the whole SQS batch on malformed batchItemFailures (#3169)

BingEdwardbcd422ad83db
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
index 1af8a5910b..dd27622d0d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
@@ -29,6 +29,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
/**
* Polls SQS queues on behalf of Lambda Event Source Mappings.
@@ -211,7 +212,7 @@ public class SqsEventSourcePoller implements Resettable {
if (result.getFunctionError() == null) {
// Only the delivered (matched) messages are subject to delete/return here; filtered-out
// messages were already deleted above, so a batchItemFailure id that names one is inert.
- Set<String> failedIds = extractBatchItemFailures(esm, result);
+ Set<String> failedIds = extractBatchItemFailures(esm, result, messages);
List<Message> toDelete = failedIds.isEmpty()
? matched
: matched.stream().filter(m -> !failedIds.contains(m.getMessageId())).toList();
@@ -293,31 +294,53 @@ public class SqsEventSourcePoller implements Resettable {
return DEFAULT_RETRY_VISIBILITY_SECONDS;
}
- private Set<String> extractBatchItemFailures(EventSourceMapping esm, InvokeResult result) {
+ /**
+ * Message IDs the function reported as failed via {@code ReportBatchItemFailures}, following
+ * the AWS success/failure conditions: an empty or null list, or an empty or null response, is
+ * a complete success, while invalid JSON, a non-array list, an entry without
+ * {@code itemIdentifier}, or an empty, null or unknown identifier fails the whole batch, so
+ * every received message is reported as failed and the delivered ones are returned to the
+ * queue. Identifiers are validated against the received batch, so a filtered-out message's
+ * id stays inert rather than failing the batch.
+ */
+ private Set<String> extractBatchItemFailures(EventSourceMapping esm, InvokeResult result,
+ List<Message> received) {
if (!esm.isReportBatchItemFailures() || result.getPayload() == null || result.getPayload().length == 0) {
return Set.of();
}
+ Set<String> receivedIds = received.stream().map(Message::getMessageId).collect(Collectors.toSet());
try {
- var root = objectMapper.readTree(result.getPayload());
- var failures = root.get("batchItemFailures");
- if (failures == null || !failures.isArray()) {
+ var failures = objectMapper.readTree(result.getPayload()).get("batchItemFailures");
+ if (failures == null || failures.isNull()) {
return Set.of();
}
+ if (!failures.isArray()) {
+ return failWholeBatch(esm, receivedIds, "batchItemFailures is not an array");
+ }
Set<String> failedIds = new HashSet<>();
for (var item : failures) {
var id = item.get("itemIdentifier");
- if (id != null && !id.isNull()) {
- failedIds.add(id.asText());
+ if (id == null || id.isNull() || id.asText().isEmpty()) {
+ return failWholeBatch(esm, receivedIds, "entry has a missing, null or empty itemIdentifier");
+ }
+ if (!receivedIds.contains(id.asText())) {
+ return failWholeBatch(esm, receivedIds,
+ "itemIdentifier " + id.asText() + " is not in the received batch");
}
+ failedIds.add(id.asText());
}
return failedIds;
} catch (Exception e) {
- LOG.warnv("ESM {0}: failed to parse batchItemFailures from Lambda response: {1}",
- esm.getUuid(), e.getMessage());
- return Set.of();
+ return failWholeBatch(esm, receivedIds, "response is not valid JSON: " + e.getMessage());
}
}
+ private Set<String> failWholeBatch(EventSourceMapping esm, Set<String> receivedIds, String reason) {
+ LOG.warnv("ESM {0}: malformed batchItemFailures response ({1}), failing the whole batch",
+ esm.getUuid(), reason);
+ return receivedIds;
+ }
+
String buildSqsEvent(List<Message> messages, EventSourceMapping esm) {
try {
var records = objectMapper.createArrayNode();
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java
index 7fc5c9ea2b..4ddfc5f99b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java
@@ -13,6 +13,8 @@ import io.github.hectorvent.floci.services.sqs.model.Message;
import io.vertx.core.Vertx;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import java.time.Instant;
@@ -589,4 +591,71 @@ class SqsEventSourcePollerTest {
throw new RuntimeException(e);
}
}
+
+ // ───────────────────────── ReportBatchItemFailures ─────────────────────────
+
+ /** Delivers m1 and m2 to a ReportBatchItemFailures mapping whose function returns {@code payload}. */
+ private EventSourceMapping pollReportingBatch(String payload) {
+ LambdaFunction fn = stubThrowFn();
+ EventSourceMapping esm = esm();
+ esm.setBatchSize(2);
+ esm.setFunctionResponseTypes(List.of("ReportBatchItemFailures"));
+ stubReceive(esm, List.of(message("m1"), message("m2")));
+ when(sqsService.getQueueAttributes(eq(esm.getQueueUrl()), any(), eq("us-east-1")))
+ .thenReturn(Map.of("VisibilityTimeout", "2"));
+ InvokeResult result = new InvokeResult();
+ result.setPayload(payload.getBytes());
+ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(result);
+
+ poller.pollAndInvoke(esm);
+ return esm;
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "{\"batchItemFailures\":[{\"itemIdentifier\":\"\"}]}",
+ "{\"batchItemFailures\":[{\"itemIdentifier\":null}]}",
+ "{\"batchItemFailures\":[{\"itemIdentifier\":\"unknown\"}]}",
+ "{\"batchItemFailures\":[{\"wrongKey\":\"m1\"}]}",
+ "{\"batchItemFailures\":\"invalid\"}",
+ "not json",
+ })
+ void malformedBatchItemFailuresResponseFailsWholeBatch(String payload) {
+ EventSourceMapping esm = pollReportingBatch(payload);
+
+ // AWS treats these as a complete failure: nothing is deleted, every delivered
+ // message goes back to the queue for retry/redrive.
+ verify(sqsService, timeout(2000)).changeMessageVisibility(esm.getQueueUrl(), "rh-m1", 2, "us-east-1");
+ verify(sqsService, timeout(2000)).changeMessageVisibility(esm.getQueueUrl(), "rh-m2", 2, "us-east-1");
+ awaitPollCompleted(esm);
+ verify(sqsService, never()).deleteMessage(any(), any(), any());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "{\"batchItemFailures\":[]}",
+ "{\"batchItemFailures\":null}",
+ "{}",
+ "null",
+ })
+ void emptyOrNullBatchItemFailuresResponseDeletesWholeBatch(String payload) {
+ EventSourceMapping esm = pollReportingBatch(payload);
+
+ verify(sqsService, timeout(2000)).deleteMessage(esm.getQueueUrl(), "rh-m1", "us-east-1");
+ verify(sqsService, timeout(2000)).deleteMessage(esm.getQueueUrl(), "rh-m2", "us-east-1");
+ awaitPollCompleted(esm);
+ verify(sqsService, never()).changeMessageVisibility(any(), any(), anyInt(), any());
+ }
+
+ @Test
+ void validPartialBatchFailureRetriesOnlyReportedMessage() {
+ EventSourceMapping esm = pollReportingBatch("{\"batchItemFailures\":[{\"itemIdentifier\":\"m2\"}]}");
+
+ verify(sqsService, timeout(2000)).deleteMessage(esm.getQueueUrl(), "rh-m1", "us-east-1");
+ verify(sqsService, timeout(2000)).changeMessageVisibility(esm.getQueueUrl(), "rh-m2", 2, "us-east-1");
+ awaitPollCompleted(esm);
+ verify(sqsService, never()).deleteMessage(esm.getQueueUrl(), "rh-m2", "us-east-1");
+ verify(sqsService, never()).changeMessageVisibility(eq(esm.getQueueUrl()), eq("rh-m1"), anyInt(), any());
+ }
}