Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): honour ReportBatchItemFailures in SQS ESM (#208)

Hector Ventura55b2f292cc87
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java
index 7eadceb8c6..4b3dd71c4a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java
@@ -249,6 +249,10 @@ public class LambdaController {
node.put("BatchSize", esm.getBatchSize());
node.put("State", esm.getState());
node.put("LastModified", (double) esm.getLastModified() / 1000.0);
+ ArrayNode responseTypes = node.putArray("FunctionResponseTypes");
+ if (esm.getFunctionResponseTypes() != null) {
+ esm.getFunctionResponseTypes().forEach(responseTypes::add);
+ }
@SuppressWarnings("unchecked")
Map<String, Object> result = objectMapper.convertValue(node, Map.class);
return result;
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
index 1f5b23a6c5..2201019ce2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
@@ -23,6 +23,7 @@ import java.nio.file.Path;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
@@ -262,6 +263,11 @@ public class LambdaService {
int batchSize = toInt(request.get("BatchSize"), 10);
boolean enabled = !Boolean.FALSE.equals(request.get("Enabled"));
+ @SuppressWarnings("unchecked")
+ List<String> functionResponseTypes = request.get("FunctionResponseTypes") instanceof List
+ ? (List<String>) request.get("FunctionResponseTypes")
+ : new ArrayList<>();
+
String queueUrl = eventSourceArn.contains(":sqs:") ? AwsArnUtils.arnToQueueUrl(eventSourceArn, config.effectiveBaseUrl()) : null;
EventSourceMapping esm = new EventSourceMapping();
@@ -274,6 +280,7 @@ public class LambdaService {
esm.setBatchSize(batchSize);
esm.setEnabled(enabled);
esm.setState(enabled ? "Enabled" : "Disabled");
+ esm.setFunctionResponseTypes(functionResponseTypes);
esm.setLastModified(System.currentTimeMillis());
esmStore.save(esm);
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
index 2cce690c62..35480db0de 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
@@ -17,7 +17,9 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -148,9 +150,13 @@ public class SqsEventSourcePoller {
fn, eventJson.getBytes(), InvocationType.RequestResponse);
if (result.getFunctionError() == null) {
- LOG.infov("ESM {0}: Lambda succeeded, deleting {1} message(s)",
- esm.getUuid(), messages.size());
- for (Message msg : messages) {
+ Set<String> failedIds = extractBatchItemFailures(esm, result);
+ List<Message> toDelete = failedIds.isEmpty()
+ ? messages
+ : messages.stream().filter(m -> !failedIds.contains(m.getMessageId())).toList();
+ LOG.infov("ESM {0}: Lambda succeeded, deleting {1} of {2} message(s) ({3} reported as failed)",
+ esm.getUuid(), toDelete.size(), messages.size(), failedIds.size());
+ for (Message msg : toDelete) {
try {
sqsService.deleteMessage(esm.getQueueUrl(),
msg.getReceiptHandle(), esm.getRegion());
@@ -172,6 +178,31 @@ public class SqsEventSourcePoller {
});
}
+ private Set<String> extractBatchItemFailures(EventSourceMapping esm, InvokeResult result) {
+ if (!esm.isReportBatchItemFailures() || result.getPayload() == null || result.getPayload().length == 0) {
+ return Set.of();
+ }
+ try {
+ var root = objectMapper.readTree(result.getPayload());
+ var failures = root.get("batchItemFailures");
+ if (failures == null || !failures.isArray()) {
+ return Set.of();
+ }
+ Set<String> failedIds = new HashSet<>();
+ for (var item : failures) {
+ var id = item.get("itemIdentifier");
+ if (id != null && !id.isNull()) {
+ failedIds.add(id.asText());
+ }
+ }
+ return failedIds;
+ } catch (Exception e) {
+ LOG.warnv("ESM {0}: failed to parse batchItemFailures from Lambda response: {1}",
+ esm.getUuid(), e.getMessage());
+ return Set.of();
+ }
+ }
+
private String buildSqsEvent(List<Message> messages, EventSourceMapping esm) {
try {
var records = objectMapper.createArrayNode();
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java
index 036f5bedbf..b42759afd1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java
@@ -3,7 +3,9 @@ package io.github.hectorvent.floci.services.lambda.model;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
@RegisterForReflection
@@ -20,6 +22,7 @@ public class EventSourceMapping {
private int batchSize = 10;
private String state = "Enabled";
private long lastModified;
+ private List<String> functionResponseTypes = new ArrayList<>();
private Map<String, String> shardSequenceNumbers = new HashMap<>();
public EventSourceMapping() {
@@ -55,6 +58,15 @@ public class EventSourceMapping {
public long getLastModified() { return lastModified; }
public void setLastModified(long lastModified) { this.lastModified = lastModified; }
+ public List<String> getFunctionResponseTypes() { return functionResponseTypes; }
+ public void setFunctionResponseTypes(List<String> functionResponseTypes) {
+ this.functionResponseTypes = functionResponseTypes != null ? functionResponseTypes : new ArrayList<>();
+ }
+
+ public boolean isReportBatchItemFailures() {
+ return functionResponseTypes != null && functionResponseTypes.contains("ReportBatchItemFailures");
+ }
+
public Map<String, String> getShardSequenceNumbers() { return shardSequenceNumbers; }
public void setShardSequenceNumbers(Map<String, String> shardSequenceNumbers) {
this.shardSequenceNumbers = shardSequenceNumbers != null ? shardSequenceNumbers : new java.util.HashMap<>();
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java
index 1fc6f428fe..fb79b20ce9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java
@@ -93,6 +93,40 @@ class EsmIntegrationTest {
@Test
@Order(4)
+ void createEventSourceMappingWithReportBatchItemFailures() {
+ String uuid = given()
+ .contentType("application/json")
+ .body("""
+ {
+ "FunctionName": "%s",
+ "EventSourceArn": "%s",
+ "BatchSize": 3,
+ "FunctionResponseTypes": ["ReportBatchItemFailures"]
+ }
+ """.formatted(FUNCTION_NAME, QUEUE_ARN))
+ .when()
+ .post(LAMBDA_BASE + "/event-source-mappings")
+ .then()
+ .statusCode(202)
+ .body("UUID", notNullValue())
+ .body("FunctionResponseTypes", hasItem("ReportBatchItemFailures"))
+ .extract()
+ .path("UUID");
+
+ // Verify it round-trips through GET
+ given()
+ .when()
+ .get(LAMBDA_BASE + "/event-source-mappings/" + uuid)
+ .then()
+ .statusCode(200)
+ .body("FunctionResponseTypes", hasItem("ReportBatchItemFailures"));
+
+ // Clean up
+ given().delete(LAMBDA_BASE + "/event-source-mappings/" + uuid).then().statusCode(202);
+ }
+
+ @Test
+ @Order(5)
void createEventSourceMappingForNonExistentFunction() {
given()
.contentType("application/json")
@@ -109,7 +143,7 @@ class EsmIntegrationTest {
}
@Test
- @Order(5)
+ @Order(6)
void createEventSourceMappingUnsupportedArn() {
given()
.contentType("application/json")
@@ -126,7 +160,7 @@ class EsmIntegrationTest {
}
@Test
- @Order(6)
+ @Order(7)
void getEventSourceMapping() {
given()
.when()
@@ -140,7 +174,7 @@ class EsmIntegrationTest {
}
@Test
- @Order(7)
+ @Order(8)
void listEventSourceMappings() {
given()
.when()
@@ -152,7 +186,7 @@ class EsmIntegrationTest {
}
@Test
- @Order(8)
+ @Order(9)
void listEventSourceMappingsByFunction() {
given()
.queryParam("FunctionName", FUNCTION_ARN)
@@ -164,7 +198,7 @@ class EsmIntegrationTest {
}
@Test
- @Order(9)
+ @Order(10)
void updateEventSourceMapping() {
given()
.contentType("application/json")
@@ -178,7 +212,7 @@ class EsmIntegrationTest {
}
@Test
- @Order(10)
+ @Order(11)
void disableEventSourceMapping() {
given()
.contentType("application/json")
@@ -191,7 +225,7 @@ class EsmIntegrationTest {
}
@Test
- @Order(11)
+ @Order(12)
void getEventSourceMappingNotFound() {
given()
.when()
@@ -201,7 +235,7 @@ class EsmIntegrationTest {
}
@Test
- @Order(12)
+ @Order(13)
void deleteEventSourceMapping() {
given()
.when()
@@ -212,7 +246,7 @@ class EsmIntegrationTest {
}
@Test
- @Order(13)
+ @Order(14)
void deleteEventSourceMappingNotFound() {
given()
.when()