Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): honour ReportBatchItemFailures in SQS ESM (#208)
Hector Ventura55b2f292cc87
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.javaindex 7eadceb8c6..4b3dd71c4a 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java@@ -249,6 +249,10 @@ public class LambdaController {node.put("BatchSize", esm.getBatchSize());node.put("State", esm.getState());node.put("LastModified", (double) esm.getLastModified() / 1000.0);+ ArrayNode responseTypes = node.putArray("FunctionResponseTypes");+ if (esm.getFunctionResponseTypes() != null) {+ esm.getFunctionResponseTypes().forEach(responseTypes::add);+ }@SuppressWarnings("unchecked")Map<String, Object> result = objectMapper.convertValue(node, Map.class);return result;diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.javaindex 1f5b23a6c5..2201019ce2 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java@@ -23,6 +23,7 @@ import java.nio.file.Path;import java.time.Instant;import java.time.ZoneOffset;import java.time.format.DateTimeFormatter;+import java.util.ArrayList;import java.util.Base64;import java.util.List;import java.util.Map;@@ -262,6 +263,11 @@ public class LambdaService {int batchSize = toInt(request.get("BatchSize"), 10);boolean enabled = !Boolean.FALSE.equals(request.get("Enabled"));+ @SuppressWarnings("unchecked")+ List<String> functionResponseTypes = request.get("FunctionResponseTypes") instanceof List+ ? (List<String>) request.get("FunctionResponseTypes")+ : new ArrayList<>();+String queueUrl = eventSourceArn.contains(":sqs:") ? AwsArnUtils.arnToQueueUrl(eventSourceArn, config.effectiveBaseUrl()) : null;EventSourceMapping esm = new EventSourceMapping();@@ -274,6 +280,7 @@ public class LambdaService {esm.setBatchSize(batchSize);esm.setEnabled(enabled);esm.setState(enabled ? "Enabled" : "Disabled");+ esm.setFunctionResponseTypes(functionResponseTypes);esm.setLastModified(System.currentTimeMillis());esmStore.save(esm);diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.javaindex 2cce690c62..35480db0de 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java@@ -17,7 +17,9 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.util.HashSet;import java.util.List;+import java.util.Set;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;@@ -148,9 +150,13 @@ public class SqsEventSourcePoller {fn, eventJson.getBytes(), InvocationType.RequestResponse);if (result.getFunctionError() == null) {- LOG.infov("ESM {0}: Lambda succeeded, deleting {1} message(s)",- esm.getUuid(), messages.size());- for (Message msg : messages) {+ Set<String> failedIds = extractBatchItemFailures(esm, result);+ List<Message> toDelete = failedIds.isEmpty()+ ? messages+ : messages.stream().filter(m -> !failedIds.contains(m.getMessageId())).toList();+ LOG.infov("ESM {0}: Lambda succeeded, deleting {1} of {2} message(s) ({3} reported as failed)",+ esm.getUuid(), toDelete.size(), messages.size(), failedIds.size());+ for (Message msg : toDelete) {try {sqsService.deleteMessage(esm.getQueueUrl(),msg.getReceiptHandle(), esm.getRegion());@@ -172,6 +178,31 @@ public class SqsEventSourcePoller {});}+ private Set<String> extractBatchItemFailures(EventSourceMapping esm, InvokeResult result) {+ if (!esm.isReportBatchItemFailures() || result.getPayload() == null || result.getPayload().length == 0) {+ return Set.of();+ }+ try {+ var root = objectMapper.readTree(result.getPayload());+ var failures = root.get("batchItemFailures");+ if (failures == null || !failures.isArray()) {+ return Set.of();+ }+ Set<String> failedIds = new HashSet<>();+ for (var item : failures) {+ var id = item.get("itemIdentifier");+ if (id != null && !id.isNull()) {+ failedIds.add(id.asText());+ }+ }+ return failedIds;+ } catch (Exception e) {+ LOG.warnv("ESM {0}: failed to parse batchItemFailures from Lambda response: {1}",+ esm.getUuid(), e.getMessage());+ return Set.of();+ }+ }+private String buildSqsEvent(List<Message> messages, EventSourceMapping esm) {try {var records = objectMapper.createArrayNode();diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.javaindex 036f5bedbf..b42759afd1 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java@@ -3,7 +3,9 @@ package io.github.hectorvent.floci.services.lambda.model;import com.fasterxml.jackson.annotation.JsonIgnoreProperties;import io.quarkus.runtime.annotations.RegisterForReflection;+import java.util.ArrayList;import java.util.HashMap;+import java.util.List;import java.util.Map;@RegisterForReflection@@ -20,6 +22,7 @@ public class EventSourceMapping {private int batchSize = 10;private String state = "Enabled";private long lastModified;+ private List<String> functionResponseTypes = new ArrayList<>();private Map<String, String> shardSequenceNumbers = new HashMap<>();public EventSourceMapping() {@@ -55,6 +58,15 @@ public class EventSourceMapping {public long getLastModified() { return lastModified; }public void setLastModified(long lastModified) { this.lastModified = lastModified; }+ public List<String> getFunctionResponseTypes() { return functionResponseTypes; }+ public void setFunctionResponseTypes(List<String> functionResponseTypes) {+ this.functionResponseTypes = functionResponseTypes != null ? functionResponseTypes : new ArrayList<>();+ }++ public boolean isReportBatchItemFailures() {+ return functionResponseTypes != null && functionResponseTypes.contains("ReportBatchItemFailures");+ }+public Map<String, String> getShardSequenceNumbers() { return shardSequenceNumbers; }public void setShardSequenceNumbers(Map<String, String> shardSequenceNumbers) {this.shardSequenceNumbers = shardSequenceNumbers != null ? shardSequenceNumbers : new java.util.HashMap<>();diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.javaindex 1fc6f428fe..fb79b20ce9 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java@@ -93,6 +93,40 @@ class EsmIntegrationTest {@Test@Order(4)+ void createEventSourceMappingWithReportBatchItemFailures() {+ String uuid = given()+ .contentType("application/json")+ .body("""+ {+ "FunctionName": "%s",+ "EventSourceArn": "%s",+ "BatchSize": 3,+ "FunctionResponseTypes": ["ReportBatchItemFailures"]+ }+ """.formatted(FUNCTION_NAME, QUEUE_ARN))+ .when()+ .post(LAMBDA_BASE + "/event-source-mappings")+ .then()+ .statusCode(202)+ .body("UUID", notNullValue())+ .body("FunctionResponseTypes", hasItem("ReportBatchItemFailures"))+ .extract()+ .path("UUID");++ // Verify it round-trips through GET+ given()+ .when()+ .get(LAMBDA_BASE + "/event-source-mappings/" + uuid)+ .then()+ .statusCode(200)+ .body("FunctionResponseTypes", hasItem("ReportBatchItemFailures"));++ // Clean up+ given().delete(LAMBDA_BASE + "/event-source-mappings/" + uuid).then().statusCode(202);+ }++ @Test+ @Order(5)void createEventSourceMappingForNonExistentFunction() {given().contentType("application/json")@@ -109,7 +143,7 @@ class EsmIntegrationTest {}@Test- @Order(5)+ @Order(6)void createEventSourceMappingUnsupportedArn() {given().contentType("application/json")@@ -126,7 +160,7 @@ class EsmIntegrationTest {}@Test- @Order(6)+ @Order(7)void getEventSourceMapping() {given().when()@@ -140,7 +174,7 @@ class EsmIntegrationTest {}@Test- @Order(7)+ @Order(8)void listEventSourceMappings() {given().when()@@ -152,7 +186,7 @@ class EsmIntegrationTest {}@Test- @Order(8)+ @Order(9)void listEventSourceMappingsByFunction() {given().queryParam("FunctionName", FUNCTION_ARN)@@ -164,7 +198,7 @@ class EsmIntegrationTest {}@Test- @Order(9)+ @Order(10)void updateEventSourceMapping() {given().contentType("application/json")@@ -178,7 +212,7 @@ class EsmIntegrationTest {}@Test- @Order(10)+ @Order(11)void disableEventSourceMapping() {given().contentType("application/json")@@ -191,7 +225,7 @@ class EsmIntegrationTest {}@Test- @Order(11)+ @Order(12)void getEventSourceMappingNotFound() {given().when()@@ -201,7 +235,7 @@ class EsmIntegrationTest {}@Test- @Order(12)+ @Order(13)void deleteEventSourceMapping() {given().when()@@ -212,7 +246,7 @@ class EsmIntegrationTest {}@Test- @Order(13)+ @Order(14)void deleteEventSourceMappingNotFound() {given().when()