Light, fluffy, and always free - The AWS Local Emulator alternative
fix(pipes): invoke non-Lambda targets per-record instead of batch envelope (#590)
Christine Nguyen02facf40f62d
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.javaindex c82630b0a4..e1937a55e4 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java@@ -19,6 +19,7 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.util.ArrayList;+import java.util.HashMap;import java.util.HashSet;import java.util.List;import java.util.Map;@@ -171,17 +172,37 @@ public class PipesPoller {return;}- // Attempt delivery; only delete matching messages after successful delivery or DLQ routing- String eventJson = wrapRecords(filtered);- boolean delivered = invokeWithDlq(pipe, eventJson, region);- if (delivered) {+ if (isLambdaTarget(pipe)) {+ String eventJson = wrapRecords(filtered);+ boolean delivered = invokeWithDlq(pipe, eventJson, region);+ if (delivered) {+ for (Message msg : messages) {+ if (matchedMessageIds.contains(msg.getMessageId())) {+ try {+ sqsService.deleteMessage(queueUrl, msg.getReceiptHandle(), region);+ } catch (Exception e) {+ LOG.warnv("Pipe {0}: failed to delete SQS message {1}: {2}",+ pipe.getName(), msg.getMessageId(), e.getMessage());+ }+ }+ }+ }+ } else {+ Map<String, Message> messagesById = new HashMap<>();for (Message msg : messages) {- if (matchedMessageIds.contains(msg.getMessageId())) {- try {- sqsService.deleteMessage(queueUrl, msg.getReceiptHandle(), region);- } catch (Exception e) {- LOG.warnv("Pipe {0}: failed to delete SQS message {1}: {2}",- pipe.getName(), msg.getMessageId(), e.getMessage());+ messagesById.put(msg.getMessageId(), msg);+ }+ for (JsonNode record : filtered) {+ String messageId = record.get("messageId").asText();+ if (invokeWithDlq(pipe, record.toString(), region)) {+ Message msg = messagesById.get(messageId);+ if (msg != null) {+ try {+ sqsService.deleteMessage(queueUrl, msg.getReceiptHandle(), region);+ } catch (Exception e) {+ LOG.warnv("Pipe {0}: failed to delete SQS message {1}: {2}",+ pipe.getName(), msg.getMessageId(), e.getMessage());+ }}}}@@ -218,10 +239,10 @@ public class PipesPoller {if (filtered.isEmpty()) {return;}- String eventJson = wrapRecords(filtered);- if (!invokeWithDlq(pipe, eventJson, region)) {+ int failed = deliverRecords(pipe, filtered, region);+ if (failed > 0) {LOG.warnv("Pipe {0}: {1} Kinesis record(s) dropped — delivery and DLQ both failed",- pipe.getName(), filtered.size());+ pipe.getName(), failed);}} catch (AwsException e) {if ("ExpiredIteratorException".equals(e.getErrorCode())) {@@ -269,10 +290,10 @@ public class PipesPoller {if (filtered.isEmpty()) {return;}- String eventJson = wrapRecords(filtered);- if (!invokeWithDlq(pipe, eventJson, region)) {+ int failed = deliverRecords(pipe, filtered, region);+ if (failed > 0) {LOG.warnv("Pipe {0}: {1} DynamoDB Stream record(s) dropped — delivery and DLQ both failed",- pipe.getName(), filtered.size());+ pipe.getName(), failed);}} catch (AwsException e) {if ("ExpiredIteratorException".equals(e.getErrorCode()) ||@@ -295,6 +316,19 @@ public class PipesPoller {// ──────────────────────────── Invocation & DLQ ────────────────────────────+ private int deliverRecords(Pipe pipe, List<JsonNode> records, String region) {+ if (isLambdaTarget(pipe)) {+ return invokeWithDlq(pipe, wrapRecords(records), region) ? 0 : records.size();+ }+ int failed = 0;+ for (JsonNode record : records) {+ if (!invokeWithDlq(pipe, record.toString(), region)) {+ failed++;+ }+ }+ return failed;+ }+private boolean invokeWithDlq(Pipe pipe, String eventJson, String region) {try {targetInvoker.invoke(pipe, eventJson, region);@@ -414,6 +448,11 @@ public class PipesPoller {// ──────────────────────────── Utilities ────────────────────────────+ private static boolean isLambdaTarget(Pipe pipe) {+ String targetArn = pipe.getTarget();+ return targetArn.contains(":lambda:") || targetArn.contains(":function:");+ }+private static String pipeKey(Pipe pipe) {return pipe.getArn();}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.javaindex cfc62b3fc2..03a1e65eb8 100644--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.java@@ -91,11 +91,11 @@ class PipesPollerIntegrationTest {.statusCode(200).extract().body().asString();- if (body.contains("hello from pipes") || body.contains("Records")) {+ if (body.contains("hello from pipes")) {break;}}- assertTrue(body.contains("hello from pipes") || body.contains("Records"),+ assertTrue(body.contains("hello from pipes"),"Target queue should contain forwarded message but got: " + body);}@@ -502,6 +502,8 @@ class PipesPollerIntegrationTest {}assertTrue(body.contains("transformed"),"Target should contain the InputTemplate-transformed message but got: " + body);+ assertTrue(body.contains("template-test-payload"),+ "InputTemplate should resolve $.body to the original message body but got: " + body);}@Test