Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(pipes): invoke non-Lambda targets per-record instead of batch envelope (#590)

Christine Nguyen02facf40f62d
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
index c82630b0a4..e1937a55e4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
@@ -19,6 +19,7 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -171,17 +172,37 @@ public class PipesPoller {
return;
}
- // Attempt delivery; only delete matching messages after successful delivery or DLQ routing
- String eventJson = wrapRecords(filtered);
- boolean delivered = invokeWithDlq(pipe, eventJson, region);
- if (delivered) {
+ if (isLambdaTarget(pipe)) {
+ String eventJson = wrapRecords(filtered);
+ boolean delivered = invokeWithDlq(pipe, eventJson, region);
+ if (delivered) {
+ for (Message msg : messages) {
+ if (matchedMessageIds.contains(msg.getMessageId())) {
+ try {
+ sqsService.deleteMessage(queueUrl, msg.getReceiptHandle(), region);
+ } catch (Exception e) {
+ LOG.warnv("Pipe {0}: failed to delete SQS message {1}: {2}",
+ pipe.getName(), msg.getMessageId(), e.getMessage());
+ }
+ }
+ }
+ }
+ } else {
+ Map<String, Message> messagesById = new HashMap<>();
for (Message msg : messages) {
- if (matchedMessageIds.contains(msg.getMessageId())) {
- try {
- sqsService.deleteMessage(queueUrl, msg.getReceiptHandle(), region);
- } catch (Exception e) {
- LOG.warnv("Pipe {0}: failed to delete SQS message {1}: {2}",
- pipe.getName(), msg.getMessageId(), e.getMessage());
+ messagesById.put(msg.getMessageId(), msg);
+ }
+ for (JsonNode record : filtered) {
+ String messageId = record.get("messageId").asText();
+ if (invokeWithDlq(pipe, record.toString(), region)) {
+ Message msg = messagesById.get(messageId);
+ if (msg != null) {
+ try {
+ sqsService.deleteMessage(queueUrl, msg.getReceiptHandle(), region);
+ } catch (Exception e) {
+ LOG.warnv("Pipe {0}: failed to delete SQS message {1}: {2}",
+ pipe.getName(), msg.getMessageId(), e.getMessage());
+ }
}
}
}
@@ -218,10 +239,10 @@ public class PipesPoller {
if (filtered.isEmpty()) {
return;
}
- String eventJson = wrapRecords(filtered);
- if (!invokeWithDlq(pipe, eventJson, region)) {
+ int failed = deliverRecords(pipe, filtered, region);
+ if (failed > 0) {
LOG.warnv("Pipe {0}: {1} Kinesis record(s) dropped — delivery and DLQ both failed",
- pipe.getName(), filtered.size());
+ pipe.getName(), failed);
}
} catch (AwsException e) {
if ("ExpiredIteratorException".equals(e.getErrorCode())) {
@@ -269,10 +290,10 @@ public class PipesPoller {
if (filtered.isEmpty()) {
return;
}
- String eventJson = wrapRecords(filtered);
- if (!invokeWithDlq(pipe, eventJson, region)) {
+ int failed = deliverRecords(pipe, filtered, region);
+ if (failed > 0) {
LOG.warnv("Pipe {0}: {1} DynamoDB Stream record(s) dropped — delivery and DLQ both failed",
- pipe.getName(), filtered.size());
+ pipe.getName(), failed);
}
} catch (AwsException e) {
if ("ExpiredIteratorException".equals(e.getErrorCode()) ||
@@ -295,6 +316,19 @@ public class PipesPoller {
// ──────────────────────────── Invocation & DLQ ────────────────────────────
+ private int deliverRecords(Pipe pipe, List<JsonNode> records, String region) {
+ if (isLambdaTarget(pipe)) {
+ return invokeWithDlq(pipe, wrapRecords(records), region) ? 0 : records.size();
+ }
+ int failed = 0;
+ for (JsonNode record : records) {
+ if (!invokeWithDlq(pipe, record.toString(), region)) {
+ failed++;
+ }
+ }
+ return failed;
+ }
+
private boolean invokeWithDlq(Pipe pipe, String eventJson, String region) {
try {
targetInvoker.invoke(pipe, eventJson, region);
@@ -414,6 +448,11 @@ public class PipesPoller {
// ──────────────────────────── Utilities ────────────────────────────
+ private static boolean isLambdaTarget(Pipe pipe) {
+ String targetArn = pipe.getTarget();
+ return targetArn.contains(":lambda:") || targetArn.contains(":function:");
+ }
+
private static String pipeKey(Pipe pipe) {
return pipe.getArn();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.java
index cfc62b3fc2..03a1e65eb8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.java
@@ -91,11 +91,11 @@ class PipesPollerIntegrationTest {
.statusCode(200)
.extract().body().asString();
- if (body.contains("hello from pipes") || body.contains("Records")) {
+ if (body.contains("hello from pipes")) {
break;
}
}
- assertTrue(body.contains("hello from pipes") || body.contains("Records"),
+ assertTrue(body.contains("hello from pipes"),
"Target queue should contain forwarded message but got: " + body);
}
@@ -502,6 +502,8 @@ class PipesPollerIntegrationTest {
}
assertTrue(body.contains("transformed"),
"Target should contain the InputTemplate-transformed message but got: " + body);
+ assertTrue(body.contains("template-test-payload"),
+ "InputTemplate should resolve $.body to the original message body but got: " + body);
}
@Test