Light, fluffy, and always free - The AWS Local Emulator alternative
feat(pipes): add filtering, input templates, batch sizes, and DLQ routing (#576)
Christine Nguyeneed72e6222dc
diff --git a/compatibility-tests/sdk-test-awscli/test/pipes.bats b/compatibility-tests/sdk-test-awscli/test/pipes.batsindex de40fd4599..65846870f8 100644--- a/compatibility-tests/sdk-test-awscli/test/pipes.bats+++ b/compatibility-tests/sdk-test-awscli/test/pipes.bats@@ -185,6 +185,96 @@ target_arn() {[ "$found" = "true" ]}+@test "Pipes: FilterCriteria filters messages" {+ create_queues+ aws_cmd pipes create-pipe \+ --name "$PIPE_NAME" \+ --source "$(source_arn)" \+ --target "$(target_arn)" \+ --role-arn "$ROLE_ARN" \+ --desired-state RUNNING \+ --source-parameters '{"FilterCriteria":{"Filters":[{"Pattern":"{\"body\": {\"status\": [\"active\"]}}"}]}}' >/dev/null++ aws_cmd sqs send-message \+ --queue-url "$SOURCE_QUEUE_URL" \+ --message-body '{"status": "active", "id": "match-1"}' >/dev/null++ aws_cmd sqs send-message \+ --queue-url "$SOURCE_QUEUE_URL" \+ --message-body '{"status": "inactive", "id": "no-match"}' >/dev/null++ # Poll target queue for matching message+ local found=false+ for i in $(seq 1 15); do+ sleep 1+ out=$(aws_cmd sqs receive-message \+ --queue-url "$TARGET_QUEUE_URL" \+ --max-number-of-messages 10 \+ --wait-time-seconds 1)+ if echo "$out" | grep -q "match-1"; then+ # Verify no non-matching message arrived+ if echo "$out" | grep -q "no-match"; then+ fail "non-matching message should not be forwarded"+ fi+ found=true+ break+ fi+ done++ [ "$found" = "true" ]++ # Source queue should be drained (non-matching messages deleted per AWS behavior)+ out=$(aws_cmd sqs get-queue-attributes \+ --queue-url "$SOURCE_QUEUE_URL" \+ --attribute-names ApproximateNumberOfMessages)+ count=$(json_get "$out" '.Attributes.ApproximateNumberOfMessages')+ [ "$count" = "0" ]+}++@test "Pipes: BatchSize in SourceParameters" {+ create_queues+ aws_cmd pipes create-pipe \+ --name "$PIPE_NAME" \+ --source "$(source_arn)" \+ --target "$(target_arn)" \+ --role-arn "$ROLE_ARN" \+ --desired-state RUNNING \+ --source-parameters '{"SqsQueueParameters":{"BatchSize":1}}' >/dev/null++ for i in 1 2 3; do+ aws_cmd sqs send-message \+ --queue-url "$SOURCE_QUEUE_URL" \+ --message-body "batch-msg-$i" >/dev/null+ done++ # Poll target queue until all 3 messages arrive+ local found_count=0+ local found1=false found2=false found3=false+ for i in $(seq 1 20); do+ sleep 1+ out=$(aws_cmd sqs receive-message \+ --queue-url "$TARGET_QUEUE_URL" \+ --max-number-of-messages 10 \+ --wait-time-seconds 1)+ if [ "$found1" = "false" ] && echo "$out" | grep -q "batch-msg-1"; then+ found1=true+ fi+ if [ "$found2" = "false" ] && echo "$out" | grep -q "batch-msg-2"; then+ found2=true+ fi+ if [ "$found3" = "false" ] && echo "$out" | grep -q "batch-msg-3"; then+ found3=true+ fi+ if [ "$found1" = "true" ] && [ "$found2" = "true" ] && [ "$found3" = "true" ]; then+ break+ fi+ done++ [ "$found1" = "true" ]+ [ "$found2" = "true" ]+ [ "$found3" = "true" ]+}+@test "Pipes: stopped pipe does not forward messages" {create_queuesaws_cmd pipes create-pipe \diff --git a/compatibility-tests/sdk-test-go/tests/pipes_test.go b/compatibility-tests/sdk-test-go/tests/pipes_test.goindex 762bb481e7..60dd5ba186 100644--- a/compatibility-tests/sdk-test-go/tests/pipes_test.go+++ b/compatibility-tests/sdk-test-go/tests/pipes_test.go@@ -2,6 +2,7 @@ package testsimport ("context"+ "fmt""strings""testing""time"@@ -12,6 +13,7 @@ import ("github.com/aws/aws-sdk-go-v2/service/pipes"pipestypes "github.com/aws/aws-sdk-go-v2/service/pipes/types""github.com/aws/aws-sdk-go-v2/service/sqs"+ sqstypes "github.com/aws/aws-sdk-go-v2/service/sqs/types""github.com/stretchr/testify/assert""github.com/stretchr/testify/require")@@ -266,6 +268,140 @@ func TestPipes(t *testing.T) {assert.True(t, found, "target queue should receive forwarded message")})+ t.Run("FilterCriteriaFiltersMessages", func(t *testing.T) {+ pipeName := "go-test-filter-pipe"+ srcQueue := "go-pipe-src-filter"+ tgtQueue := "go-pipe-tgt-filter"++ srcResp, _ := sqsSvc.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String(srcQueue)})+ srcURL := aws.ToString(srcResp.QueueUrl)+ tgtResp, _ := sqsSvc.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String(tgtQueue)})+ tgtURL := aws.ToString(tgtResp.QueueUrl)++ t.Cleanup(func() {+ pipesSvc.DeletePipe(ctx, &pipes.DeletePipeInput{Name: aws.String(pipeName)})+ sqsSvc.DeleteQueue(ctx, &sqs.DeleteQueueInput{QueueUrl: aws.String(srcURL)})+ sqsSvc.DeleteQueue(ctx, &sqs.DeleteQueueInput{QueueUrl: aws.String(tgtURL)})+ })++ pipesSvc.CreatePipe(ctx, &pipes.CreatePipeInput{+ Name: aws.String(pipeName),+ Source: aws.String(sqsArn(srcQueue)),+ Target: aws.String(sqsArn(tgtQueue)),+ RoleArn: aws.String(pipesRoleArn),+ DesiredState: pipestypes.RequestedPipeStateRunning,+ SourceParameters: &pipestypes.PipeSourceParameters{+ FilterCriteria: &pipestypes.FilterCriteria{+ Filters: []pipestypes.Filter{+ {Pattern: aws.String(`{"body": {"status": ["active"]}}`)},+ },+ },+ },+ })++ sqsSvc.SendMessage(ctx, &sqs.SendMessageInput{+ QueueUrl: aws.String(srcURL),+ MessageBody: aws.String(`{"status": "active", "id": "match-1"}`),+ })+ sqsSvc.SendMessage(ctx, &sqs.SendMessageInput{+ QueueUrl: aws.String(srcURL),+ MessageBody: aws.String(`{"status": "inactive", "id": "no-match"}`),+ })++ found := false+ for i := 0; i < 15; i++ {+ time.Sleep(1 * time.Second)+ r, err := sqsSvc.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{+ QueueUrl: aws.String(tgtURL),+ MaxNumberOfMessages: 10,+ WaitTimeSeconds: 1,+ })+ if err == nil && len(r.Messages) > 0 {+ hasMatch := false+ hasNoMatch := false+ for _, m := range r.Messages {+ if strings.Contains(aws.ToString(m.Body), "match-1") {+ hasMatch = true+ }+ if strings.Contains(aws.ToString(m.Body), "no-match") {+ hasNoMatch = true+ }+ }+ if hasMatch {+ assert.False(t, hasNoMatch, "non-matching message should not be forwarded")+ found = true+ break+ }+ }+ }+ assert.True(t, found, "target queue should receive matching message")++ attrResp, err := sqsSvc.GetQueueAttributes(ctx, &sqs.GetQueueAttributesInput{+ QueueUrl: aws.String(srcURL),+ AttributeNames: []sqstypes.QueueAttributeName{sqstypes.QueueAttributeNameApproximateNumberOfMessages},+ })+ require.NoError(t, err)+ assert.Equal(t, "0", attrResp.Attributes[string(sqstypes.QueueAttributeNameApproximateNumberOfMessages)])+ })++ t.Run("BatchSizeInSourceParameters", func(t *testing.T) {+ pipeName := "go-test-batch-pipe"+ srcQueue := "go-pipe-src-batch"+ tgtQueue := "go-pipe-tgt-batch"++ srcResp, _ := sqsSvc.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String(srcQueue)})+ srcURL := aws.ToString(srcResp.QueueUrl)+ tgtResp, _ := sqsSvc.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String(tgtQueue)})+ tgtURL := aws.ToString(tgtResp.QueueUrl)++ t.Cleanup(func() {+ pipesSvc.DeletePipe(ctx, &pipes.DeletePipeInput{Name: aws.String(pipeName)})+ sqsSvc.DeleteQueue(ctx, &sqs.DeleteQueueInput{QueueUrl: aws.String(srcURL)})+ sqsSvc.DeleteQueue(ctx, &sqs.DeleteQueueInput{QueueUrl: aws.String(tgtURL)})+ })++ pipesSvc.CreatePipe(ctx, &pipes.CreatePipeInput{+ Name: aws.String(pipeName),+ Source: aws.String(sqsArn(srcQueue)),+ Target: aws.String(sqsArn(tgtQueue)),+ RoleArn: aws.String(pipesRoleArn),+ DesiredState: pipestypes.RequestedPipeStateRunning,+ SourceParameters: &pipestypes.PipeSourceParameters{+ SqsQueueParameters: &pipestypes.PipeSourceSqsQueueParameters{+ BatchSize: aws.Int32(1),+ },+ },+ })++ for i := 1; i <= 3; i++ {+ sqsSvc.SendMessage(ctx, &sqs.SendMessageInput{+ QueueUrl: aws.String(srcURL),+ MessageBody: aws.String(fmt.Sprintf("batch-msg-%d", i)),+ })+ }++ foundMessages := make(map[string]bool)+ for i := 0; i < 20 && len(foundMessages) < 3; i++ {+ time.Sleep(1 * time.Second)+ r, err := sqsSvc.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{+ QueueUrl: aws.String(tgtURL),+ MaxNumberOfMessages: 10,+ WaitTimeSeconds: 1,+ })+ if err == nil {+ for _, msg := range r.Messages {+ for j := 1; j <= 3; j++ {+ key := fmt.Sprintf("batch-msg-%d", j)+ if strings.Contains(aws.ToString(msg.Body), key) {+ foundMessages[key] = true+ }+ }+ }+ }+ }+ assert.Len(t, foundMessages, 3, "all 3 messages should arrive at target")+ })+t.Run("StoppedPipeDoesNotForward", func(t *testing.T) {pipeName := "go-test-nofwd-pipe"srcQueue := "go-pipe-src-nofwd"diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.javaindex be6970d243..9f340226ae 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.java@@ -186,6 +186,127 @@ class PipesTest {@Test@Order(11)+ void filterCriteriaFiltersMessages() throws InterruptedException {+ String filterPipeName = TestFixtures.uniqueName("pipe-filter");+ String filterSrc = TestFixtures.uniqueName("pipe-filter-src");+ String filterTgt = TestFixtures.uniqueName("pipe-filter-tgt");+ String filterSrcUrl = sqs.createQueue(CreateQueueRequest.builder()+ .queueName(filterSrc).build()).queueUrl();+ String filterTgtUrl = sqs.createQueue(CreateQueueRequest.builder()+ .queueName(filterTgt).build()).queueUrl();++ try {+ pipes.createPipe(CreatePipeRequest.builder()+ .name(filterPipeName)+ .source(sqsArn(filterSrc))+ .target(sqsArn(filterTgt))+ .roleArn(ROLE_ARN)+ .desiredState(RequestedPipeState.RUNNING)+ .sourceParameters(PipeSourceParameters.builder()+ .filterCriteria(FilterCriteria.builder()+ .filters(Filter.builder()+ .pattern("{\"body\": {\"status\": [\"active\"]}}")+ .build())+ .build())+ .build())+ .build());++ sqs.sendMessage(SendMessageRequest.builder()+ .queueUrl(filterSrcUrl)+ .messageBody("{\"status\": \"active\", \"id\": \"match-1\"}")+ .build());+ sqs.sendMessage(SendMessageRequest.builder()+ .queueUrl(filterSrcUrl)+ .messageBody("{\"status\": \"inactive\", \"id\": \"no-match\"}")+ .build());++ boolean found = false;+ for (int i = 0; i < 15; i++) {+ Thread.sleep(1000);+ ReceiveMessageResponse recv = sqs.receiveMessage(ReceiveMessageRequest.builder()+ .queueUrl(filterTgtUrl)+ .maxNumberOfMessages(10)+ .waitTimeSeconds(1)+ .build());+ if (recv.messages().stream().anyMatch(m -> m.body().contains("match-1"))) {+ assertThat(recv.messages().stream().noneMatch(m -> m.body().contains("no-match")))+ .as("non-matching message should not be forwarded").isTrue();+ found = true;+ break;+ }+ }+ assertThat(found).as("target queue should receive matching message").isTrue();++ GetQueueAttributesResponse attrs = sqs.getQueueAttributes(GetQueueAttributesRequest.builder()+ .queueUrl(filterSrcUrl)+ .attributeNames(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES)+ .build());+ assertThat(attrs.attributes().get(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES)).isEqualTo("0");+ } finally {+ try { pipes.deletePipe(DeletePipeRequest.builder().name(filterPipeName).build()); } catch (Exception ignored) {}+ try { sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(filterSrcUrl).build()); } catch (Exception ignored) {}+ try { sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(filterTgtUrl).build()); } catch (Exception ignored) {}+ }+ }++ @Test+ @Order(12)+ void batchSizeInSourceParameters() throws InterruptedException {+ String batchPipeName = TestFixtures.uniqueName("pipe-batch");+ String batchSrc = TestFixtures.uniqueName("pipe-batch-src");+ String batchTgt = TestFixtures.uniqueName("pipe-batch-tgt");+ String batchSrcUrl = sqs.createQueue(CreateQueueRequest.builder()+ .queueName(batchSrc).build()).queueUrl();+ String batchTgtUrl = sqs.createQueue(CreateQueueRequest.builder()+ .queueName(batchTgt).build()).queueUrl();++ try {+ pipes.createPipe(CreatePipeRequest.builder()+ .name(batchPipeName)+ .source(sqsArn(batchSrc))+ .target(sqsArn(batchTgt))+ .roleArn(ROLE_ARN)+ .desiredState(RequestedPipeState.RUNNING)+ .sourceParameters(PipeSourceParameters.builder()+ .sqsQueueParameters(PipeSourceSqsQueueParameters.builder()+ .batchSize(1)+ .build())+ .build())+ .build());++ for (int i = 1; i <= 3; i++) {+ sqs.sendMessage(SendMessageRequest.builder()+ .queueUrl(batchSrcUrl)+ .messageBody("batch-msg-" + i)+ .build());+ }++ java.util.Set<String> foundMessages = new java.util.HashSet<>();+ for (int i = 0; i < 20 && foundMessages.size() < 3; i++) {+ Thread.sleep(1000);+ ReceiveMessageResponse recv = sqs.receiveMessage(ReceiveMessageRequest.builder()+ .queueUrl(batchTgtUrl)+ .maxNumberOfMessages(10)+ .waitTimeSeconds(1)+ .build());+ for (Message msg : recv.messages()) {+ for (int j = 1; j <= 3; j++) {+ if (msg.body().contains("batch-msg-" + j)) {+ foundMessages.add("batch-msg-" + j);+ }+ }+ }+ }+ assertThat(foundMessages).hasSize(3);+ } finally {+ try { pipes.deletePipe(DeletePipeRequest.builder().name(batchPipeName).build()); } catch (Exception ignored) {}+ try { sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(batchSrcUrl).build()); } catch (Exception ignored) {}+ try { sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(batchTgtUrl).build()); } catch (Exception ignored) {}+ }+ }++ @Test+ @Order(13)void stoppedPipeDoesNotForward() throws InterruptedException {String nfPipeName = TestFixtures.uniqueName("pipe-nofwd");String nfSrc = TestFixtures.uniqueName("pipe-nofwd-src");diff --git a/compatibility-tests/sdk-test-node/tests/pipes.test.ts b/compatibility-tests/sdk-test-node/tests/pipes.test.tsindex 3269db8c57..0178f7c7b8 100644--- a/compatibility-tests/sdk-test-node/tests/pipes.test.ts+++ b/compatibility-tests/sdk-test-node/tests/pipes.test.ts@@ -315,6 +315,139 @@ describe('Pipes Polling', () => {}});+ it('should filter messages with FilterCriteria', async () => {+ const pipeName = `node-pipe-filter-${uniqueName()}`;+ const srcQueue = `node-pipe-src-filter-${uniqueName()}`;+ const tgtQueue = `node-pipe-tgt-filter-${uniqueName()}`;++ const srcResp = await sqs.send(new CreateQueueCommand({ QueueName: srcQueue }));+ const srcUrl = srcResp.QueueUrl!;+ const tgtResp = await sqs.send(new CreateQueueCommand({ QueueName: tgtQueue }));+ const tgtUrl = tgtResp.QueueUrl!;++ try {+ await pipes.send(+ new CreatePipeCommand({+ Name: pipeName,+ Source: sqsArn(srcQueue),+ Target: sqsArn(tgtQueue),+ RoleArn: ROLE_ARN,+ DesiredState: RequestedPipeState.RUNNING,+ SourceParameters: {+ FilterCriteria: {+ Filters: [+ { Pattern: '{"body": {"status": ["active"]}}' },+ ],+ },+ },+ })+ );++ await sqs.send(+ new SendMessageCommand({+ QueueUrl: srcUrl,+ MessageBody: JSON.stringify({ status: 'active', id: 'match-1' }),+ })+ );+ await sqs.send(+ new SendMessageCommand({+ QueueUrl: srcUrl,+ MessageBody: JSON.stringify({ status: 'inactive', id: 'no-match' }),+ })+ );++ let found = false;+ for (let i = 0; i < 15; i++) {+ await sleep(1000);+ const r = await sqs.send(+ new ReceiveMessageCommand({+ QueueUrl: tgtUrl,+ MaxNumberOfMessages: 10,+ WaitTimeSeconds: 1,+ })+ );+ if (r.Messages?.some((m) => m.Body?.includes('match-1'))) {+ const hasNonMatch = r.Messages.some((m) => m.Body?.includes('no-match'));+ expect(hasNonMatch).toBe(false);+ found = true;+ break;+ }+ }+ expect(found).toBe(true);++ // Source queue should be drained (non-matching messages deleted per AWS behavior)+ const attrs = await sqs.send(+ new GetQueueAttributesCommand({+ QueueUrl: srcUrl,+ AttributeNames: ['ApproximateNumberOfMessages'],+ })+ );+ expect(attrs.Attributes?.ApproximateNumberOfMessages).toBe('0');+ } finally {+ await pipes.send(new DeletePipeCommand({ Name: pipeName })).catch(() => {});+ await sqs.send(new DeleteQueueCommand({ QueueUrl: srcUrl })).catch(() => {});+ await sqs.send(new DeleteQueueCommand({ QueueUrl: tgtUrl })).catch(() => {});+ }+ });++ it('should respect BatchSize in SourceParameters', async () => {+ const pipeName = `node-pipe-batch-${uniqueName()}`;+ const srcQueue = `node-pipe-src-batch-${uniqueName()}`;+ const tgtQueue = `node-pipe-tgt-batch-${uniqueName()}`;++ const srcResp = await sqs.send(new CreateQueueCommand({ QueueName: srcQueue }));+ const srcUrl = srcResp.QueueUrl!;+ const tgtResp = await sqs.send(new CreateQueueCommand({ QueueName: tgtQueue }));+ const tgtUrl = tgtResp.QueueUrl!;++ try {+ await pipes.send(+ new CreatePipeCommand({+ Name: pipeName,+ Source: sqsArn(srcQueue),+ Target: sqsArn(tgtQueue),+ RoleArn: ROLE_ARN,+ DesiredState: RequestedPipeState.RUNNING,+ SourceParameters: {+ SqsQueueParameters: {+ BatchSize: 1,+ },+ },+ })+ );++ for (let i = 1; i <= 3; i++) {+ await sqs.send(+ new SendMessageCommand({ QueueUrl: srcUrl, MessageBody: `batch-msg-${i}` })+ );+ }++ const foundMessages = new Set<string>();+ for (let i = 0; i < 20 && foundMessages.size < 3; i++) {+ await sleep(1000);+ const r = await sqs.send(+ new ReceiveMessageCommand({+ QueueUrl: tgtUrl,+ MaxNumberOfMessages: 10,+ WaitTimeSeconds: 1,+ })+ );+ for (const msg of r.Messages ?? []) {+ for (let j = 1; j <= 3; j++) {+ if (msg.Body?.includes(`batch-msg-${j}`)) {+ foundMessages.add(`batch-msg-${j}`);+ }+ }+ }+ }+ expect(foundMessages.size).toBe(3);+ } finally {+ await pipes.send(new DeletePipeCommand({ Name: pipeName })).catch(() => {});+ await sqs.send(new DeleteQueueCommand({ QueueUrl: srcUrl })).catch(() => {});+ await sqs.send(new DeleteQueueCommand({ QueueUrl: tgtUrl })).catch(() => {});+ }+ });+it('should not forward messages when pipe is stopped', async () => {const pipeName = `node-pipe-nofwd-${uniqueName()}`;const srcQueue = `node-pipe-src-nofwd-${uniqueName()}`;diff --git a/compatibility-tests/sdk-test-python/tests/test_pipes.py b/compatibility-tests/sdk-test-python/tests/test_pipes.pyindex a3d56f6b48..c159765781 100644--- a/compatibility-tests/sdk-test-python/tests/test_pipes.py+++ b/compatibility-tests/sdk-test-python/tests/test_pipes.py@@ -223,6 +223,111 @@ class TestPipesPolling:sqs_client.delete_queue(QueueUrl=src_url)sqs_client.delete_queue(QueueUrl=tgt_url)+ def test_filter_criteria_filters_messages(self, pipes_client, sqs_client, unique_name):+ """Test that FilterCriteria only forwards matching messages."""+ src_name = f"pipe-src-{unique_name}"+ tgt_name = f"pipe-tgt-{unique_name}"+ pipe_name = f"pipe-{unique_name}"++ src_resp = sqs_client.create_queue(QueueName=src_name)+ src_url = src_resp["QueueUrl"]+ tgt_resp = sqs_client.create_queue(QueueName=tgt_name)+ tgt_url = tgt_resp["QueueUrl"]+ try:+ pipes_client.create_pipe(+ Name=pipe_name,+ Source=sqs_arn(src_name),+ Target=sqs_arn(tgt_name),+ RoleArn=ROLE_ARN,+ DesiredState="RUNNING",+ SourceParameters={+ "FilterCriteria": {+ "Filters": [+ {"Pattern": '{"body": {"status": ["active"]}}'},+ ],+ },+ },+ )++ sqs_client.send_message(+ QueueUrl=src_url,+ MessageBody='{"status": "active", "id": "match-1"}',+ )+ sqs_client.send_message(+ QueueUrl=src_url,+ MessageBody='{"status": "inactive", "id": "no-match"}',+ )++ found = False+ for _ in range(15):+ time.sleep(1)+ resp = sqs_client.receive_message(+ QueueUrl=tgt_url, MaxNumberOfMessages=10, WaitTimeSeconds=1+ )+ msgs = resp.get("Messages", [])+ if any("match-1" in m["Body"] for m in msgs):+ assert not any("no-match" in m["Body"] for m in msgs), \+ "Non-matching message should not be forwarded"+ found = True+ break++ assert found, "Target queue should receive matching message"++ attrs = sqs_client.get_queue_attributes(+ QueueUrl=src_url, AttributeNames=["ApproximateNumberOfMessages"]+ )+ assert attrs["Attributes"]["ApproximateNumberOfMessages"] == "0"+ finally:+ pipes_client.delete_pipe(Name=pipe_name)+ sqs_client.delete_queue(QueueUrl=src_url)+ sqs_client.delete_queue(QueueUrl=tgt_url)++ def test_batch_size_in_source_parameters(self, pipes_client, sqs_client, unique_name):+ """Test that BatchSize in SourceParameters is respected."""+ src_name = f"pipe-src-{unique_name}"+ tgt_name = f"pipe-tgt-{unique_name}"+ pipe_name = f"pipe-{unique_name}"++ src_resp = sqs_client.create_queue(QueueName=src_name)+ src_url = src_resp["QueueUrl"]+ tgt_resp = sqs_client.create_queue(QueueName=tgt_name)+ tgt_url = tgt_resp["QueueUrl"]+ try:+ pipes_client.create_pipe(+ Name=pipe_name,+ Source=sqs_arn(src_name),+ Target=sqs_arn(tgt_name),+ RoleArn=ROLE_ARN,+ DesiredState="RUNNING",+ SourceParameters={+ "SqsQueueParameters": {+ "BatchSize": 1,+ },+ },+ )++ for i in range(1, 4):+ sqs_client.send_message(QueueUrl=src_url, MessageBody=f"batch-msg-{i}")++ found_messages = set()+ for _ in range(20):+ if len(found_messages) >= 3:+ break+ time.sleep(1)+ resp = sqs_client.receive_message(+ QueueUrl=tgt_url, MaxNumberOfMessages=10, WaitTimeSeconds=1+ )+ for msg in resp.get("Messages", []):+ for j in range(1, 4):+ if f"batch-msg-{j}" in msg["Body"]:+ found_messages.add(f"batch-msg-{j}")++ assert len(found_messages) == 3, "All 3 messages should arrive at target"+ finally:+ pipes_client.delete_pipe(Name=pipe_name)+ sqs_client.delete_queue(QueueUrl=src_url)+ sqs_client.delete_queue(QueueUrl=tgt_url)+def test_stopped_pipe_does_not_forward(self, pipes_client, sqs_client, unique_name):"""Test that a STOPPED pipe does not forward messages."""src_name = f"pipe-src-{unique_name}"diff --git a/compatibility-tests/sdk-test-rust/tests/pipes_test.rs b/compatibility-tests/sdk-test-rust/tests/pipes_test.rsindex d31cd29185..976990898c 100644--- a/compatibility-tests/sdk-test-rust/tests/pipes_test.rs+++ b/compatibility-tests/sdk-test-rust/tests/pipes_test.rs@@ -338,6 +338,191 @@ async fn test_pipes_sqs_to_sqs_forwarding() {assert!(found, "target queue should receive forwarded message");}+#[tokio::test(flavor = "multi_thread")]+async fn test_pipes_filter_criteria() {+ let pipes = common::pipes_client().await;+ let sqs = common::sqs_client().await;+ let src_queue = "rust-pipe-src-filter";+ let tgt_queue = "rust-pipe-tgt-filter";+ let pipe_name = "rust-pipe-filter";++ let src_resp = sqs.create_queue().queue_name(src_queue).send().await.expect("create src queue");+ let src_url = src_resp.queue_url().unwrap_or("").to_string();+ let tgt_resp = sqs.create_queue().queue_name(tgt_queue).send().await.expect("create tgt queue");+ let tgt_url = tgt_resp.queue_url().unwrap_or("").to_string();++ let _guard = common::CleanupGuard::new({+ let pipes = pipes.clone();+ let sqs = sqs.clone();+ let src_url = src_url.clone();+ let tgt_url = tgt_url.clone();+ async move {+ let _ = pipes.delete_pipe().name(pipe_name).send().await;+ let _ = sqs.delete_queue().queue_url(&src_url).send().await;+ let _ = sqs.delete_queue().queue_url(&tgt_url).send().await;+ }+ });++ pipes+ .create_pipe()+ .name(pipe_name)+ .source(sqs_arn(src_queue))+ .target(sqs_arn(tgt_queue))+ .role_arn(ROLE_ARN)+ .desired_state(aws_sdk_pipes::types::RequestedPipeState::Running)+ .source_parameters(+ aws_sdk_pipes::types::PipeSourceParameters::builder()+ .filter_criteria(+ aws_sdk_pipes::types::FilterCriteria::builder()+ .filters(+ aws_sdk_pipes::types::Filter::builder()+ .pattern(r#"{"body": {"status": ["active"]}}"#)+ .build(),+ )+ .build(),+ )+ .build(),+ )+ .send()+ .await+ .expect("create pipe");++ sqs.send_message()+ .queue_url(&src_url)+ .message_body(r#"{"status": "active", "id": "match-1"}"#)+ .send()+ .await+ .expect("send matching message");++ sqs.send_message()+ .queue_url(&src_url)+ .message_body(r#"{"status": "inactive", "id": "no-match"}"#)+ .send()+ .await+ .expect("send non-matching message");++ let mut found = false;+ for _ in 0..15 {+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;+ let recv = sqs+ .receive_message()+ .queue_url(&tgt_url)+ .max_number_of_messages(10)+ .wait_time_seconds(1)+ .send()+ .await;+ if let Ok(r) = recv {+ if let Some(msgs) = r.messages() {+ if msgs.iter().any(|m| m.body().unwrap_or("").contains("match-1")) {+ assert!(+ !msgs.iter().any(|m| m.body().unwrap_or("").contains("no-match")),+ "non-matching message should not be forwarded"+ );+ found = true;+ break;+ }+ }+ }+ }+ assert!(found, "target queue should receive matching message");++ let attrs = sqs+ .get_queue_attributes()+ .queue_url(&src_url)+ .attribute_names(aws_sdk_sqs::types::QueueAttributeName::ApproximateNumberOfMessages)+ .send()+ .await+ .expect("get queue attributes");+ assert_eq!(+ attrs.attributes().unwrap().get(&aws_sdk_sqs::types::QueueAttributeName::ApproximateNumberOfMessages).map(|s| s.as_str()),+ Some("0"),+ "source queue should be drained"+ );+}++#[tokio::test(flavor = "multi_thread")]+async fn test_pipes_batch_size() {+ let pipes = common::pipes_client().await;+ let sqs = common::sqs_client().await;+ let src_queue = "rust-pipe-src-batch";+ let tgt_queue = "rust-pipe-tgt-batch";+ let pipe_name = "rust-pipe-batch";++ let src_resp = sqs.create_queue().queue_name(src_queue).send().await.expect("create src queue");+ let src_url = src_resp.queue_url().unwrap_or("").to_string();+ let tgt_resp = sqs.create_queue().queue_name(tgt_queue).send().await.expect("create tgt queue");+ let tgt_url = tgt_resp.queue_url().unwrap_or("").to_string();++ let _guard = common::CleanupGuard::new({+ let pipes = pipes.clone();+ let sqs = sqs.clone();+ let src_url = src_url.clone();+ let tgt_url = tgt_url.clone();+ async move {+ let _ = pipes.delete_pipe().name(pipe_name).send().await;+ let _ = sqs.delete_queue().queue_url(&src_url).send().await;+ let _ = sqs.delete_queue().queue_url(&tgt_url).send().await;+ }+ });++ pipes+ .create_pipe()+ .name(pipe_name)+ .source(sqs_arn(src_queue))+ .target(sqs_arn(tgt_queue))+ .role_arn(ROLE_ARN)+ .desired_state(aws_sdk_pipes::types::RequestedPipeState::Running)+ .source_parameters(+ aws_sdk_pipes::types::PipeSourceParameters::builder()+ .sqs_queue_parameters(+ aws_sdk_pipes::types::PipeSourceSqsQueueParameters::builder()+ .batch_size(1)+ .build(),+ )+ .build(),+ )+ .send()+ .await+ .expect("create pipe");++ for i in 1..=3 {+ sqs.send_message()+ .queue_url(&src_url)+ .message_body(format!("batch-msg-{}", i))+ .send()+ .await+ .expect("send message");+ }++ let mut found_messages = std::collections::HashSet::new();+ for _ in 0..20 {+ if found_messages.len() >= 3 {+ break;+ }+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;+ let recv = sqs+ .receive_message()+ .queue_url(&tgt_url)+ .max_number_of_messages(10)+ .wait_time_seconds(1)+ .send()+ .await;+ if let Ok(r) = recv {+ if let Some(msgs) = r.messages() {+ for msg in msgs {+ for j in 1..=3 {+ let key = format!("batch-msg-{}", j);+ if msg.body().unwrap_or("").contains(&key) {+ found_messages.insert(key);+ }+ }+ }+ }+ }+ }+ assert_eq!(found_messages.len(), 3, "all 3 messages should arrive at target");+}+#[tokio::test(flavor = "multi_thread")]async fn test_pipes_stopped_pipe_does_not_forward() {let pipes = common::pipes_client().await;diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcher.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcher.javanew file mode 100644index 0000000000..f8f9467427--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcher.java@@ -0,0 +1,201 @@+package io.github.hectorvent.floci.services.pipes;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.math.BigDecimal;+import java.util.ArrayList;+import java.util.List;++@ApplicationScoped+public class PipesFilterMatcher {++ private static final Logger LOG = Logger.getLogger(PipesFilterMatcher.class);++ private final ObjectMapper objectMapper;++ @Inject+ public PipesFilterMatcher(ObjectMapper objectMapper) {+ this.objectMapper = objectMapper;+ }++ public List<JsonNode> applyFilterCriteria(List<JsonNode> records, JsonNode sourceParameters) {+ if (sourceParameters == null) {+ return records;+ }+ JsonNode filters = sourceParameters.path("FilterCriteria").path("Filters");+ if (filters.isMissingNode() || !filters.isArray() || filters.isEmpty()) {+ return records;+ }+ List<JsonNode> matched = new ArrayList<>();+ for (JsonNode record : records) {+ if (matchesAnyFilter(record, filters)) {+ matched.add(record);+ }+ }+ return matched;+ }++ private boolean matchesAnyFilter(JsonNode record, JsonNode filters) {+ for (JsonNode filter : filters) {+ JsonNode patternNode = filter.path("Pattern");+ if (patternNode.isMissingNode()) {+ continue;+ }+ String patternStr = patternNode.isTextual() ? patternNode.asText() : patternNode.toString();+ if (matchesRecord(record, patternStr)) {+ return true;+ }+ }+ return false;+ }++ boolean matchesRecord(JsonNode record, String pattern) {+ if (pattern == null || pattern.isBlank()) {+ return true;+ }+ try {+ JsonNode patternNode = objectMapper.readTree(pattern);+ return matchesNode(record, patternNode);+ } catch (Exception e) {+ LOG.warnv("Failed to parse filter pattern: {0}", e.getMessage());+ return false;+ }+ }++ private boolean matchesNode(JsonNode actual, JsonNode pattern) {+ if (!pattern.isObject()) {+ return false;+ }+ var fields = pattern.fields();+ while (fields.hasNext()) {+ var field = fields.next();+ String key = field.getKey();+ JsonNode patternValue = field.getValue();+ JsonNode actualValue = actual.path(key);++ if (patternValue.isArray()) {+ if (!matchesArrayField(patternValue, actualValue)) {+ return false;+ }+ } else if (patternValue.isObject()) {+ JsonNode resolvedActual = resolveActualForObject(actualValue);+ if (!matchesNode(resolvedActual, patternValue)) {+ return false;+ }+ } else {+ LOG.warnv("Invalid filter pattern: scalar value for key \"{0}\". " ++ "Pattern values must be arrays or objects.", key);+ return false;+ }+ }+ return true;+ }++ private JsonNode resolveActualForObject(JsonNode actualValue) {+ if (actualValue.isTextual()) {+ try {+ JsonNode parsed = objectMapper.readTree(actualValue.asText());+ if (parsed.isObject() || parsed.isArray()) {+ return parsed;+ }+ } catch (Exception ignored) {+ }+ }+ return actualValue;+ }++ private boolean matchesArrayField(JsonNode patternArray, JsonNode actualValue) {+ for (JsonNode element : patternArray) {+ if (matchesSingleElement(element, actualValue)) {+ return true;+ }+ }+ return false;+ }++ private boolean matchesSingleElement(JsonNode element, JsonNode actualValue) {+ boolean valueExists = !actualValue.isMissingNode() && !actualValue.isNull();+ String actualStr = valueExists && actualValue.isTextual() ? actualValue.asText() : null;++ if (element.isTextual()) {+ return actualStr != null && actualStr.equals(element.asText());+ }+ if (element.isNull()) {+ return !valueExists;+ }+ if (element.isNumber()) {+ return valueExists && actualValue.isNumber()+ && actualValue.decimalValue().compareTo(element.decimalValue()) == 0;+ }+ if (element.isObject()) {+ if (element.has("prefix")) {+ return actualStr != null && actualStr.startsWith(element.get("prefix").asText());+ }+ if (element.has("suffix")) {+ return actualStr != null && actualStr.endsWith(element.get("suffix").asText());+ }+ if (element.has("equals-ignore-case")) {+ return actualStr != null && actualStr.equalsIgnoreCase(element.get("equals-ignore-case").asText());+ }+ if (element.has("anything-but")) {+ JsonNode anythingBut = element.get("anything-but");+ if (anythingBut.isArray()) {+ if (!valueExists) {+ return true;+ }+ for (JsonNode v : anythingBut) {+ if (v.isTextual() && v.asText().equals(actualStr)) {+ return false;+ }+ if (v.isNumber() && actualValue.isNumber()+ && actualValue.decimalValue().compareTo(v.decimalValue()) == 0) {+ return false;+ }+ }+ return true;+ }+ if (anythingBut.isObject() && anythingBut.has("prefix")) {+ return !valueExists || (actualStr != null && !actualStr.startsWith(anythingBut.get("prefix").asText()));+ }+ if (anythingBut.isTextual()) {+ return !valueExists || (actualStr != null && !actualStr.equals(anythingBut.asText()));+ }+ }+ if (element.has("exists")) {+ boolean shouldExist = element.get("exists").asBoolean();+ return shouldExist == valueExists;+ }+ if (element.has("numeric")) {+ return matchesNumericFilter(element.get("numeric"), actualValue);+ }+ }+ return false;+ }++ private boolean matchesNumericFilter(JsonNode numericArray, JsonNode actualValue) {+ if (!actualValue.isNumber() || !numericArray.isArray()) {+ return false;+ }+ BigDecimal actual = actualValue.decimalValue();+ for (int i = 0; i + 1 < numericArray.size(); i += 2) {+ String op = numericArray.get(i).asText();+ BigDecimal operand = numericArray.get(i + 1).decimalValue();+ boolean result = switch (op) {+ case "=" -> actual.compareTo(operand) == 0;+ case ">" -> actual.compareTo(operand) > 0;+ case ">=" -> actual.compareTo(operand) >= 0;+ case "<" -> actual.compareTo(operand) < 0;+ case "<=" -> actual.compareTo(operand) <= 0;+ default -> false;+ };+ if (!result) {+ return false;+ }+ }+ return true;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.javaindex 18385ac225..e2be06c1b5 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.pipes;+import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;@@ -17,8 +18,11 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.util.ArrayList;+import java.util.HashSet;import java.util.List;import java.util.Map;+import java.util.Set;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;@@ -35,6 +39,7 @@ public class PipesPoller {private final KinesisService kinesisService;private final DynamoDbStreamService dynamoDbStreamService;private final PipesTargetInvoker targetInvoker;+ private final PipesFilterMatcher filterMatcher;private final ObjectMapper objectMapper;private final String baseUrl;private final ConcurrentHashMap<String, Long> timerIds = new ConcurrentHashMap<>();@@ -53,6 +58,7 @@ public class PipesPoller {KinesisService kinesisService,DynamoDbStreamService dynamoDbStreamService,PipesTargetInvoker targetInvoker,+ PipesFilterMatcher filterMatcher,ObjectMapper objectMapper,EmulatorConfig config) {this.vertx = vertx;@@ -60,6 +66,7 @@ public class PipesPoller {this.kinesisService = kinesisService;this.dynamoDbStreamService = dynamoDbStreamService;this.targetInvoker = targetInvoker;+ this.filterMatcher = filterMatcher;this.objectMapper = objectMapper;this.baseUrl = config.effectiveBaseUrl();}@@ -129,19 +136,54 @@ public class PipesPoller {private void pollSqs(Pipe pipe, String region) {String queueUrl = AwsArnUtils.arnToQueueUrl(pipe.getSource(), baseUrl);- List<Message> messages = sqsService.receiveMessage(queueUrl, DEFAULT_BATCH_SIZE, 30, 0, region);+ int batchSize = getBatchSize(pipe, "SqsQueueParameters");+ List<Message> messages = sqsService.receiveMessage(queueUrl, batchSize, 30, 0, region);if (messages.isEmpty()) {return;}LOG.infov("Pipe {0}: received {1} SQS message(s)", pipe.getName(), messages.size());- String eventJson = buildSqsEvent(messages, pipe);- targetInvoker.invoke(pipe, eventJson, region);++ List<ObjectNode> recordNodes = buildSqsRecordNodes(messages, pipe);+ List<JsonNode> filtered = filterMatcher.applyFilterCriteria(+ new ArrayList<>(recordNodes), pipe.getSourceParameters());++ // Build a set of messageIds that matched the filter+ Set<String> matchedMessageIds = new HashSet<>();+ for (JsonNode node : filtered) {+ if (node.has("messageId")) {+ matchedMessageIds.add(node.get("messageId").asText());+ }+ }++ // Delete non-matching messages immediately (AWS behavior: filtered-out messages are consumed)for (Message msg : messages) {- try {- sqsService.deleteMessage(queueUrl, msg.getReceiptHandle(), region);- } catch (Exception e) {- LOG.warnv("Pipe {0}: failed to delete SQS message {1}: {2}",- pipe.getName(), msg.getMessageId(), e.getMessage());+ if (!matchedMessageIds.contains(msg.getMessageId())) {+ try {+ sqsService.deleteMessage(queueUrl, msg.getReceiptHandle(), region);+ } catch (Exception e) {+ LOG.warnv("Pipe {0}: failed to delete SQS message {1}: {2}",+ pipe.getName(), msg.getMessageId(), e.getMessage());+ }+ }+ }++ if (filtered.isEmpty()) {+ return;+ }++ // Attempt delivery; only delete matching messages after successful delivery or DLQ routing+ String eventJson = wrapRecords(filtered);+ boolean delivered = invokeWithDlq(pipe, eventJson, region);+ if (delivered) {+ for (Message msg : messages) {+ if (matchedMessageIds.contains(msg.getMessageId())) {+ try {+ sqsService.deleteMessage(queueUrl, msg.getReceiptHandle(), region);+ } catch (Exception e) {+ LOG.warnv("Pipe {0}: failed to delete SQS message {1}: {2}",+ pipe.getName(), msg.getMessageId(), e.getMessage());+ }+ }}}}@@ -149,14 +191,17 @@ public class PipesPoller {private void pollKinesis(Pipe pipe, String region) {String pipeKey = pipeKey(pipe);String streamName = extractResourceName(pipe.getSource());+ int batchSize = getBatchSize(pipe, "KinesisStreamParameters");String iterator = kinesisIterators.get(pipeKey);if (iterator == null) {iterator = initKinesisIterator(streamName, region);- if (iterator == null) return;+ if (iterator == null) {+ return;+ }}try {@SuppressWarnings("unchecked")- Map<String, Object> result = kinesisService.getRecords(iterator, DEFAULT_BATCH_SIZE, region);+ Map<String, Object> result = kinesisService.getRecords(iterator, batchSize, region);String nextIterator = (String) result.get("NextShardIterator");if (nextIterator != null) {kinesisIterators.put(pipeKey, nextIterator);@@ -167,8 +212,14 @@ public class PipesPoller {return;}LOG.infov("Pipe {0}: received {1} Kinesis record(s)", pipe.getName(), records.size());- String eventJson = buildKinesisEvent(records, pipe, region);- targetInvoker.invoke(pipe, eventJson, region);+ List<ObjectNode> recordNodes = buildKinesisRecordNodes(records, pipe, region);+ List<JsonNode> filtered = filterMatcher.applyFilterCriteria(+ new ArrayList<>(recordNodes), pipe.getSourceParameters());+ if (filtered.isEmpty()) {+ return;+ }+ String eventJson = wrapRecords(filtered);+ invokeWithDlq(pipe, eventJson, region);} catch (AwsException e) {if ("ExpiredIteratorException".equals(e.getErrorCode())) {kinesisIterators.remove(pipeKey);@@ -190,13 +241,16 @@ public class PipesPoller {private void pollDynamoDbStreams(Pipe pipe, String region) {String pipeKey = pipeKey(pipe);String streamArn = pipe.getSource();+ int batchSize = getBatchSize(pipe, "DynamoDBStreamParameters");String iterator = dynamoDbIterators.get(pipeKey);if (iterator == null) {iterator = initDynamoDbIterator(streamArn);- if (iterator == null) return;+ if (iterator == null) {+ return;+ }}try {- var result = dynamoDbStreamService.getRecords(iterator, DEFAULT_BATCH_SIZE);+ var result = dynamoDbStreamService.getRecords(iterator, batchSize);String nextIterator = result.nextShardIterator();if (nextIterator != null) {dynamoDbIterators.put(pipeKey, nextIterator);@@ -206,8 +260,14 @@ public class PipesPoller {return;}LOG.infov("Pipe {0}: received {1} DynamoDB Stream record(s)", pipe.getName(), records.size());- String eventJson = buildDynamoDbEvent(records, pipe, region);- targetInvoker.invoke(pipe, eventJson, region);+ List<ObjectNode> recordNodes = buildDynamoDbRecordNodes(records, pipe, region);+ List<JsonNode> filtered = filterMatcher.applyFilterCriteria(+ new ArrayList<>(recordNodes), pipe.getSourceParameters());+ if (filtered.isEmpty()) {+ return;+ }+ String eventJson = wrapRecords(filtered);+ invokeWithDlq(pipe, eventJson, region);} catch (AwsException e) {if ("ExpiredIteratorException".equals(e.getErrorCode()) ||"TrimmedDataAccessException".equals(e.getErrorCode())) {@@ -227,72 +287,62 @@ public class PipesPoller {}}- // ──────────────────────────── Event Builders ────────────────────────────+ // ──────────────────────────── Invocation & DLQ ────────────────────────────- private String buildSqsEvent(List<Message> messages, Pipe pipe) {+ private boolean invokeWithDlq(Pipe pipe, String eventJson, String region) {try {- var records = objectMapper.createArrayNode();- for (Message msg : messages) {- ObjectNode record = objectMapper.createObjectNode();- record.put("messageId", msg.getMessageId());- record.put("receiptHandle", msg.getReceiptHandle());- record.put("body", msg.getBody());- ObjectNode attrs = record.putObject("attributes");- attrs.put("ApproximateReceiveCount", String.valueOf(msg.getReceiveCount()));- attrs.put("SentTimestamp", String.valueOf(System.currentTimeMillis()));- record.putObject("messageAttributes");- record.put("md5OfBody", msg.getMd5OfBody() != null ? msg.getMd5OfBody() : "");- record.put("eventSource", "aws:sqs");- record.put("eventSourceARN", pipe.getSource());- record.put("awsRegion", extractRegionFromArn(pipe.getSource()));- records.add(record);- }- ObjectNode root = objectMapper.createObjectNode();- root.set("Records", records);- return objectMapper.writeValueAsString(root);+ targetInvoker.invoke(pipe, eventJson, region);+ return true;} catch (Exception e) {- return "{\"Records\":[]}";+ LOG.warnv("Pipe {0}: delivery failed: {1} ({2})",+ pipe.getName(), e.getMessage(), e.getClass().getSimpleName());+ return sendToDeadLetterQueue(pipe, eventJson, region);}}- private String buildKinesisEvent(List<?> records, Pipe pipe, String region) {+ private boolean sendToDeadLetterQueue(Pipe pipe, String payload, String region) {+ String dlqArn = getDlqArn(pipe);+ if (dlqArn == null) {+ return false;+ }try {- var recordsArray = objectMapper.createArrayNode();- for (Object record : records) {- ObjectNode node = objectMapper.valueToTree(record);- ObjectNode eventRecord = objectMapper.createObjectNode();- eventRecord.put("eventSource", "aws:kinesis");- eventRecord.put("eventSourceARN", pipe.getSource());- eventRecord.put("awsRegion", region);- eventRecord.put("eventID", pipe.getSource() + ":" +- node.path("SequenceNumber").asText());- ObjectNode kinesis = eventRecord.putObject("kinesis");- kinesis.put("partitionKey", node.path("PartitionKey").asText());- kinesis.put("sequenceNumber", node.path("SequenceNumber").asText());- kinesis.put("approximateArrivalTimestamp",- node.path("ApproximateArrivalTimestamp").asDouble());- String data = node.path("Data").asText();- kinesis.put("data", data);- recordsArray.add(eventRecord);- }- ObjectNode root = objectMapper.createObjectNode();- root.set("Records", recordsArray);- return objectMapper.writeValueAsString(root);+ String queueUrl = AwsArnUtils.arnToQueueUrl(dlqArn, baseUrl);+ sqsService.sendMessage(queueUrl, payload, 0);+ LOG.infov("Pipe {0}: sent failed records to DLQ {1}", pipe.getName(), dlqArn);+ return true;} catch (Exception e) {- return "{\"Records\":[]}";+ LOG.errorv("Pipe {0}: failed to send to DLQ {1}: {2}",+ pipe.getName(), dlqArn, e.getMessage());+ return false;+ }+ }++ private String getDlqArn(Pipe pipe) {+ JsonNode sp = pipe.getSourceParameters();+ if (sp == null) {+ return null;+ }+ for (String key : List.of("SqsQueueParameters", "KinesisStreamParameters", "DynamoDBStreamParameters")) {+ JsonNode dlq = sp.path(key).path("DeadLetterConfig").path("Arn");+ if (!dlq.isMissingNode() && dlq.isTextual()) {+ return dlq.asText();+ }+ }+ return null;+ }++ private int getBatchSize(Pipe pipe, String sourceParamKey) {+ JsonNode sp = pipe.getSourceParameters();+ if (sp != null && sp.has(sourceParamKey)) {+ return sp.path(sourceParamKey).path("BatchSize").asInt(DEFAULT_BATCH_SIZE);}+ return DEFAULT_BATCH_SIZE;}- private String buildDynamoDbEvent(List<?> records, Pipe pipe, String region) {+ private String wrapRecords(List<JsonNode> records) {try {var recordsArray = objectMapper.createArrayNode();- for (Object record : records) {- ObjectNode node = objectMapper.valueToTree(record);- node.put("eventSource", "aws:dynamodb");- node.put("eventSourceARN", pipe.getSource());- node.put("awsRegion", region);- recordsArray.add(node);- }+ records.forEach(recordsArray::add);ObjectNode root = objectMapper.createObjectNode();root.set("Records", recordsArray);return objectMapper.writeValueAsString(root);@@ -301,6 +351,61 @@ public class PipesPoller {}}+ // ──────────────────────────── Record Builders ────────────────────────────++ private List<ObjectNode> buildSqsRecordNodes(List<Message> messages, Pipe pipe) {+ List<ObjectNode> nodes = new ArrayList<>();+ for (Message msg : messages) {+ ObjectNode record = objectMapper.createObjectNode();+ record.put("messageId", msg.getMessageId());+ record.put("receiptHandle", msg.getReceiptHandle());+ record.put("body", msg.getBody());+ ObjectNode attrs = record.putObject("attributes");+ attrs.put("ApproximateReceiveCount", String.valueOf(msg.getReceiveCount()));+ attrs.put("SentTimestamp", String.valueOf(System.currentTimeMillis()));+ record.putObject("messageAttributes");+ record.put("md5OfBody", msg.getMd5OfBody() != null ? msg.getMd5OfBody() : "");+ record.put("eventSource", "aws:sqs");+ record.put("eventSourceARN", pipe.getSource());+ record.put("awsRegion", extractRegionFromArn(pipe.getSource()));+ nodes.add(record);+ }+ return nodes;+ }++ private List<ObjectNode> buildKinesisRecordNodes(List<?> records, Pipe pipe, String region) {+ List<ObjectNode> nodes = new ArrayList<>();+ for (Object record : records) {+ ObjectNode node = objectMapper.valueToTree(record);+ ObjectNode eventRecord = objectMapper.createObjectNode();+ eventRecord.put("eventSource", "aws:kinesis");+ eventRecord.put("eventSourceARN", pipe.getSource());+ eventRecord.put("awsRegion", region);+ eventRecord.put("eventID", pipe.getSource() + ":" ++ node.path("SequenceNumber").asText());+ ObjectNode kinesis = eventRecord.putObject("kinesis");+ kinesis.put("partitionKey", node.path("PartitionKey").asText());+ kinesis.put("sequenceNumber", node.path("SequenceNumber").asText());+ kinesis.put("approximateArrivalTimestamp",+ node.path("ApproximateArrivalTimestamp").asDouble());+ kinesis.put("data", node.path("Data").asText());+ nodes.add(eventRecord);+ }+ return nodes;+ }++ private List<ObjectNode> buildDynamoDbRecordNodes(List<?> records, Pipe pipe, String region) {+ List<ObjectNode> nodes = new ArrayList<>();+ for (Object record : records) {+ ObjectNode node = objectMapper.valueToTree(record);+ node.put("eventSource", "aws:dynamodb");+ node.put("eventSourceARN", pipe.getSource());+ node.put("awsRegion", region);+ nodes.add(node);+ }+ return nodes;+ }+// ──────────────────────────── Utilities ────────────────────────────private static String pipeKey(Pipe pipe) {@@ -314,7 +419,9 @@ public class PipesPoller {private static String extractResourceName(String arn) {int slashIdx = arn.lastIndexOf('/');- if (slashIdx >= 0) return arn.substring(slashIdx + 1);+ if (slashIdx >= 0) {+ return arn.substring(slashIdx + 1);+ }int colonIdx = arn.lastIndexOf(':');return colonIdx >= 0 ? arn.substring(colonIdx + 1) : arn;}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesTargetInvoker.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesTargetInvoker.javaindex 9b4fd72c28..561dab202f 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesTargetInvoker.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesTargetInvoker.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.pipes;+import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;@@ -17,11 +18,14 @@ import org.jboss.logging.Logger;import java.util.List;import java.util.Map;import java.util.UUID;+import java.util.regex.Matcher;+import java.util.regex.Pattern;@ApplicationScopedpublic class PipesTargetInvoker {private static final Logger LOG = Logger.getLogger(PipesTargetInvoker.class);+ private static final Pattern TEMPLATE_PLACEHOLDER = Pattern.compile("<(\\$[^>]+)>");private final LambdaService lambdaService;private final SqsService sqsService;@@ -50,23 +54,24 @@ public class PipesTargetInvoker {public void invoke(Pipe pipe, String payload, String region) {String targetArn = pipe.getTarget();- try {- if (targetArn.contains(":lambda:") || targetArn.contains(":function:")) {- invokeLambda(targetArn, payload, region);- } else if (targetArn.contains(":sqs:")) {- invokeSqs(targetArn, payload);- } else if (targetArn.contains(":sns:")) {- invokeSns(targetArn, payload, region);- } else if (targetArn.contains(":events:")) {- invokeEventBridge(targetArn, payload, region);- } else if (targetArn.contains(":states:")) {- invokeStepFunctions(targetArn, payload, region);- } else {- LOG.warnv("Pipe {0}: unsupported target ARN type: {1}", pipe.getName(), targetArn);- }- } catch (Exception e) {- LOG.warnv("Pipe {0}: failed to deliver to target {1}: {2}",- pipe.getName(), targetArn, e.getMessage());++ JsonNode tp = pipe.getTargetParameters();+ if (tp != null && tp.has("InputTemplate")) {+ payload = applyInputTemplate(tp.get("InputTemplate").asText(), payload);+ }++ if (targetArn.contains(":lambda:") || targetArn.contains(":function:")) {+ invokeLambda(targetArn, payload, region);+ } else if (targetArn.contains(":sqs:")) {+ invokeSqs(targetArn, payload);+ } else if (targetArn.contains(":sns:")) {+ invokeSns(targetArn, payload, region);+ } else if (targetArn.contains(":events:")) {+ invokeEventBridge(targetArn, payload, region);+ } else if (targetArn.contains(":states:")) {+ invokeStepFunctions(targetArn, payload, region);+ } else {+ LOG.warnv("Pipe {0}: unsupported target ARN type: {1}", pipe.getName(), targetArn);}}@@ -109,6 +114,41 @@ public class PipesTargetInvoker {LOG.debugv("Pipe delivered to Step Functions: {0}", arn);}+ String applyInputTemplate(String template, String payload) {+ Matcher m = TEMPLATE_PLACEHOLDER.matcher(template);+ StringBuilder sb = new StringBuilder();+ while (m.find()) {+ String jsonPath = m.group(1);+ String value = extractJsonPath(jsonPath, payload);+ m.appendReplacement(sb, Matcher.quoteReplacement(value != null ? value : ""));+ }+ m.appendTail(sb);+ return sb.toString();+ }++ String extractJsonPath(String jsonPath, String json) {+ if (jsonPath == null || json == null) {+ return null;+ }+ try {+ String path = jsonPath.startsWith("$") ? jsonPath.substring(1) : jsonPath;+ // Translate array indices [N] to /N for Jackson JSON Pointer+ path = path.replaceAll("\\[(\\d+)]", "/$1");+ String pointer = path.replace('.', '/');+ JsonNode node = objectMapper.readTree(json).at(pointer);+ if (node.isMissingNode() || node.isNull()) {+ return null;+ }+ if (node.isValueNode()) {+ return objectMapper.writeValueAsString(node);+ }+ return node.toString();+ } catch (Exception e) {+ LOG.warnv("Failed to extract JSONPath {0}: {1}", jsonPath, e.getMessage());+ return null;+ }+ }+private static String extractRegionFromArn(String arn, String defaultRegion) {String[] parts = arn.split(":");return parts.length >= 4 && !parts[3].isEmpty() ? parts[3] : defaultRegion;diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.javanew file mode 100644index 0000000000..77cb1ad97e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.java@@ -0,0 +1,247 @@+package io.github.hectorvent.floci.services.pipes;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;++import static org.junit.jupiter.api.Assertions.*;++class PipesFilterMatcherTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();+ private PipesFilterMatcher matcher;++ @BeforeEach+ void setUp() {+ matcher = new PipesFilterMatcher(MAPPER);+ }++ // ──────────────────────────── applyFilterCriteria ────────────────────────────++ @Test+ void noFilterCriteria_returnsAllRecords() throws Exception {+ List<JsonNode> records = List.of(+ MAPPER.readTree("{\"body\": \"hello\"}"),+ MAPPER.readTree("{\"body\": \"world\"}")+ );+ List<JsonNode> result = matcher.applyFilterCriteria(records, null);+ assertEquals(2, result.size());+ }++ @Test+ void emptyFiltersArray_returnsAllRecords() throws Exception {+ List<JsonNode> records = List.of(MAPPER.readTree("{\"body\": \"hello\"}"));+ JsonNode sp = MAPPER.readTree("{\"FilterCriteria\": {\"Filters\": []}}");+ List<JsonNode> result = matcher.applyFilterCriteria(records, sp);+ assertEquals(1, result.size());+ }++ @Test+ void multipleFilters_orSemantics() throws Exception {+ List<JsonNode> records = List.of(+ MAPPER.readTree("{\"eventSource\": \"aws:sqs\"}"),+ MAPPER.readTree("{\"eventSource\": \"aws:kinesis\"}"),+ MAPPER.readTree("{\"eventSource\": \"aws:dynamodb\"}")+ );+ JsonNode sp = MAPPER.readTree("""+ {"FilterCriteria": {"Filters": [+ {"Pattern": "{\\"eventSource\\": [\\"aws:sqs\\"]}"},+ {"Pattern": "{\\"eventSource\\": [\\"aws:kinesis\\"]}"}+ ]}}""");+ List<JsonNode> result = matcher.applyFilterCriteria(records, sp);+ assertEquals(2, result.size());+ assertEquals("aws:sqs", result.get(0).get("eventSource").asText());+ assertEquals("aws:kinesis", result.get(1).get("eventSource").asText());+ }++ // ──────────────────────────── matchesRecord ────────────────────────────++ @Test+ void exactStringMatch() throws Exception {+ JsonNode record = MAPPER.readTree("{\"eventSource\": \"aws:sqs\", \"body\": \"hello\"}");+ assertTrue(matcher.matchesRecord(record, "{\"eventSource\": [\"aws:sqs\"]}"));+ assertFalse(matcher.matchesRecord(record, "{\"eventSource\": [\"aws:kinesis\"]}"));+ }++ @Test+ void nullPatternMatchesEverything() throws Exception {+ JsonNode record = MAPPER.readTree("{\"body\": \"hello\"}");+ assertTrue(matcher.matchesRecord(record, null));+ assertTrue(matcher.matchesRecord(record, ""));+ }++ @Test+ void allFieldsMustMatch_andSemantics() throws Exception {+ JsonNode record = MAPPER.readTree("{\"eventSource\": \"aws:sqs\", \"awsRegion\": \"us-east-1\"}");+ assertTrue(matcher.matchesRecord(record,+ "{\"eventSource\": [\"aws:sqs\"], \"awsRegion\": [\"us-east-1\"]}"));+ assertFalse(matcher.matchesRecord(record,+ "{\"eventSource\": [\"aws:sqs\"], \"awsRegion\": [\"eu-west-1\"]}"));+ }++ @Test+ void nestedJsonBodyMatch_parsesStringField() throws Exception {+ ObjectNode record = MAPPER.createObjectNode();+ record.put("messageId", "msg-1");+ record.put("body", "{\"status\": \"active\", \"count\": 5}");+ record.put("eventSource", "aws:sqs");++ assertTrue(matcher.matchesRecord(record, "{\"body\": {\"status\": [\"active\"]}}"));+ assertFalse(matcher.matchesRecord(record, "{\"body\": {\"status\": [\"inactive\"]}}"));+ }++ @Test+ void nestedObjectMatch_recursesDirectly() throws Exception {+ JsonNode record = MAPPER.readTree("{\"detail\": {\"status\": \"active\", \"type\": \"order\"}}");+ assertTrue(matcher.matchesRecord(record, "{\"detail\": {\"status\": [\"active\"]}}"));+ assertFalse(matcher.matchesRecord(record, "{\"detail\": {\"status\": [\"inactive\"]}}"));+ }++ // ──────────────────────────── Operators ────────────────────────────++ @Test+ void prefixOperator() throws Exception {+ JsonNode record = MAPPER.readTree("{\"body\": \"order-12345\"}");+ assertTrue(matcher.matchesRecord(record, "{\"body\": [{\"prefix\": \"order-\"}]}"));+ assertFalse(matcher.matchesRecord(record, "{\"body\": [{\"prefix\": \"invoice-\"}]}"));+ }++ @Test+ void suffixOperator() throws Exception {+ JsonNode record = MAPPER.readTree("{\"body\": \"report.json\"}");+ assertTrue(matcher.matchesRecord(record, "{\"body\": [{\"suffix\": \".json\"}]}"));+ assertFalse(matcher.matchesRecord(record, "{\"body\": [{\"suffix\": \".xml\"}]}"));+ }++ @Test+ void equalsIgnoreCase() throws Exception {+ JsonNode record = MAPPER.readTree("{\"status\": \"Active\"}");+ assertTrue(matcher.matchesRecord(record, "{\"status\": [{\"equals-ignore-case\": \"active\"}]}"));+ assertTrue(matcher.matchesRecord(record, "{\"status\": [{\"equals-ignore-case\": \"ACTIVE\"}]}"));+ assertFalse(matcher.matchesRecord(record, "{\"status\": [{\"equals-ignore-case\": \"inactive\"}]}"));+ }++ @Test+ void anythingBut_array() throws Exception {+ JsonNode record = MAPPER.readTree("{\"eventSource\": \"aws:sqs\"}");+ assertTrue(matcher.matchesRecord(record,+ "{\"eventSource\": [{\"anything-but\": [\"aws:kinesis\", \"aws:dynamodb\"]}]}"));+ assertFalse(matcher.matchesRecord(record,+ "{\"eventSource\": [{\"anything-but\": [\"aws:sqs\", \"aws:kinesis\"]}]}"));+ }++ @Test+ void anythingBut_string() throws Exception {+ JsonNode record = MAPPER.readTree("{\"eventSource\": \"aws:sqs\"}");+ assertTrue(matcher.matchesRecord(record,+ "{\"eventSource\": [{\"anything-but\": \"aws:kinesis\"}]}"));+ assertFalse(matcher.matchesRecord(record,+ "{\"eventSource\": [{\"anything-but\": \"aws:sqs\"}]}"));+ }++ @Test+ void anythingButPrefix() throws Exception {+ JsonNode record = MAPPER.readTree("{\"eventSource\": \"custom:source\"}");+ assertTrue(matcher.matchesRecord(record,+ "{\"eventSource\": [{\"anything-but\": {\"prefix\": \"aws:\"}}]}"));++ JsonNode awsRecord = MAPPER.readTree("{\"eventSource\": \"aws:sqs\"}");+ assertFalse(matcher.matchesRecord(awsRecord,+ "{\"eventSource\": [{\"anything-but\": {\"prefix\": \"aws:\"}}]}"));+ }++ @Test+ void existsTrue_matchesWhenPresent() throws Exception {+ JsonNode record = MAPPER.readTree("{\"body\": \"hello\"}");+ assertTrue(matcher.matchesRecord(record, "{\"body\": [{\"exists\": true}]}"));+ assertFalse(matcher.matchesRecord(record, "{\"missingField\": [{\"exists\": true}]}"));+ }++ @Test+ void existsFalse_matchesWhenAbsent() throws Exception {+ JsonNode record = MAPPER.readTree("{\"body\": \"hello\"}");+ assertTrue(matcher.matchesRecord(record, "{\"missingField\": [{\"exists\": false}]}"));+ assertFalse(matcher.matchesRecord(record, "{\"body\": [{\"exists\": false}]}"));+ }++ @Test+ void nullMatch() throws Exception {+ JsonNode record = MAPPER.readTree("{\"body\": null}");+ assertTrue(matcher.matchesRecord(record, "{\"body\": [null]}"));++ JsonNode nonNull = MAPPER.readTree("{\"body\": \"hello\"}");+ assertFalse(matcher.matchesRecord(nonNull, "{\"body\": [null]}"));+ }++ @Test+ void numericExactMatch() throws Exception {+ JsonNode record = MAPPER.readTree("{\"count\": 42}");+ assertTrue(matcher.matchesRecord(record, "{\"count\": [42]}"));+ assertFalse(matcher.matchesRecord(record, "{\"count\": [99]}"));+ }++ @Test+ void numericRangeFilter() throws Exception {+ JsonNode record = MAPPER.readTree("{\"price\": 50}");+ assertTrue(matcher.matchesRecord(record, "{\"price\": [{\"numeric\": [\">\", 10, \"<\", 100]}]}"));+ assertFalse(matcher.matchesRecord(record, "{\"price\": [{\"numeric\": [\">\", 100]}]}"));+ }++ @Test+ void multipleValuesInArray_orWithinField() throws Exception {+ JsonNode record = MAPPER.readTree("{\"status\": \"pending\"}");+ assertTrue(matcher.matchesRecord(record, "{\"status\": [\"active\", \"pending\"]}"));+ assertFalse(matcher.matchesRecord(record, "{\"status\": [\"active\", \"completed\"]}"));+ }++ // ──────────────────────────── Blocker coverage ────────────────────────────++ @Test+ void scalarPatternValue_doesNotMatch() throws Exception {+ JsonNode record = MAPPER.readTree("{\"status\": \"active\"}");+ assertFalse(matcher.matchesRecord(record, "{\"status\": \"active\"}"));+ }++ @Test+ void anythingBut_array_matchesMissingField() throws Exception {+ JsonNode record = MAPPER.readTree("{\"other\": \"value\"}");+ assertTrue(matcher.matchesRecord(record,+ "{\"eventSource\": [{\"anything-but\": [\"aws:sqs\"]}]}"));+ }++ @Test+ void anythingBut_string_matchesMissingField() throws Exception {+ JsonNode record = MAPPER.readTree("{\"other\": \"value\"}");+ assertTrue(matcher.matchesRecord(record,+ "{\"eventSource\": [{\"anything-but\": \"aws:sqs\"}]}"));+ }++ @Test+ void anythingButPrefix_matchesMissingField() throws Exception {+ JsonNode record = MAPPER.readTree("{\"other\": \"value\"}");+ assertTrue(matcher.matchesRecord(record,+ "{\"eventSource\": [{\"anything-but\": {\"prefix\": \"aws:\"}}]}"));+ }++ @Test+ void existsTrue_doesNotMatchNullValue() throws Exception {+ JsonNode record = MAPPER.readTree("{\"body\": null}");+ assertFalse(matcher.matchesRecord(record, "{\"body\": [{\"exists\": true}]}"));+ }++ @Test+ void prefixAgainstNonString_doesNotMatch() throws Exception {+ JsonNode record = MAPPER.readTree("{\"count\": 42}");+ assertFalse(matcher.matchesRecord(record, "{\"count\": [{\"prefix\": \"4\"}]}"));+ }++ @Test+ void numericFilter_withNonNumber_doesNotMatch() throws Exception {+ JsonNode record = MAPPER.readTree("{\"count\": \"not-a-number\"}");+ assertFalse(matcher.matchesRecord(record, "{\"count\": [{\"numeric\": [\"=\", 42]}]}"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.javaindex 78178e4240..cfc62b3fc2 100644--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerIntegrationTest.java@@ -5,6 +5,7 @@ import org.junit.jupiter.api.*;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.*;+import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTest@@ -166,4 +167,456 @@ class PipesPollerIntegrationTest {.then().statusCode(200);}++ // ──────────────────────────── FilterCriteria Tests ────────────────────────────++ @Test+ @Order(10)+ void createFilterSourceQueue() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", "pipe-filter-source")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(11)+ void createFilterTargetQueue() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", "pipe-filter-target")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(12)+ void createPipeWithFilterCriteria() {+ given()+ .contentType("application/json")+ .body("""+ {+ "Source": "arn:aws:sqs:us-east-1:000000000000:pipe-filter-source",+ "Target": "arn:aws:sqs:us-east-1:000000000000:pipe-filter-target",+ "RoleArn": "arn:aws:iam::000000000000:role/pipe-role",+ "DesiredState": "RUNNING",+ "SourceParameters": {+ "FilterCriteria": {+ "Filters": [+ {"Pattern": "{\\"body\\": {\\"status\\": [\\"active\\"]}}"}+ ]+ }+ }+ }+ """)+ .when()+ .post("/v1/pipes/filter-pipe")+ .then()+ .statusCode(200)+ .body("CurrentState", equalTo("RUNNING"));+ }++ @Test+ @Order(13)+ void sendMatchingAndNonMatchingMessages() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "SendMessage")+ .formParam("QueueUrl", "http://localhost:4566/000000000000/pipe-filter-source")+ .formParam("MessageBody", "{\"status\": \"active\", \"id\": \"match-1\"}")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "SendMessage")+ .formParam("QueueUrl", "http://localhost:4566/000000000000/pipe-filter-source")+ .formParam("MessageBody", "{\"status\": \"inactive\", \"id\": \"no-match\"}")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(14)+ void onlyMatchingMessageForwardedToTarget() throws Exception {+ String body = null;+ for (int i = 0; i < 10; i++) {+ Thread.sleep(500);+ body = given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", "http://localhost:4566/000000000000/pipe-filter-target")+ .formParam("MaxNumberOfMessages", "10")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().body().asString();++ if (body.contains("match-1")) {+ break;+ }+ }+ assertTrue(body.contains("match-1"),+ "Target should contain the matching message");+ org.junit.jupiter.api.Assertions.assertFalse(body.contains("no-match"),+ "Target should NOT contain the non-matching message");+ }++ @Test+ @Order(15)+ void nonMatchingMessageDeletedFromSource() throws Exception {+ for (int i = 0; i < 10; i++) {+ Thread.sleep(500);+ String body = given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "GetQueueAttributes")+ .formParam("QueueUrl", "http://localhost:4566/000000000000/pipe-filter-source")+ .formParam("AttributeName.1", "ApproximateNumberOfMessages")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().body().asString();++ if (body.contains("<Value>0</Value>")) {+ return;+ }+ }+ org.junit.jupiter.api.Assertions.fail(+ "Source queue should be drained (non-matching messages deleted per AWS behavior)");+ }++ @Test+ @Order(16)+ void cleanupFilterPipe() {+ given()+ .contentType("application/json")+ .when()+ .delete("/v1/pipes/filter-pipe")+ .then()+ .statusCode(200);+ }++ // ──────────────────────────── Batch Size Tests ────────────────────────────++ @Test+ @Order(20)+ void createBatchSourceQueue() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", "pipe-batch-source")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(21)+ void createBatchTargetQueue() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", "pipe-batch-target")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(22)+ void createPipeWithBatchSize() {+ given()+ .contentType("application/json")+ .body("""+ {+ "Source": "arn:aws:sqs:us-east-1:000000000000:pipe-batch-source",+ "Target": "arn:aws:sqs:us-east-1:000000000000:pipe-batch-target",+ "RoleArn": "arn:aws:iam::000000000000:role/pipe-role",+ "DesiredState": "RUNNING",+ "SourceParameters": {+ "SqsQueueParameters": {+ "BatchSize": 1+ }+ }+ }+ """)+ .when()+ .post("/v1/pipes/batch-pipe")+ .then()+ .statusCode(200)+ .body("CurrentState", equalTo("RUNNING"));+ }++ @Test+ @Order(23)+ void sendMultipleMessagesForBatchTest() {+ for (int i = 1; i <= 3; i++) {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "SendMessage")+ .formParam("QueueUrl", "http://localhost:4566/000000000000/pipe-batch-source")+ .formParam("MessageBody", "batch-msg-" + i)+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }+ }++ @Test+ @Order(24)+ void allBatchMessagesEventuallyForwarded() throws Exception {+ java.util.Set<String> found = new java.util.HashSet<>();+ for (int i = 0; i < 20 && found.size() < 3; i++) {+ Thread.sleep(500);+ String body = given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", "http://localhost:4566/000000000000/pipe-batch-target")+ .formParam("MaxNumberOfMessages", "10")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().body().asString();++ for (int j = 1; j <= 3; j++) {+ if (body.contains("batch-msg-" + j)) found.add("batch-msg-" + j);+ }+ }+ assertEquals(3, found.size(),+ "All 3 batch messages should eventually reach the target, found: " + found);+ }++ @Test+ @Order(25)+ void cleanupBatchPipe() {+ given()+ .contentType("application/json")+ .when()+ .delete("/v1/pipes/batch-pipe")+ .then()+ .statusCode(200);+ }++ // ──────────────────────────── InputTemplate Tests ────────────────────────────++ @Test+ @Order(30)+ void createInputTemplateSourceQueue() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", "pipe-tmpl-source")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(31)+ void createInputTemplateTargetQueue() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", "pipe-tmpl-target")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(32)+ void createPipeWithInputTemplate() {+ given()+ .contentType("application/json")+ .body("""+ {+ "Source": "arn:aws:sqs:us-east-1:000000000000:pipe-tmpl-source",+ "Target": "arn:aws:sqs:us-east-1:000000000000:pipe-tmpl-target",+ "RoleArn": "arn:aws:iam::000000000000:role/pipe-role",+ "DesiredState": "RUNNING",+ "TargetParameters": {+ "InputTemplate": "{\\"transformed\\": <$.body>}"+ }+ }+ """)+ .when()+ .post("/v1/pipes/tmpl-pipe")+ .then()+ .statusCode(200)+ .body("CurrentState", equalTo("RUNNING"));+ }++ @Test+ @Order(33)+ void sendMessageForInputTemplate() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "SendMessage")+ .formParam("QueueUrl", "http://localhost:4566/000000000000/pipe-tmpl-source")+ .formParam("MessageBody", "template-test-payload")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(34)+ void inputTemplateTransformsPayload() throws Exception {+ String body = null;+ for (int i = 0; i < 10; i++) {+ Thread.sleep(500);+ body = given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", "http://localhost:4566/000000000000/pipe-tmpl-target")+ .formParam("MaxNumberOfMessages", "1")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().body().asString();++ if (body.contains("transformed")) {+ break;+ }+ }+ assertTrue(body.contains("transformed"),+ "Target should contain the InputTemplate-transformed message but got: " + body);+ }++ @Test+ @Order(35)+ void cleanupInputTemplatePipe() {+ given()+ .contentType("application/json")+ .when()+ .delete("/v1/pipes/tmpl-pipe")+ .then()+ .statusCode(200);+ }++ // ──────────────────────────── DLQ Routing Tests ────────────────────────────++ @Test+ @Order(40)+ void createDlqSourceQueue() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", "pipe-dlq-source")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(41)+ void createDlqQueue() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", "pipe-dlq-queue")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(42)+ void createPipeWithDlqAndBadTarget() {+ given()+ .contentType("application/json")+ .body("""+ {+ "Source": "arn:aws:sqs:us-east-1:000000000000:pipe-dlq-source",+ "Target": "arn:aws:lambda:us-east-1:000000000000:function:nonexistent-fn",+ "RoleArn": "arn:aws:iam::000000000000:role/pipe-role",+ "DesiredState": "RUNNING",+ "SourceParameters": {+ "SqsQueueParameters": {+ "DeadLetterConfig": {+ "Arn": "arn:aws:sqs:us-east-1:000000000000:pipe-dlq-queue"+ }+ }+ }+ }+ """)+ .when()+ .post("/v1/pipes/dlq-pipe")+ .then()+ .statusCode(200)+ .body("CurrentState", equalTo("RUNNING"));+ }++ @Test+ @Order(43)+ void sendMessageForDlqTest() {+ given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "SendMessage")+ .formParam("QueueUrl", "http://localhost:4566/000000000000/pipe-dlq-source")+ .formParam("MessageBody", "should-end-up-in-dlq")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(44)+ void failedDeliveryRoutesToDlq() throws Exception {+ String body = null;+ for (int i = 0; i < 15; i++) {+ Thread.sleep(500);+ body = given()+ .contentType(SQS_CONTENT_TYPE)+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", "http://localhost:4566/000000000000/pipe-dlq-queue")+ .formParam("MaxNumberOfMessages", "1")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().body().asString();++ if (body.contains("should-end-up-in-dlq")) {+ break;+ }+ }+ assertTrue(body.contains("should-end-up-in-dlq"),+ "DLQ should contain the failed message but got: " + body);+ }++ @Test+ @Order(45)+ void cleanupDlqPipe() {+ given()+ .contentType("application/json")+ .when()+ .delete("/v1/pipes/dlq-pipe")+ .then()+ .statusCode(200);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesTargetInvokerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesTargetInvokerTest.javanew file mode 100644index 0000000000..be5b8ecfba--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesTargetInvokerTest.java@@ -0,0 +1,163 @@+package io.github.hectorvent.floci.services.pipes;++import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.eventbridge.EventBridgeService;+import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.pipes.model.DesiredState;+import io.github.hectorvent.floci.services.pipes.model.Pipe;+import io.github.hectorvent.floci.services.sns.SnsService;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.extension.ExtendWith;+import org.mockito.ArgumentCaptor;+import org.mockito.Mock;+import org.mockito.junit.jupiter.MockitoExtension;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.*;+import static org.mockito.Mockito.*;++@ExtendWith(MockitoExtension.class)+class PipesTargetInvokerTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();++ @Mock private LambdaService lambdaService;+ @Mock private SqsService sqsService;+ @Mock private SnsService snsService;+ @Mock private EventBridgeService eventBridgeService;+ @Mock private StepFunctionsService stepFunctionsService;+ @Mock private EmulatorConfig config;++ private PipesTargetInvoker invoker;++ @BeforeEach+ void setUp() {+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ invoker = new PipesTargetInvoker(lambdaService, sqsService, snsService,+ eventBridgeService, stepFunctionsService, MAPPER, config);+ }++ private Pipe createPipe(String targetArn, ObjectNode targetParameters) {+ Pipe pipe = new Pipe();+ pipe.setName("test-pipe");+ pipe.setArn("arn:aws:pipes:us-east-1:000000000000:pipe/test-pipe");+ pipe.setSource("arn:aws:sqs:us-east-1:000000000000:source");+ pipe.setTarget(targetArn);+ pipe.setDesiredState(DesiredState.RUNNING);+ pipe.setTargetParameters(targetParameters);+ return pipe;+ }++ @Test+ void inputTemplate_replacesPlaceholders() {+ ObjectNode tp = MAPPER.createObjectNode();+ tp.put("InputTemplate", "{\"id\": <$.messageId>, \"content\": <$.body>}");++ Pipe pipe = createPipe("arn:aws:sqs:us-east-1:000000000000:target", tp);+ String payload = "{\"messageId\": \"msg-123\", \"body\": \"hello world\"}";++ invoker.invoke(pipe, payload, "us-east-1");++ ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);+ verify(sqsService).sendMessage(anyString(), captor.capture(), eq(0));+ String sent = captor.getValue();+ assertEquals("{\"id\": \"msg-123\", \"content\": \"hello world\"}", sent);+ }++ @Test+ void inputTemplate_missingFieldReplacesWithEmpty() {+ ObjectNode tp = MAPPER.createObjectNode();+ tp.put("InputTemplate", "{\"id\": \"<$.nonexistent>\"}");++ Pipe pipe = createPipe("arn:aws:sqs:us-east-1:000000000000:target", tp);+ String payload = "{\"messageId\": \"msg-123\"}";++ invoker.invoke(pipe, payload, "us-east-1");++ ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);+ verify(sqsService).sendMessage(anyString(), captor.capture(), eq(0));+ assertEquals("{\"id\": \"\"}", captor.getValue());+ }++ @Test+ void noInputTemplate_passesPayloadUnchanged() {+ Pipe pipe = createPipe("arn:aws:sqs:us-east-1:000000000000:target", null);+ String payload = "{\"Records\": []}";++ invoker.invoke(pipe, payload, "us-east-1");++ verify(sqsService).sendMessage(anyString(), eq(payload), eq(0));+ }++ @Test+ void invoke_throwsOnDeliveryFailure() {+ Pipe pipe = createPipe("arn:aws:lambda:us-east-1:000000000000:function:my-fn", null);+ doThrow(new RuntimeException("boom")).when(lambdaService)+ .invoke(anyString(), anyString(), any(byte[].class), any(InvocationType.class));++ assertThrows(RuntimeException.class, () ->+ invoker.invoke(pipe, "{}", "us-east-1"));+ }++ @Test+ void inputTemplate_objectValuePreservedAsJson() {+ ObjectNode tp = MAPPER.createObjectNode();+ tp.put("InputTemplate", "{\"data\": <$.nested>}");++ Pipe pipe = createPipe("arn:aws:sqs:us-east-1:000000000000:target", tp);+ String payload = "{\"nested\": {\"key\": \"value\"}}";++ invoker.invoke(pipe, payload, "us-east-1");++ ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);+ verify(sqsService).sendMessage(anyString(), captor.capture(), eq(0));+ assertEquals("{\"data\": {\"key\":\"value\"}}", captor.getValue());+ }++ // ──────────────────────────── applyInputTemplate unit ────────────────────────────++ @Test+ void applyInputTemplate_noPlaceholders_returnsTemplate() {+ String result = invoker.applyInputTemplate("static text", "{}");+ assertEquals("static text", result);+ }++ @Test+ void extractJsonPath_returnsTextValue() {+ assertEquals("\"hello\"", invoker.extractJsonPath("$.body", "{\"body\": \"hello\"}"));+ }++ @Test+ void extractJsonPath_returnsNullForMissing() {+ assertNull(invoker.extractJsonPath("$.missing", "{\"body\": \"hello\"}"));+ }++ @Test+ void extractJsonPath_arrayIndex() {+ String json = "{\"Records\": [{\"body\": \"first\"}, {\"body\": \"second\"}]}";+ assertEquals("\"first\"", invoker.extractJsonPath("$.Records[0].body", json));+ assertEquals("\"second\"", invoker.extractJsonPath("$.Records[1].body", json));+ }++ @Test+ void extractJsonPath_numericValue() {+ assertEquals("42", invoker.extractJsonPath("$.count", "{\"count\": 42}"));+ }++ @Test+ void extractJsonPath_booleanValue() {+ assertEquals("true", invoker.extractJsonPath("$.active", "{\"active\": true}"));+ }++ @Test+ void inputTemplate_numericNotQuoted() {+ String result = invoker.applyInputTemplate("{\"count\": <$.count>}", "{\"count\": 42}");+ assertEquals("{\"count\": 42}", result);+ }+}