Light, fluffy, and always free - The AWS Local Emulator alternative
feat(pipes): support Kafka pipe sources and polling (#1260)
Liam MacPherson86c4aae2e2f6
diff --git a/docs/services/pipes.md b/docs/services/pipes.mdindex f1e96ce653..9683824198 100644--- a/docs/services/pipes.md+++ b/docs/services/pipes.md@@ -81,11 +81,11 @@ Floci emulates EventBridge Pipes with the following supported source and target- Amazon SQS queues- Amazon Kinesis streams- Amazon DynamoDB streams-- Kafka topics (MSK)+- Kafka topics (MSK and self-managed via `smk://`)**Targets:**- Lambda functions- SQS queues- SNS topics- Kinesis streams-- Step Functions state machines\ No newline at end of file+- Step Functions state machinesdiff --git a/pom.xml b/pom.xmlindex b2d55e7ca7..fd296299b7 100644--- a/pom.xml+++ b/pom.xml@@ -100,6 +100,13 @@<version>9.2.1</version></dependency>+ <!-- Kafka client for MSK/self-managed Kafka pipe polling -->+ <dependency>+ <groupId>org.apache.kafka</groupId>+ <artifactId>kafka-clients</artifactId>+ <version>4.3.0</version>+ </dependency>+<!-- Docker client for Lambda container lifecycle management --><dependency>diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.javanew file mode 100644index 0000000000..a9c80cdddd--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java@@ -0,0 +1,211 @@+package io.github.hectorvent.floci.services.pipes;++import com.fasterxml.jackson.databind.JsonNode;+import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.msk.MskService;+import io.github.hectorvent.floci.services.pipes.model.Pipe;+import jakarta.annotation.PreDestroy;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.apache.kafka.clients.consumer.ConsumerConfig;+import org.apache.kafka.clients.consumer.ConsumerRecords;+import org.apache.kafka.clients.consumer.KafkaConsumer;+import org.apache.kafka.clients.consumer.OffsetAndMetadata;+import org.apache.kafka.common.TopicPartition;+import org.apache.kafka.common.serialization.ByteArrayDeserializer;+import org.jboss.logging.Logger;++import java.time.Duration;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.Properties;+import java.util.UUID;+import java.util.concurrent.ConcurrentHashMap;++@ApplicationScoped+public class PipesKafkaConsumerManager {++ private static final Logger LOG = Logger.getLogger(PipesKafkaConsumerManager.class);+ private static final Duration POLL_TIMEOUT = Duration.ofMillis(100);+ private static final String DEFAULT_STARTING_POSITION = "LATEST";++ private final MskService mskService;+ private final ConcurrentHashMap<String, KafkaConsumer<byte[], byte[]>> consumers = new ConcurrentHashMap<>();++ @Inject+ public PipesKafkaConsumerManager(MskService mskService) {+ this.mskService = mskService;+ }++ @PreDestroy+ void shutdown() {+ consumers.values().forEach(this::closeQuietly);+ consumers.clear();+ }++ public ConsumerRecords<byte[], byte[]> poll(Pipe pipe) {+ KafkaConsumer<byte[], byte[]> consumer = consumers.computeIfAbsent(pipe.getArn(), ignored -> createConsumer(pipe));+ try {+ return consumer.poll(POLL_TIMEOUT);+ } catch (RuntimeException e) {+ close(pipe);+ throw e;+ }+ }++ public void commit(Pipe pipe) {+ KafkaConsumer<byte[], byte[]> consumer = consumers.get(pipe.getArn());+ if (consumer == null) {+ return;+ }+ try {+ consumer.commitSync();+ } catch (RuntimeException e) {+ close(pipe);+ throw e;+ }+ }++ public void commit(Pipe pipe, Map<TopicPartition, OffsetAndMetadata> offsets) {+ if (offsets.isEmpty()) {+ return;+ }+ KafkaConsumer<byte[], byte[]> consumer = consumers.get(pipe.getArn());+ if (consumer == null) {+ return;+ }+ try {+ consumer.commitSync(offsets);+ } catch (RuntimeException e) {+ close(pipe);+ throw e;+ }+ }++ public void close(Pipe pipe) {+ KafkaConsumer<byte[], byte[]> consumer = consumers.remove(pipe.getArn());+ if (consumer != null) {+ closeQuietly(consumer);+ }+ }++ String resolveBootstrapServers(Pipe pipe) {+ if (isSelfManagedSource(pipe)) {+ List<String> servers = new ArrayList<>();+ servers.add(pipe.getSource().substring("smk://".length()));+ JsonNode additionalServers = kafkaParameters(pipe).path("AdditionalBootstrapServers");+ if (additionalServers.isArray()) {+ additionalServers.forEach(node -> {+ String server = node.asText(null);+ if (server != null && !server.isBlank()) {+ servers.add(server);+ }+ });+ }+ return String.join(",", servers);+ }+ if (isManagedSource(pipe)) {+ return mskService.getBootstrapBrokers(pipe.getSource());+ }+ throw new AwsException("ValidationException", "Unsupported Kafka source: " + pipe.getSource(), 400);+ }++ String resolveTopicName(Pipe pipe) {+ JsonNode params = kafkaParameters(pipe);+ String topic = params.path("TopicName").asText(null);+ if (topic == null || topic.isBlank()) {+ throw new AwsException("ValidationException",+ "SourceParameters." + parameterBlockName(pipe) + ".TopicName is required", 400);+ }+ return topic;+ }++ int resolveBatchSize(Pipe pipe, int defaultBatchSize) {+ JsonNode params = kafkaParameters(pipe);+ return params.path("BatchSize").asInt(defaultBatchSize);+ }++ String resolveConsumerGroupId(Pipe pipe) {+ return resolveConsumerGroupId(pipe, kafkaParameters(pipe));+ }++ private KafkaConsumer<byte[], byte[]> createConsumer(Pipe pipe) {+ String bootstrapServers = resolveBootstrapServers(pipe);+ String topicName = resolveTopicName(pipe);+ JsonNode params = kafkaParameters(pipe);++ Properties properties = new Properties();+ properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);+ properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());+ properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());+ properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");+ properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, resolveOffsetReset(params));+ properties.put(ConsumerConfig.GROUP_ID_CONFIG, resolveConsumerGroupId(pipe, params));+ properties.put(ConsumerConfig.CLIENT_ID_CONFIG, "floci-pipes-" + UUID.randomUUID());+ properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, Integer.toString(resolveBatchSize(pipe, 100)));++ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(properties);+ consumer.subscribe(List.of(topicName));+ LOG.infov("Pipe {0}: subscribed Kafka consumer to topic {1} via {2}",+ pipe.getName(), topicName, bootstrapServers);+ return consumer;+ }++ private JsonNode kafkaParameters(Pipe pipe) {+ JsonNode sourceParameters = pipe.getSourceParameters();+ if (sourceParameters == null) {+ throw new AwsException("ValidationException", "Kafka pipe SourceParameters are required", 400);+ }+ JsonNode parameters = sourceParameters.path(parameterBlockName(pipe));+ if (parameters.isMissingNode()) {+ throw new AwsException("ValidationException",+ "SourceParameters." + parameterBlockName(pipe) + " is required", 400);+ }+ return parameters;+ }++ private String resolveOffsetReset(JsonNode params) {+ String startingPosition = params.path("StartingPosition").asText(DEFAULT_STARTING_POSITION);+ return "TRIM_HORIZON".equalsIgnoreCase(startingPosition) ? "earliest" : "latest";+ }++ private String resolveConsumerGroupId(Pipe pipe, JsonNode params) {+ String configured = params.path("ConsumerGroupID").asText(null);+ if (configured != null && !configured.isBlank()) {+ return configured;+ }++ String sourceFingerprint = isSelfManagedSource(pipe)+ ? pipe.getSource().substring("smk://".length())+ : AwsArnUtils.parse(pipe.getSource()).resource();+ return "floci-pipes-" + pipe.getName() + "-" + Integer.toUnsignedString(sourceFingerprint.hashCode());+ }++ private static boolean isManagedSource(Pipe pipe) {+ return pipe.getSource().contains(":kafka:");+ }++ private static boolean isSelfManagedSource(Pipe pipe) {+ return pipe.getSource().startsWith("smk://");+ }++ private static String parameterBlockName(Pipe pipe) {+ if (isManagedSource(pipe)) {+ return "ManagedStreamingKafkaParameters";+ }+ if (isSelfManagedSource(pipe)) {+ return "SelfManagedKafkaParameters";+ }+ throw new AwsException("ValidationException", "Unsupported Kafka source: " + pipe.getSource(), 400);+ }++ private void closeQuietly(KafkaConsumer<byte[], byte[]> consumer) {+ try {+ consumer.close();+ } catch (Exception e) {+ LOG.debugv("Ignoring Kafka consumer close error: {0}", e.getMessage());+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.javaindex 225c22983a..e93a139e03 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.pipes;+import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;@@ -17,8 +18,14 @@ import io.vertx.core.Vertx;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import org.apache.kafka.clients.consumer.ConsumerRecord;+import org.apache.kafka.clients.consumer.ConsumerRecords;+import org.apache.kafka.clients.consumer.OffsetAndMetadata;+import org.apache.kafka.common.TopicPartition;+import org.apache.kafka.common.header.Header;import org.jboss.logging.Logger;+import java.nio.charset.StandardCharsets;import java.util.ArrayList;import java.util.Base64;import java.util.HashMap;@@ -41,6 +48,7 @@ public class PipesPoller {private final SqsService sqsService;private final KinesisService kinesisService;private final DynamoDbStreamService dynamoDbStreamService;+ private final PipesKafkaConsumerManager kafkaConsumerManager;private final PipesTargetInvoker targetInvoker;private final PipesFilterMatcher filterMatcher;private final ObjectMapper objectMapper;@@ -60,6 +68,7 @@ public class PipesPoller {SqsService sqsService,KinesisService kinesisService,DynamoDbStreamService dynamoDbStreamService,+ PipesKafkaConsumerManager kafkaConsumerManager,PipesTargetInvoker targetInvoker,PipesFilterMatcher filterMatcher,ObjectMapper objectMapper,@@ -68,6 +77,7 @@ public class PipesPoller {this.sqsService = sqsService;this.kinesisService = kinesisService;this.dynamoDbStreamService = dynamoDbStreamService;+ this.kafkaConsumerManager = kafkaConsumerManager;this.targetInvoker = targetInvoker;this.filterMatcher = filterMatcher;this.objectMapper = objectMapper;@@ -99,6 +109,7 @@ public class PipesPoller {vertx.cancelTimer(timerId);kinesisIterators.remove(pipeKey);dynamoDbIterators.remove(pipeKey);+ kafkaConsumerManager.close(pipe);LOG.infov("Pipe {0}: stopped polling", pipe.getName());}}@@ -125,6 +136,8 @@ public class PipesPoller {pollKinesis(pipe, region);} else if (sourceArn.contains(":dynamodb:")) {pollDynamoDbStreams(pipe, region);+ } else if (isKafkaSource(sourceArn)) {+ pollKafka(pipe, region);} else {LOG.warnv("Pipe {0}: unsupported source type: {1}", pipe.getName(), sourceArn);}@@ -310,6 +323,155 @@ public class PipesPoller {}}+ void pollKafka(Pipe pipe, String region) {+ ConsumerRecords<byte[], byte[]> records = kafkaConsumerManager.poll(pipe);+ if (records.isEmpty()) {+ return;+ }++ List<ConsumerRecord<byte[], byte[]>> batch = new ArrayList<>(records.count());+ records.forEach(batch::add);++ LOG.infov("Pipe {0}: received {1} Kafka record(s)", pipe.getName(), batch.size());+ List<ObjectNode> deliveryNodes = buildKafkaRecordNodes(batch, pipe);+ List<ObjectNode> filterNodes = buildKafkaFilterNodes(deliveryNodes, batch);+ List<JsonNode> filtered = filterMatcher.applyFilterCriteria(new ArrayList<>(filterNodes), pipe.getSourceParameters());+ if (filtered.isEmpty()) {+ kafkaConsumerManager.commit(pipe);+ return;+ }++ Map<String, JsonNode> deliveryRecordsByIdentity = new HashMap<>();+ for (JsonNode record : deliveryNodes) {+ deliveryRecordsByIdentity.put(kafkaRecordIdentity(record), record);+ }++ List<JsonNode> deliveryRecords = new ArrayList<>(filtered.size());+ for (JsonNode record : filtered) {+ JsonNode deliveryRecord = deliveryRecordsByIdentity.get(kafkaRecordIdentity(record));+ if (deliveryRecord != null) {+ deliveryRecords.add(deliveryRecord);+ }+ }+ if (deliveryRecords.isEmpty()) {+ kafkaConsumerManager.commit(pipe);+ return;+ }++ int failed = isLambdaTarget(pipe)+ ? deliverKafkaLambdaRecords(pipe, region, batch, deliveryRecordsByIdentity, filtered)+ : deliverKafkaRecords(pipe, region, batch, deliveryRecordsByIdentity, filtered);+ if (failed == 0) {+ return;+ }++ LOG.warnv("Pipe {0}: {1} Kafka record(s) not committed because delivery failed",+ pipe.getName(), failed);+ }++ private int deliverKafkaLambdaRecords(Pipe pipe,+ String region,+ List<ConsumerRecord<byte[], byte[]>> batch,+ Map<String, JsonNode> deliveryRecordsByIdentity,+ List<JsonNode> filtered) {+ Set<String> matchedIdentities = new HashSet<>(filtered.size());+ filtered.forEach(record -> matchedIdentities.add(kafkaRecordIdentity(record)));++ Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = groupKafkaRecordsByPartition(batch);+ int failed = 0;++ for (Map.Entry<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> entry : recordsByPartition.entrySet()) {+ failed += deliverKafkaLambdaPartition(pipe, region, entry.getKey(), entry.getValue(),+ deliveryRecordsByIdentity, matchedIdentities, offsetsToCommit);+ }++ if (!offsetsToCommit.isEmpty()) {+ kafkaConsumerManager.commit(pipe, offsetsToCommit);+ }+ return failed;+ }++ private int deliverKafkaLambdaPartition(Pipe pipe,+ String region,+ TopicPartition partition,+ List<ConsumerRecord<byte[], byte[]>> partitionRecords,+ Map<String, JsonNode> deliveryRecordsByIdentity,+ Set<String> matchedIdentities,+ Map<TopicPartition, OffsetAndMetadata> offsetsToCommit) {+ List<JsonNode> pendingBatch = new ArrayList<>();+ long pendingOffset = -1L;++ for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {+ String identity = kafkaRecordIdentity(record);+ if (!matchedIdentities.contains(identity)) {+ if (!pendingBatch.isEmpty()) {+ if (!invokeWithDlq(pipe, wrapRecords(pendingBatch), region)) {+ return pendingBatch.size();+ }+ offsetsToCommit.put(partition, new OffsetAndMetadata(pendingOffset));+ pendingBatch.clear();+ }+ offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));+ continue;+ }++ JsonNode deliveryRecord = deliveryRecordsByIdentity.get(identity);+ if (deliveryRecord != null) {+ pendingBatch.add(deliveryRecord);+ pendingOffset = record.offset() + 1;+ }+ }++ if (!pendingBatch.isEmpty()) {+ if (!invokeWithDlq(pipe, wrapRecords(pendingBatch), region)) {+ return pendingBatch.size();+ }+ offsetsToCommit.put(partition, new OffsetAndMetadata(pendingOffset));+ }+ return 0;+ }++ private int deliverKafkaRecords(Pipe pipe,+ String region,+ List<ConsumerRecord<byte[], byte[]>> batch,+ Map<String, JsonNode> deliveryRecordsByIdentity,+ List<JsonNode> filtered) {+ Set<String> matchedIdentities = new HashSet<>(filtered.size());+ filtered.forEach(record -> matchedIdentities.add(kafkaRecordIdentity(record)));++ Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();+ Set<TopicPartition> blockedPartitions = new HashSet<>();+ int failed = 0;++ for (ConsumerRecord<byte[], byte[]> record : batch) {+ TopicPartition partition = new TopicPartition(record.topic(), record.partition());+ if (blockedPartitions.contains(partition)) {+ continue;+ }++ String identity = kafkaRecordIdentity(record);+ if (!matchedIdentities.contains(identity)) {+ offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));+ continue;+ }++ JsonNode deliveryRecord = deliveryRecordsByIdentity.get(identity);+ if (deliveryRecord != null && invokeWithDlq(pipe, deliveryRecord.toString(), region)) {+ offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));+ continue;+ }++ blockedPartitions.add(partition);+ failed++;+ }++ if (!offsetsToCommit.isEmpty()) {+ kafkaConsumerManager.commit(pipe, offsetsToCommit);+ }+ return failed;+ }+private String initDynamoDbIterator(String streamArn) {try {return dynamoDbStreamService.getShardIterator(@@ -469,6 +631,58 @@ public class PipesPoller {return nodes;}+ private List<ObjectNode> buildKafkaRecordNodes(List<ConsumerRecord<byte[], byte[]>> records, Pipe pipe) {+ List<ObjectNode> nodes = new ArrayList<>();+ String eventSource = pipe.getSource().contains(":kafka:") ? "aws:kafka" : "SelfManagedKafka";+ String bootstrapServers = kafkaConsumerManager.resolveBootstrapServers(pipe);+ for (ConsumerRecord<byte[], byte[]> record : records) {+ ObjectNode node = objectMapper.createObjectNode();+ node.put("eventSource", eventSource);+ if (pipe.getSource().contains(":kafka:")) {+ node.put("eventSourceArn", pipe.getSource());+ } else {+ node.put("bootstrapServers", bootstrapServers);+ }+ node.put("topic", record.topic());+ node.put("partition", record.partition());+ node.put("offset", record.offset());+ node.put("timestamp", record.timestamp());+ node.put("timestampType", record.timestampType().name());+ putKafkaBinaryField(node, "key", record.key());+ putKafkaBinaryField(node, "value", record.value());++ var headersNode = node.putArray("headers");+ for (Header header : record.headers()) {+ ObjectNode headerNode = objectMapper.createObjectNode();+ byte[] headerValue = header.value();+ if (headerValue == null) {+ headerNode.putNull(header.key());+ } else {+ var values = headerNode.putArray(header.key());+ for (byte b : headerValue) {+ values.add(b & 0xFF);+ }+ }+ headersNode.add(headerNode);+ }+ nodes.add(node);+ }+ return nodes;+ }++ private List<ObjectNode> buildKafkaFilterNodes(List<ObjectNode> deliveryNodes,+ List<ConsumerRecord<byte[], byte[]>> records) {+ List<ObjectNode> nodes = new ArrayList<>(deliveryNodes.size());+ for (int i = 0; i < deliveryNodes.size(); i++) {+ ObjectNode node = deliveryNodes.get(i).deepCopy();+ ConsumerRecord<byte[], byte[]> record = records.get(i);+ applyDecodedKafkaField(node, "key", record.key());+ applyDecodedKafkaField(node, "value", record.value());+ nodes.add(node);+ }+ return nodes;+ }+// ──────────────────────────── Utilities ────────────────────────────private static boolean isLambdaTarget(Pipe pipe) {@@ -476,6 +690,57 @@ public class PipesPoller {return targetArn.contains(":lambda:") || targetArn.contains(":function:");}+ private static boolean isKafkaSource(String sourceArn) {+ return sourceArn.startsWith("smk://") || sourceArn.contains(":kafka:");+ }++ private static String base64(byte[] value) {+ return Base64.getEncoder().encodeToString(value);+ }++ private static String kafkaRecordIdentity(JsonNode record) {+ return record.path("topic").asText() + ":"+ + record.path("partition").asText() + ":"+ + record.path("offset").asText();+ }++ private static String kafkaRecordIdentity(ConsumerRecord<byte[], byte[]> record) {+ return record.topic() + ":" + record.partition() + ":" + record.offset();+ }++ private void applyDecodedKafkaField(ObjectNode node, String fieldName, byte[] value) {+ if (value == null) {+ node.putNull(fieldName);+ return;+ }+ String decoded = new String(value, StandardCharsets.UTF_8);+ try {+ JsonNode parsed = objectMapper.readTree(decoded);+ node.set(fieldName, parsed);+ } catch (JsonProcessingException e) {+ LOG.debugv("Kafka {0} is not valid JSON: {1}", fieldName, e.getOriginalMessage());+ node.put(fieldName, decoded);+ }+ }++ private void putKafkaBinaryField(ObjectNode node, String fieldName, byte[] value) {+ if (value == null) {+ node.putNull(fieldName);+ return;+ }+ node.put(fieldName, base64(value));+ }++ private Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> groupKafkaRecordsByPartition(+ List<ConsumerRecord<byte[], byte[]>> records) {+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = new HashMap<>();+ for (ConsumerRecord<byte[], byte[]> record : records) {+ TopicPartition partition = new TopicPartition(record.topic(), record.partition());+ recordsByPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);+ }+ return recordsByPartition;+ }+private static String pipeKey(Pipe pipe) {return pipe.getArn();}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.javaindex 366b66a602..ea212b32b6 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java@@ -72,6 +72,8 @@ public class PipesService implements TagHandler {throw new AwsException("ValidationException", "RoleArn is required", 400);}+ validateSourceConfiguration(source, sourceParameters);+String key = region + "::" + name;if (storage.get(key).isPresent()) {throw new AwsException("ConflictException",@@ -125,6 +127,8 @@ public class PipesService implements TagHandler {.orElseThrow(() -> new AwsException("NotFoundException","Pipe " + name + " does not exist.", 404));+ validateSourceConfiguration(pipe.getSource(), sourceParameters != null ? sourceParameters : pipe.getSourceParameters());+if (target != null) pipe.setTarget(target);if (roleArn != null) pipe.setRoleArn(roleArn);if (description != null) pipe.setDescription(description);@@ -240,4 +244,29 @@ public class PipesService implements TagHandler {.orElseThrow(() -> new AwsException("NotFoundException","Resource " + arn + " does not exist.", 404));}++ private void validateSourceConfiguration(String source, JsonNode sourceParameters) {+ if (source == null) {+ return;+ }+ if (source.startsWith("smk://")) {+ requireKafkaParameters(sourceParameters, "SelfManagedKafkaParameters");+ return;+ }+ if (source.contains(":kafka:")) {+ requireKafkaParameters(sourceParameters, "ManagedStreamingKafkaParameters");+ }+ }++ private void requireKafkaParameters(JsonNode sourceParameters, String parameterBlock) {+ if (sourceParameters == null || sourceParameters.path(parameterBlock).isMissingNode()) {+ throw new AwsException("ValidationException",+ "SourceParameters." + parameterBlock + " is required", 400);+ }+ String topicName = sourceParameters.path(parameterBlock).path("TopicName").asText(null);+ if (topicName == null || topicName.isBlank()) {+ throw new AwsException("ValidationException",+ "SourceParameters." + parameterBlock + ".TopicName is required", 400);+ }+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.javaindex 77cb1ad97e..9fff5ee76b 100644--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.java@@ -101,6 +101,28 @@ class PipesFilterMatcherTest {assertFalse(matcher.matchesRecord(record, "{\"detail\": {\"status\": [\"inactive\"]}}"));}+ @Test+ void kafkaDecodedValueCanBeFilteredAsJson() throws Exception {+ JsonNode record = MAPPER.readTree("""+ {+ "eventSource": "aws:kafka",+ "topic": "orders",+ "value": {"status": "active", "id": "order-1"}+ }+ """);++ assertTrue(matcher.matchesRecord(record, "{\"value\": {\"status\": [\"active\"]}}"));+ assertFalse(matcher.matchesRecord(record, "{\"value\": {\"status\": [\"inactive\"]}}"));+ }++ @Test+ void kafkaDecodedKeyCanBeFilteredAsString() throws Exception {+ JsonNode record = MAPPER.readTree("{\"key\": \"customer-123\", \"topic\": \"orders\"}");++ assertTrue(matcher.matchesRecord(record, "{\"key\": [{\"prefix\": \"customer-\"}]}"));+ assertFalse(matcher.matchesRecord(record, "{\"key\": [\"customer-999\"]}"));+ }+// ──────────────────────────── Operators ────────────────────────────@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.javanew file mode 100644index 0000000000..125a87775f--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java@@ -0,0 +1,164 @@+package io.github.hectorvent.floci.services.pipes;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.services.msk.MskService;+import io.github.hectorvent.floci.services.pipes.model.Pipe;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.mockito.Mockito;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.Mockito.when;++class PipesKafkaConsumerManagerTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();++ private MskService mskService;+ private PipesKafkaConsumerManager manager;++ @BeforeEach+ void setUp() {+ mskService = Mockito.mock(MskService.class);+ manager = new PipesKafkaConsumerManager(mskService);+ }++ @Test+ void resolveBootstrapServersForSelfManagedKafkaIncludesAdditionalServers() throws Exception {+ Pipe pipe = new Pipe();+ pipe.setSource("smk://broker-1:9092");+ pipe.setSourceParameters(MAPPER.readTree("""+ {+ "SelfManagedKafkaParameters": {+ "TopicName": "orders",+ "AdditionalBootstrapServers": ["broker-2:9092", "broker-3:9092"]+ }+ }+ """));++ assertEquals("broker-1:9092,broker-2:9092,broker-3:9092", manager.resolveBootstrapServers(pipe));+ }++ @Test+ void resolveBootstrapServersForManagedKafkaUsesMskService() throws Exception {+ Pipe pipe = new Pipe();+ pipe.setSource("arn:aws:kafka:us-east-1:000000000000:cluster/demo/uuid");+ pipe.setSourceParameters(MAPPER.readTree("""+ {+ "ManagedStreamingKafkaParameters": {+ "TopicName": "orders"+ }+ }+ """));+ when(mskService.getBootstrapBrokers(pipe.getSource())).thenReturn("localhost:19092");++ assertEquals("localhost:19092", manager.resolveBootstrapServers(pipe));+ }++ @Test+ void resolveTopicNameUsesMatchingParameterBlock() throws Exception {+ Pipe managed = new Pipe();+ managed.setSource("arn:aws:kafka:us-east-1:000000000000:cluster/demo/uuid");+ managed.setSourceParameters(MAPPER.readTree("""+ {+ "ManagedStreamingKafkaParameters": {+ "TopicName": "managed-topic"+ }+ }+ """));++ Pipe selfManaged = new Pipe();+ selfManaged.setSource("smk://localhost:9092");+ selfManaged.setSourceParameters(MAPPER.readTree("""+ {+ "SelfManagedKafkaParameters": {+ "TopicName": "self-topic"+ }+ }+ """));++ assertEquals("managed-topic", manager.resolveTopicName(managed));+ assertEquals("self-topic", manager.resolveTopicName(selfManaged));+ }++ @Test+ @DisplayName("resolveTopicName reports the matching parameter block")+ void resolveTopicNameReportsMatchingParameterBlock() throws Exception {+ Pipe managed = new Pipe();+ managed.setSource("arn:aws:kafka:us-east-1:000000000000:cluster/demo/uuid");+ managed.setSourceParameters(MAPPER.readTree("""+ {+ "ManagedStreamingKafkaParameters": {}+ }+ """));++ Pipe selfManaged = new Pipe();+ selfManaged.setSource("smk://localhost:9092");+ selfManaged.setSourceParameters(MAPPER.readTree("""+ {+ "SelfManagedKafkaParameters": {}+ }+ """));++ assertEquals("SourceParameters.ManagedStreamingKafkaParameters.TopicName is required",+ assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,+ () -> manager.resolveTopicName(managed)).getMessage());+ assertEquals("SourceParameters.SelfManagedKafkaParameters.TopicName is required",+ assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,+ () -> manager.resolveTopicName(selfManaged)).getMessage());+ }++ @Test+ void resolveTopicNameReportsMissingParameterBlock() {+ Pipe managed = new Pipe();+ managed.setSource("arn:aws:kafka:us-east-1:000000000000:cluster/demo/uuid");+ managed.setSourceParameters(MAPPER.createObjectNode());++ Pipe selfManaged = new Pipe();+ selfManaged.setSource("smk://localhost:9092");+ selfManaged.setSourceParameters(MAPPER.createObjectNode());++ assertEquals("SourceParameters.ManagedStreamingKafkaParameters is required",+ assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,+ () -> manager.resolveTopicName(managed)).getMessage());+ assertEquals("SourceParameters.SelfManagedKafkaParameters is required",+ assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,+ () -> manager.resolveTopicName(selfManaged)).getMessage());+ }++ @Test+ void resolveConsumerGroupIdUsesConfiguredValueOrStableFallback() throws Exception {+ Pipe configured = new Pipe();+ configured.setName("orders-pipe");+ configured.setSource("smk://broker-1:9092");+ configured.setSourceParameters(MAPPER.readTree("""+ {+ "SelfManagedKafkaParameters": {+ "TopicName": "orders",+ "ConsumerGroupID": "custom-group"+ }+ }+ """));++ Pipe fallback = new Pipe();+ fallback.setName("orders-pipe");+ fallback.setSource("arn:aws:kafka:us-east-1:000000000000:cluster/demo/uuid");+ fallback.setSourceParameters(MAPPER.readTree("""+ {+ "ManagedStreamingKafkaParameters": {+ "TopicName": "orders"+ }+ }+ """));++ assertEquals("custom-group", manager.resolveConsumerGroupId(configured));++ String fallbackGroupId = manager.resolveConsumerGroupId(fallback);+ assertEquals(fallbackGroupId, manager.resolveConsumerGroupId(fallback));+ org.junit.jupiter.api.Assertions.assertTrue(+ fallbackGroupId.startsWith("floci-pipes-orders-pipe-"),+ "Expected generated consumer group id prefix, got: " + fallbackGroupId);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.javanew file mode 100644index 0000000000..ec91255438--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java@@ -0,0 +1,224 @@+package io.github.hectorvent.floci.services.pipes;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;+import io.github.hectorvent.floci.services.kinesis.KinesisService;+import io.github.hectorvent.floci.services.pipes.model.Pipe;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.vertx.core.Vertx;+import org.apache.kafka.clients.consumer.ConsumerRecord;+import org.apache.kafka.clients.consumer.ConsumerRecords;+import org.apache.kafka.clients.consumer.OffsetAndMetadata;+import org.apache.kafka.common.TopicPartition;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.extension.ExtendWith;+import org.mockito.ArgumentCaptor;+import org.mockito.Mock;+import org.mockito.junit.jupiter.MockitoExtension;++import java.nio.charset.StandardCharsets;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.anyMap;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verifyNoMoreInteractions;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++@ExtendWith(MockitoExtension.class)+class PipesPollerTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();++ @Mock private Vertx vertx;+ @Mock private SqsService sqsService;+ @Mock private KinesisService kinesisService;+ @Mock private DynamoDbStreamService dynamoDbStreamService;+ @Mock private PipesKafkaConsumerManager kafkaConsumerManager;+ @Mock private PipesTargetInvoker targetInvoker;+ @Mock private EmulatorConfig config;++ private PipesPoller poller;++ @BeforeEach+ void setUp() {+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ poller = new PipesPoller(vertx, sqsService, kinesisService, dynamoDbStreamService,+ kafkaConsumerManager, targetInvoker, new PipesFilterMatcher(MAPPER), MAPPER, config);+ }++ @Test+ void pollKafka_filtersUsingDecodedPayloadButDeliversOriginalRecord() throws Exception {+ Pipe pipe = selfManagedKafkaPipe();+ byte[] key = "customer-123".getBytes(StandardCharsets.UTF_8);+ byte[] value = "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8);+ ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>("orders", 0, 42L, key, value);+ record.headers().add("traceId", "abc123".getBytes(StandardCharsets.UTF_8));++ when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");++ poller.pollKafka(pipe, "us-east-1");++ ArgumentCaptor<String> payloadCaptor = ArgumentCaptor.forClass(String.class);+ verify(targetInvoker).invoke(eq(pipe), payloadCaptor.capture(), eq("us-east-1"));+ verify(kafkaConsumerManager).commit(eq(pipe), anyMap());++ JsonNode delivered = MAPPER.readTree(payloadCaptor.getValue());+ assertEquals("orders", delivered.path("topic").asText());+ assertTrue(delivered.path("partition").isInt());+ assertEquals("broker-1:9092", delivered.path("bootstrapServers").asText());+ assertEquals("Y3VzdG9tZXItMTIz", delivered.path("key").asText());+ assertEquals("eyJzdGF0dXMiOiJhY3RpdmUiLCJpZCI6Im9yZGVyLTEifQ==", delivered.path("value").asText());+ assertTrue(delivered.path("headers").get(0).path("traceId").isArray());+ assertEquals(List.of(97, 98, 99, 49, 50, 51),+ MAPPER.convertValue(delivered.path("headers").get(0).path("traceId"), List.class));+ assertFalse(delivered.has("eventSourceKey"));+ assertTrue(delivered.path("value").isTextual());+ }++ @Test+ void pollKafka_doesNotCommitWhenDeliveryFails() throws Exception {+ Pipe pipe = selfManagedKafkaPipe();+ ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>(+ "orders", 0, 7L, null, "{\"status\":\"active\"}".getBytes(StandardCharsets.UTF_8));++ when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");+ org.mockito.Mockito.doThrow(new RuntimeException("boom"))+ .when(targetInvoker).invoke(eq(pipe), anyString(), eq("us-east-1"));++ poller.pollKafka(pipe, "us-east-1");++ verify(kafkaConsumerManager, never()).commit(pipe);+ verify(kafkaConsumerManager, never()).commit(eq(pipe), anyMap());+ }++ @Test+ void pollKafka_commitsDeliveredPrefixWhenLaterRecordFails() throws Exception {+ Pipe pipe = selfManagedKafkaPipe();+ ConsumerRecord<byte[], byte[]> first = new ConsumerRecord<>(+ "orders", 0, 0L, null, "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8));+ ConsumerRecord<byte[], byte[]> second = new ConsumerRecord<>(+ "orders", 0, 1L, null, "{\"status\":\"active\",\"id\":\"order-2\"}".getBytes(StandardCharsets.UTF_8));++ when(kafkaConsumerManager.poll(pipe)).thenReturn(records(first, second));+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");+ org.mockito.Mockito.doNothing()+ .doThrow(new RuntimeException("boom"))+ .when(targetInvoker).invoke(eq(pipe), anyString(), eq("us-east-1"));++ poller.pollKafka(pipe, "us-east-1");++ ArgumentCaptor<Map<TopicPartition, OffsetAndMetadata>> offsetsCaptor = ArgumentCaptor.forClass(Map.class);+ verify(kafkaConsumerManager).commit(eq(pipe), offsetsCaptor.capture());+ assertEquals(1L, offsetsCaptor.getValue().get(new TopicPartition("orders", 0)).offset());+ }++ @Test+ void pollKafka_representsNullKeyAndValueAsJsonNull() throws Exception {+ Pipe pipe = nullableSelfManagedKafkaPipe();+ ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>("orders", 0, 3L, null, null);++ when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");++ poller.pollKafka(pipe, "us-east-1");++ ArgumentCaptor<String> payloadCaptor = ArgumentCaptor.forClass(String.class);+ verify(targetInvoker).invoke(eq(pipe), payloadCaptor.capture(), eq("us-east-1"));+ JsonNode delivered = MAPPER.readTree(payloadCaptor.getValue());+ assertTrue(delivered.path("key").isNull());+ assertTrue(delivered.path("value").isNull());+ }++ @Test+ void pollKafka_lambdaCommitsSuccessfulPrefixBeforeLaterFailure() throws Exception {+ Pipe pipe = lambdaSelfManagedKafkaPipe();+ ConsumerRecord<byte[], byte[]> first = new ConsumerRecord<>(+ "orders", 0, 0L, null, "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8));+ ConsumerRecord<byte[], byte[]> skipped = new ConsumerRecord<>(+ "orders", 0, 1L, null, "{\"status\":\"inactive\",\"id\":\"order-2\"}".getBytes(StandardCharsets.UTF_8));+ ConsumerRecord<byte[], byte[]> failing = new ConsumerRecord<>(+ "orders", 0, 2L, null, "{\"status\":\"active\",\"id\":\"order-3\"}".getBytes(StandardCharsets.UTF_8));++ when(kafkaConsumerManager.poll(pipe)).thenReturn(records(first, skipped, failing));+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");+ org.mockito.Mockito.doNothing()+ .doThrow(new RuntimeException("boom"))+ .when(targetInvoker).invoke(eq(pipe), anyString(), eq("us-east-1"));++ poller.pollKafka(pipe, "us-east-1");++ ArgumentCaptor<Map<TopicPartition, OffsetAndMetadata>> offsetsCaptor = ArgumentCaptor.forClass(Map.class);+ verify(kafkaConsumerManager).commit(eq(pipe), offsetsCaptor.capture());+ assertEquals(2L, offsetsCaptor.getValue().get(new TopicPartition("orders", 0)).offset());+ }++ private Pipe selfManagedKafkaPipe() throws Exception {+ Pipe pipe = new Pipe();+ pipe.setName("orders-pipe");+ pipe.setArn("arn:aws:pipes:us-east-1:000000000000:pipe/orders-pipe");+ pipe.setSource("smk://broker-1:9092");+ pipe.setTarget("arn:aws:sqs:us-east-1:000000000000:orders-target");+ pipe.setSourceParameters(MAPPER.readTree("""+ {+ "SelfManagedKafkaParameters": {+ "TopicName": "orders"+ },+ "FilterCriteria": {+ "Filters": [+ {"Pattern": "{\\\"value\\\": {\\\"status\\\": [\\\"active\\\"]}}"}+ ]+ }+ }+ """));+ return pipe;+ }++ private Pipe nullableSelfManagedKafkaPipe() throws Exception {+ Pipe pipe = new Pipe();+ pipe.setName("nullable-orders-pipe");+ pipe.setArn("arn:aws:pipes:us-east-1:000000000000:pipe/nullable-orders-pipe");+ pipe.setSource("smk://broker-1:9092");+ pipe.setTarget("arn:aws:sqs:us-east-1:000000000000:orders-target");+ pipe.setSourceParameters(MAPPER.readTree("""+ {+ "SelfManagedKafkaParameters": {+ "TopicName": "orders"+ },+ "FilterCriteria": {+ "Filters": [+ {"Pattern": "{\\\"key\\\": [{\\\"exists\\\": false}]}"}+ ]+ }+ }+ """));+ return pipe;+ }++ private Pipe lambdaSelfManagedKafkaPipe() throws Exception {+ Pipe pipe = selfManagedKafkaPipe();+ pipe.setTarget("arn:aws:lambda:us-east-1:000000000000:function:orders-target");+ return pipe;+ }++ private ConsumerRecords<byte[], byte[]> records(ConsumerRecord<byte[], byte[]>... records) {+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> byPartition = new java.util.LinkedHashMap<>();+ for (ConsumerRecord<byte[], byte[]> record : records) {+ TopicPartition topicPartition = new TopicPartition(record.topic(), record.partition());+ byPartition.computeIfAbsent(topicPartition, ignored -> new java.util.ArrayList<>()).add(record);+ }+ return new ConsumerRecords<>(byPartition);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.javaindex 36cd367122..619ecef9b3 100644--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java@@ -107,6 +107,33 @@ class PipesServiceTest {assertEquals("ValidationException", ex.getErrorCode());}+ @Test+ void createPipeWithManagedKafkaSourceRequiresTopicParameters() {+ AwsException ex = assertThrows(AwsException.class, () ->+ pipesService.createPipe("msk-pipe",+ "arn:aws:kafka:us-east-1:000000000000:cluster/test/uuid",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null, null, null, null, null, "us-east-1"));++ assertEquals("ValidationException", ex.getErrorCode());+ }++ @Test+ void createPipeWithSelfManagedKafkaSourceRequiresTopicParameters() throws Exception {+ var mapper = new com.fasterxml.jackson.databind.ObjectMapper();+ AwsException ex = assertThrows(AwsException.class, () ->+ pipesService.createPipe("smk-pipe",+ "smk://localhost:9092",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null,+ mapper.readTree("{\"SelfManagedKafkaParameters\":{}}"),+ null, null, null, "us-east-1"));++ assertEquals("ValidationException", ex.getErrorCode());+ }+@Testvoid describePipe() {pipesService.createPipe("my-pipe",