Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(pipes): support Kafka pipe sources and polling (#1260)

Liam MacPherson86c4aae2e2f6
diff --git a/docs/services/pipes.md b/docs/services/pipes.md
index f1e96ce653..9683824198 100644
--- a/docs/services/pipes.md
+++ b/docs/services/pipes.md
@@ -81,11 +81,11 @@ Floci emulates EventBridge Pipes with the following supported source and target
- Amazon SQS queues
- Amazon Kinesis streams
- Amazon DynamoDB streams
-- Kafka topics (MSK)
+- Kafka topics (MSK and self-managed via `smk://`)
**Targets:**
- Lambda functions
- SQS queues
- SNS topics
- Kinesis streams
-- Step Functions state machines
\ No newline at end of file
+- Step Functions state machines
diff --git a/pom.xml b/pom.xml
index b2d55e7ca7..fd296299b7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,6 +100,13 @@
<version>9.2.1</version>
</dependency>
+ <!-- Kafka client for MSK/self-managed Kafka pipe polling -->
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>4.3.0</version>
+ </dependency>
+
<!-- Docker client for Lambda container lifecycle management -->
<dependency>
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java
new file mode 100644
index 0000000000..a9c80cdddd
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java
@@ -0,0 +1,211 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.msk.MskService;
+import io.github.hectorvent.floci.services.pipes.model.Pipe;
+import jakarta.annotation.PreDestroy;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.jboss.logging.Logger;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+
+@ApplicationScoped
+public class PipesKafkaConsumerManager {
+
+ private static final Logger LOG = Logger.getLogger(PipesKafkaConsumerManager.class);
+ private static final Duration POLL_TIMEOUT = Duration.ofMillis(100);
+ private static final String DEFAULT_STARTING_POSITION = "LATEST";
+
+ private final MskService mskService;
+ private final ConcurrentHashMap<String, KafkaConsumer<byte[], byte[]>> consumers = new ConcurrentHashMap<>();
+
+ @Inject
+ public PipesKafkaConsumerManager(MskService mskService) {
+ this.mskService = mskService;
+ }
+
+ @PreDestroy
+ void shutdown() {
+ consumers.values().forEach(this::closeQuietly);
+ consumers.clear();
+ }
+
+ public ConsumerRecords<byte[], byte[]> poll(Pipe pipe) {
+ KafkaConsumer<byte[], byte[]> consumer = consumers.computeIfAbsent(pipe.getArn(), ignored -> createConsumer(pipe));
+ try {
+ return consumer.poll(POLL_TIMEOUT);
+ } catch (RuntimeException e) {
+ close(pipe);
+ throw e;
+ }
+ }
+
+ public void commit(Pipe pipe) {
+ KafkaConsumer<byte[], byte[]> consumer = consumers.get(pipe.getArn());
+ if (consumer == null) {
+ return;
+ }
+ try {
+ consumer.commitSync();
+ } catch (RuntimeException e) {
+ close(pipe);
+ throw e;
+ }
+ }
+
+ public void commit(Pipe pipe, Map<TopicPartition, OffsetAndMetadata> offsets) {
+ if (offsets.isEmpty()) {
+ return;
+ }
+ KafkaConsumer<byte[], byte[]> consumer = consumers.get(pipe.getArn());
+ if (consumer == null) {
+ return;
+ }
+ try {
+ consumer.commitSync(offsets);
+ } catch (RuntimeException e) {
+ close(pipe);
+ throw e;
+ }
+ }
+
+ public void close(Pipe pipe) {
+ KafkaConsumer<byte[], byte[]> consumer = consumers.remove(pipe.getArn());
+ if (consumer != null) {
+ closeQuietly(consumer);
+ }
+ }
+
+ String resolveBootstrapServers(Pipe pipe) {
+ if (isSelfManagedSource(pipe)) {
+ List<String> servers = new ArrayList<>();
+ servers.add(pipe.getSource().substring("smk://".length()));
+ JsonNode additionalServers = kafkaParameters(pipe).path("AdditionalBootstrapServers");
+ if (additionalServers.isArray()) {
+ additionalServers.forEach(node -> {
+ String server = node.asText(null);
+ if (server != null && !server.isBlank()) {
+ servers.add(server);
+ }
+ });
+ }
+ return String.join(",", servers);
+ }
+ if (isManagedSource(pipe)) {
+ return mskService.getBootstrapBrokers(pipe.getSource());
+ }
+ throw new AwsException("ValidationException", "Unsupported Kafka source: " + pipe.getSource(), 400);
+ }
+
+ String resolveTopicName(Pipe pipe) {
+ JsonNode params = kafkaParameters(pipe);
+ String topic = params.path("TopicName").asText(null);
+ if (topic == null || topic.isBlank()) {
+ throw new AwsException("ValidationException",
+ "SourceParameters." + parameterBlockName(pipe) + ".TopicName is required", 400);
+ }
+ return topic;
+ }
+
+ int resolveBatchSize(Pipe pipe, int defaultBatchSize) {
+ JsonNode params = kafkaParameters(pipe);
+ return params.path("BatchSize").asInt(defaultBatchSize);
+ }
+
+ String resolveConsumerGroupId(Pipe pipe) {
+ return resolveConsumerGroupId(pipe, kafkaParameters(pipe));
+ }
+
+ private KafkaConsumer<byte[], byte[]> createConsumer(Pipe pipe) {
+ String bootstrapServers = resolveBootstrapServers(pipe);
+ String topicName = resolveTopicName(pipe);
+ JsonNode params = kafkaParameters(pipe);
+
+ Properties properties = new Properties();
+ properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
+ properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
+ properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
+ properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, resolveOffsetReset(params));
+ properties.put(ConsumerConfig.GROUP_ID_CONFIG, resolveConsumerGroupId(pipe, params));
+ properties.put(ConsumerConfig.CLIENT_ID_CONFIG, "floci-pipes-" + UUID.randomUUID());
+ properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, Integer.toString(resolveBatchSize(pipe, 100)));
+
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(properties);
+ consumer.subscribe(List.of(topicName));
+ LOG.infov("Pipe {0}: subscribed Kafka consumer to topic {1} via {2}",
+ pipe.getName(), topicName, bootstrapServers);
+ return consumer;
+ }
+
+ private JsonNode kafkaParameters(Pipe pipe) {
+ JsonNode sourceParameters = pipe.getSourceParameters();
+ if (sourceParameters == null) {
+ throw new AwsException("ValidationException", "Kafka pipe SourceParameters are required", 400);
+ }
+ JsonNode parameters = sourceParameters.path(parameterBlockName(pipe));
+ if (parameters.isMissingNode()) {
+ throw new AwsException("ValidationException",
+ "SourceParameters." + parameterBlockName(pipe) + " is required", 400);
+ }
+ return parameters;
+ }
+
+ private String resolveOffsetReset(JsonNode params) {
+ String startingPosition = params.path("StartingPosition").asText(DEFAULT_STARTING_POSITION);
+ return "TRIM_HORIZON".equalsIgnoreCase(startingPosition) ? "earliest" : "latest";
+ }
+
+ private String resolveConsumerGroupId(Pipe pipe, JsonNode params) {
+ String configured = params.path("ConsumerGroupID").asText(null);
+ if (configured != null && !configured.isBlank()) {
+ return configured;
+ }
+
+ String sourceFingerprint = isSelfManagedSource(pipe)
+ ? pipe.getSource().substring("smk://".length())
+ : AwsArnUtils.parse(pipe.getSource()).resource();
+ return "floci-pipes-" + pipe.getName() + "-" + Integer.toUnsignedString(sourceFingerprint.hashCode());
+ }
+
+ private static boolean isManagedSource(Pipe pipe) {
+ return pipe.getSource().contains(":kafka:");
+ }
+
+ private static boolean isSelfManagedSource(Pipe pipe) {
+ return pipe.getSource().startsWith("smk://");
+ }
+
+ private static String parameterBlockName(Pipe pipe) {
+ if (isManagedSource(pipe)) {
+ return "ManagedStreamingKafkaParameters";
+ }
+ if (isSelfManagedSource(pipe)) {
+ return "SelfManagedKafkaParameters";
+ }
+ throw new AwsException("ValidationException", "Unsupported Kafka source: " + pipe.getSource(), 400);
+ }
+
+ private void closeQuietly(KafkaConsumer<byte[], byte[]> consumer) {
+ try {
+ consumer.close();
+ } catch (Exception e) {
+ LOG.debugv("Ignoring Kafka consumer close error: {0}", e.getMessage());
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
index 225c22983a..e93a139e03 100644
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.pipes;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -17,8 +18,14 @@ import io.vertx.core.Vertx;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.header.Header;
import org.jboss.logging.Logger;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap;
@@ -41,6 +48,7 @@ public class PipesPoller {
private final SqsService sqsService;
private final KinesisService kinesisService;
private final DynamoDbStreamService dynamoDbStreamService;
+ private final PipesKafkaConsumerManager kafkaConsumerManager;
private final PipesTargetInvoker targetInvoker;
private final PipesFilterMatcher filterMatcher;
private final ObjectMapper objectMapper;
@@ -60,6 +68,7 @@ public class PipesPoller {
SqsService sqsService,
KinesisService kinesisService,
DynamoDbStreamService dynamoDbStreamService,
+ PipesKafkaConsumerManager kafkaConsumerManager,
PipesTargetInvoker targetInvoker,
PipesFilterMatcher filterMatcher,
ObjectMapper objectMapper,
@@ -68,6 +77,7 @@ public class PipesPoller {
this.sqsService = sqsService;
this.kinesisService = kinesisService;
this.dynamoDbStreamService = dynamoDbStreamService;
+ this.kafkaConsumerManager = kafkaConsumerManager;
this.targetInvoker = targetInvoker;
this.filterMatcher = filterMatcher;
this.objectMapper = objectMapper;
@@ -99,6 +109,7 @@ public class PipesPoller {
vertx.cancelTimer(timerId);
kinesisIterators.remove(pipeKey);
dynamoDbIterators.remove(pipeKey);
+ kafkaConsumerManager.close(pipe);
LOG.infov("Pipe {0}: stopped polling", pipe.getName());
}
}
@@ -125,6 +136,8 @@ public class PipesPoller {
pollKinesis(pipe, region);
} else if (sourceArn.contains(":dynamodb:")) {
pollDynamoDbStreams(pipe, region);
+ } else if (isKafkaSource(sourceArn)) {
+ pollKafka(pipe, region);
} else {
LOG.warnv("Pipe {0}: unsupported source type: {1}", pipe.getName(), sourceArn);
}
@@ -310,6 +323,155 @@ public class PipesPoller {
}
}
+ void pollKafka(Pipe pipe, String region) {
+ ConsumerRecords<byte[], byte[]> records = kafkaConsumerManager.poll(pipe);
+ if (records.isEmpty()) {
+ return;
+ }
+
+ List<ConsumerRecord<byte[], byte[]>> batch = new ArrayList<>(records.count());
+ records.forEach(batch::add);
+
+ LOG.infov("Pipe {0}: received {1} Kafka record(s)", pipe.getName(), batch.size());
+ List<ObjectNode> deliveryNodes = buildKafkaRecordNodes(batch, pipe);
+ List<ObjectNode> filterNodes = buildKafkaFilterNodes(deliveryNodes, batch);
+ List<JsonNode> filtered = filterMatcher.applyFilterCriteria(new ArrayList<>(filterNodes), pipe.getSourceParameters());
+ if (filtered.isEmpty()) {
+ kafkaConsumerManager.commit(pipe);
+ return;
+ }
+
+ Map<String, JsonNode> deliveryRecordsByIdentity = new HashMap<>();
+ for (JsonNode record : deliveryNodes) {
+ deliveryRecordsByIdentity.put(kafkaRecordIdentity(record), record);
+ }
+
+ List<JsonNode> deliveryRecords = new ArrayList<>(filtered.size());
+ for (JsonNode record : filtered) {
+ JsonNode deliveryRecord = deliveryRecordsByIdentity.get(kafkaRecordIdentity(record));
+ if (deliveryRecord != null) {
+ deliveryRecords.add(deliveryRecord);
+ }
+ }
+ if (deliveryRecords.isEmpty()) {
+ kafkaConsumerManager.commit(pipe);
+ return;
+ }
+
+ int failed = isLambdaTarget(pipe)
+ ? deliverKafkaLambdaRecords(pipe, region, batch, deliveryRecordsByIdentity, filtered)
+ : deliverKafkaRecords(pipe, region, batch, deliveryRecordsByIdentity, filtered);
+ if (failed == 0) {
+ return;
+ }
+
+ LOG.warnv("Pipe {0}: {1} Kafka record(s) not committed because delivery failed",
+ pipe.getName(), failed);
+ }
+
+ private int deliverKafkaLambdaRecords(Pipe pipe,
+ String region,
+ List<ConsumerRecord<byte[], byte[]>> batch,
+ Map<String, JsonNode> deliveryRecordsByIdentity,
+ List<JsonNode> filtered) {
+ Set<String> matchedIdentities = new HashSet<>(filtered.size());
+ filtered.forEach(record -> matchedIdentities.add(kafkaRecordIdentity(record)));
+
+ Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = groupKafkaRecordsByPartition(batch);
+ int failed = 0;
+
+ for (Map.Entry<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> entry : recordsByPartition.entrySet()) {
+ failed += deliverKafkaLambdaPartition(pipe, region, entry.getKey(), entry.getValue(),
+ deliveryRecordsByIdentity, matchedIdentities, offsetsToCommit);
+ }
+
+ if (!offsetsToCommit.isEmpty()) {
+ kafkaConsumerManager.commit(pipe, offsetsToCommit);
+ }
+ return failed;
+ }
+
+ private int deliverKafkaLambdaPartition(Pipe pipe,
+ String region,
+ TopicPartition partition,
+ List<ConsumerRecord<byte[], byte[]>> partitionRecords,
+ Map<String, JsonNode> deliveryRecordsByIdentity,
+ Set<String> matchedIdentities,
+ Map<TopicPartition, OffsetAndMetadata> offsetsToCommit) {
+ List<JsonNode> pendingBatch = new ArrayList<>();
+ long pendingOffset = -1L;
+
+ for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {
+ String identity = kafkaRecordIdentity(record);
+ if (!matchedIdentities.contains(identity)) {
+ if (!pendingBatch.isEmpty()) {
+ if (!invokeWithDlq(pipe, wrapRecords(pendingBatch), region)) {
+ return pendingBatch.size();
+ }
+ offsetsToCommit.put(partition, new OffsetAndMetadata(pendingOffset));
+ pendingBatch.clear();
+ }
+ offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));
+ continue;
+ }
+
+ JsonNode deliveryRecord = deliveryRecordsByIdentity.get(identity);
+ if (deliveryRecord != null) {
+ pendingBatch.add(deliveryRecord);
+ pendingOffset = record.offset() + 1;
+ }
+ }
+
+ if (!pendingBatch.isEmpty()) {
+ if (!invokeWithDlq(pipe, wrapRecords(pendingBatch), region)) {
+ return pendingBatch.size();
+ }
+ offsetsToCommit.put(partition, new OffsetAndMetadata(pendingOffset));
+ }
+ return 0;
+ }
+
+ private int deliverKafkaRecords(Pipe pipe,
+ String region,
+ List<ConsumerRecord<byte[], byte[]>> batch,
+ Map<String, JsonNode> deliveryRecordsByIdentity,
+ List<JsonNode> filtered) {
+ Set<String> matchedIdentities = new HashSet<>(filtered.size());
+ filtered.forEach(record -> matchedIdentities.add(kafkaRecordIdentity(record)));
+
+ Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();
+ Set<TopicPartition> blockedPartitions = new HashSet<>();
+ int failed = 0;
+
+ for (ConsumerRecord<byte[], byte[]> record : batch) {
+ TopicPartition partition = new TopicPartition(record.topic(), record.partition());
+ if (blockedPartitions.contains(partition)) {
+ continue;
+ }
+
+ String identity = kafkaRecordIdentity(record);
+ if (!matchedIdentities.contains(identity)) {
+ offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));
+ continue;
+ }
+
+ JsonNode deliveryRecord = deliveryRecordsByIdentity.get(identity);
+ if (deliveryRecord != null && invokeWithDlq(pipe, deliveryRecord.toString(), region)) {
+ offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));
+ continue;
+ }
+
+ blockedPartitions.add(partition);
+ failed++;
+ }
+
+ if (!offsetsToCommit.isEmpty()) {
+ kafkaConsumerManager.commit(pipe, offsetsToCommit);
+ }
+ return failed;
+ }
+
private String initDynamoDbIterator(String streamArn) {
try {
return dynamoDbStreamService.getShardIterator(
@@ -469,6 +631,58 @@ public class PipesPoller {
return nodes;
}
+ private List<ObjectNode> buildKafkaRecordNodes(List<ConsumerRecord<byte[], byte[]>> records, Pipe pipe) {
+ List<ObjectNode> nodes = new ArrayList<>();
+ String eventSource = pipe.getSource().contains(":kafka:") ? "aws:kafka" : "SelfManagedKafka";
+ String bootstrapServers = kafkaConsumerManager.resolveBootstrapServers(pipe);
+ for (ConsumerRecord<byte[], byte[]> record : records) {
+ ObjectNode node = objectMapper.createObjectNode();
+ node.put("eventSource", eventSource);
+ if (pipe.getSource().contains(":kafka:")) {
+ node.put("eventSourceArn", pipe.getSource());
+ } else {
+ node.put("bootstrapServers", bootstrapServers);
+ }
+ node.put("topic", record.topic());
+ node.put("partition", record.partition());
+ node.put("offset", record.offset());
+ node.put("timestamp", record.timestamp());
+ node.put("timestampType", record.timestampType().name());
+ putKafkaBinaryField(node, "key", record.key());
+ putKafkaBinaryField(node, "value", record.value());
+
+ var headersNode = node.putArray("headers");
+ for (Header header : record.headers()) {
+ ObjectNode headerNode = objectMapper.createObjectNode();
+ byte[] headerValue = header.value();
+ if (headerValue == null) {
+ headerNode.putNull(header.key());
+ } else {
+ var values = headerNode.putArray(header.key());
+ for (byte b : headerValue) {
+ values.add(b & 0xFF);
+ }
+ }
+ headersNode.add(headerNode);
+ }
+ nodes.add(node);
+ }
+ return nodes;
+ }
+
+ private List<ObjectNode> buildKafkaFilterNodes(List<ObjectNode> deliveryNodes,
+ List<ConsumerRecord<byte[], byte[]>> records) {
+ List<ObjectNode> nodes = new ArrayList<>(deliveryNodes.size());
+ for (int i = 0; i < deliveryNodes.size(); i++) {
+ ObjectNode node = deliveryNodes.get(i).deepCopy();
+ ConsumerRecord<byte[], byte[]> record = records.get(i);
+ applyDecodedKafkaField(node, "key", record.key());
+ applyDecodedKafkaField(node, "value", record.value());
+ nodes.add(node);
+ }
+ return nodes;
+ }
+
// ──────────────────────────── Utilities ────────────────────────────
private static boolean isLambdaTarget(Pipe pipe) {
@@ -476,6 +690,57 @@ public class PipesPoller {
return targetArn.contains(":lambda:") || targetArn.contains(":function:");
}
+ private static boolean isKafkaSource(String sourceArn) {
+ return sourceArn.startsWith("smk://") || sourceArn.contains(":kafka:");
+ }
+
+ private static String base64(byte[] value) {
+ return Base64.getEncoder().encodeToString(value);
+ }
+
+ private static String kafkaRecordIdentity(JsonNode record) {
+ return record.path("topic").asText() + ":"
+ + record.path("partition").asText() + ":"
+ + record.path("offset").asText();
+ }
+
+ private static String kafkaRecordIdentity(ConsumerRecord<byte[], byte[]> record) {
+ return record.topic() + ":" + record.partition() + ":" + record.offset();
+ }
+
+ private void applyDecodedKafkaField(ObjectNode node, String fieldName, byte[] value) {
+ if (value == null) {
+ node.putNull(fieldName);
+ return;
+ }
+ String decoded = new String(value, StandardCharsets.UTF_8);
+ try {
+ JsonNode parsed = objectMapper.readTree(decoded);
+ node.set(fieldName, parsed);
+ } catch (JsonProcessingException e) {
+ LOG.debugv("Kafka {0} is not valid JSON: {1}", fieldName, e.getOriginalMessage());
+ node.put(fieldName, decoded);
+ }
+ }
+
+ private void putKafkaBinaryField(ObjectNode node, String fieldName, byte[] value) {
+ if (value == null) {
+ node.putNull(fieldName);
+ return;
+ }
+ node.put(fieldName, base64(value));
+ }
+
+ private Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> groupKafkaRecordsByPartition(
+ List<ConsumerRecord<byte[], byte[]>> records) {
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = new HashMap<>();
+ for (ConsumerRecord<byte[], byte[]> record : records) {
+ TopicPartition partition = new TopicPartition(record.topic(), record.partition());
+ recordsByPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);
+ }
+ return recordsByPartition;
+ }
+
private static String pipeKey(Pipe pipe) {
return pipe.getArn();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java
index 366b66a602..ea212b32b6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java
@@ -72,6 +72,8 @@ public class PipesService implements TagHandler {
throw new AwsException("ValidationException", "RoleArn is required", 400);
}
+ validateSourceConfiguration(source, sourceParameters);
+
String key = region + "::" + name;
if (storage.get(key).isPresent()) {
throw new AwsException("ConflictException",
@@ -125,6 +127,8 @@ public class PipesService implements TagHandler {
.orElseThrow(() -> new AwsException("NotFoundException",
"Pipe " + name + " does not exist.", 404));
+ validateSourceConfiguration(pipe.getSource(), sourceParameters != null ? sourceParameters : pipe.getSourceParameters());
+
if (target != null) pipe.setTarget(target);
if (roleArn != null) pipe.setRoleArn(roleArn);
if (description != null) pipe.setDescription(description);
@@ -240,4 +244,29 @@ public class PipesService implements TagHandler {
.orElseThrow(() -> new AwsException("NotFoundException",
"Resource " + arn + " does not exist.", 404));
}
+
+ private void validateSourceConfiguration(String source, JsonNode sourceParameters) {
+ if (source == null) {
+ return;
+ }
+ if (source.startsWith("smk://")) {
+ requireKafkaParameters(sourceParameters, "SelfManagedKafkaParameters");
+ return;
+ }
+ if (source.contains(":kafka:")) {
+ requireKafkaParameters(sourceParameters, "ManagedStreamingKafkaParameters");
+ }
+ }
+
+ private void requireKafkaParameters(JsonNode sourceParameters, String parameterBlock) {
+ if (sourceParameters == null || sourceParameters.path(parameterBlock).isMissingNode()) {
+ throw new AwsException("ValidationException",
+ "SourceParameters." + parameterBlock + " is required", 400);
+ }
+ String topicName = sourceParameters.path(parameterBlock).path("TopicName").asText(null);
+ if (topicName == null || topicName.isBlank()) {
+ throw new AwsException("ValidationException",
+ "SourceParameters." + parameterBlock + ".TopicName is required", 400);
+ }
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.java
index 77cb1ad97e..9fff5ee76b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesFilterMatcherTest.java
@@ -101,6 +101,28 @@ class PipesFilterMatcherTest {
assertFalse(matcher.matchesRecord(record, "{\"detail\": {\"status\": [\"inactive\"]}}"));
}
+ @Test
+ void kafkaDecodedValueCanBeFilteredAsJson() throws Exception {
+ JsonNode record = MAPPER.readTree("""
+ {
+ "eventSource": "aws:kafka",
+ "topic": "orders",
+ "value": {"status": "active", "id": "order-1"}
+ }
+ """);
+
+ assertTrue(matcher.matchesRecord(record, "{\"value\": {\"status\": [\"active\"]}}"));
+ assertFalse(matcher.matchesRecord(record, "{\"value\": {\"status\": [\"inactive\"]}}"));
+ }
+
+ @Test
+ void kafkaDecodedKeyCanBeFilteredAsString() throws Exception {
+ JsonNode record = MAPPER.readTree("{\"key\": \"customer-123\", \"topic\": \"orders\"}");
+
+ assertTrue(matcher.matchesRecord(record, "{\"key\": [{\"prefix\": \"customer-\"}]}"));
+ assertFalse(matcher.matchesRecord(record, "{\"key\": [\"customer-999\"]}"));
+ }
+
// ──────────────────────────── Operators ────────────────────────────
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java
new file mode 100644
index 0000000000..125a87775f
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java
@@ -0,0 +1,164 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.services.msk.MskService;
+import io.github.hectorvent.floci.services.pipes.model.Pipe;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.when;
+
+class PipesKafkaConsumerManagerTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private MskService mskService;
+ private PipesKafkaConsumerManager manager;
+
+ @BeforeEach
+ void setUp() {
+ mskService = Mockito.mock(MskService.class);
+ manager = new PipesKafkaConsumerManager(mskService);
+ }
+
+ @Test
+ void resolveBootstrapServersForSelfManagedKafkaIncludesAdditionalServers() throws Exception {
+ Pipe pipe = new Pipe();
+ pipe.setSource("smk://broker-1:9092");
+ pipe.setSourceParameters(MAPPER.readTree("""
+ {
+ "SelfManagedKafkaParameters": {
+ "TopicName": "orders",
+ "AdditionalBootstrapServers": ["broker-2:9092", "broker-3:9092"]
+ }
+ }
+ """));
+
+ assertEquals("broker-1:9092,broker-2:9092,broker-3:9092", manager.resolveBootstrapServers(pipe));
+ }
+
+ @Test
+ void resolveBootstrapServersForManagedKafkaUsesMskService() throws Exception {
+ Pipe pipe = new Pipe();
+ pipe.setSource("arn:aws:kafka:us-east-1:000000000000:cluster/demo/uuid");
+ pipe.setSourceParameters(MAPPER.readTree("""
+ {
+ "ManagedStreamingKafkaParameters": {
+ "TopicName": "orders"
+ }
+ }
+ """));
+ when(mskService.getBootstrapBrokers(pipe.getSource())).thenReturn("localhost:19092");
+
+ assertEquals("localhost:19092", manager.resolveBootstrapServers(pipe));
+ }
+
+ @Test
+ void resolveTopicNameUsesMatchingParameterBlock() throws Exception {
+ Pipe managed = new Pipe();
+ managed.setSource("arn:aws:kafka:us-east-1:000000000000:cluster/demo/uuid");
+ managed.setSourceParameters(MAPPER.readTree("""
+ {
+ "ManagedStreamingKafkaParameters": {
+ "TopicName": "managed-topic"
+ }
+ }
+ """));
+
+ Pipe selfManaged = new Pipe();
+ selfManaged.setSource("smk://localhost:9092");
+ selfManaged.setSourceParameters(MAPPER.readTree("""
+ {
+ "SelfManagedKafkaParameters": {
+ "TopicName": "self-topic"
+ }
+ }
+ """));
+
+ assertEquals("managed-topic", manager.resolveTopicName(managed));
+ assertEquals("self-topic", manager.resolveTopicName(selfManaged));
+ }
+
+ @Test
+ @DisplayName("resolveTopicName reports the matching parameter block")
+ void resolveTopicNameReportsMatchingParameterBlock() throws Exception {
+ Pipe managed = new Pipe();
+ managed.setSource("arn:aws:kafka:us-east-1:000000000000:cluster/demo/uuid");
+ managed.setSourceParameters(MAPPER.readTree("""
+ {
+ "ManagedStreamingKafkaParameters": {}
+ }
+ """));
+
+ Pipe selfManaged = new Pipe();
+ selfManaged.setSource("smk://localhost:9092");
+ selfManaged.setSourceParameters(MAPPER.readTree("""
+ {
+ "SelfManagedKafkaParameters": {}
+ }
+ """));
+
+ assertEquals("SourceParameters.ManagedStreamingKafkaParameters.TopicName is required",
+ assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,
+ () -> manager.resolveTopicName(managed)).getMessage());
+ assertEquals("SourceParameters.SelfManagedKafkaParameters.TopicName is required",
+ assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,
+ () -> manager.resolveTopicName(selfManaged)).getMessage());
+ }
+
+ @Test
+ void resolveTopicNameReportsMissingParameterBlock() {
+ Pipe managed = new Pipe();
+ managed.setSource("arn:aws:kafka:us-east-1:000000000000:cluster/demo/uuid");
+ managed.setSourceParameters(MAPPER.createObjectNode());
+
+ Pipe selfManaged = new Pipe();
+ selfManaged.setSource("smk://localhost:9092");
+ selfManaged.setSourceParameters(MAPPER.createObjectNode());
+
+ assertEquals("SourceParameters.ManagedStreamingKafkaParameters is required",
+ assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,
+ () -> manager.resolveTopicName(managed)).getMessage());
+ assertEquals("SourceParameters.SelfManagedKafkaParameters is required",
+ assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,
+ () -> manager.resolveTopicName(selfManaged)).getMessage());
+ }
+
+ @Test
+ void resolveConsumerGroupIdUsesConfiguredValueOrStableFallback() throws Exception {
+ Pipe configured = new Pipe();
+ configured.setName("orders-pipe");
+ configured.setSource("smk://broker-1:9092");
+ configured.setSourceParameters(MAPPER.readTree("""
+ {
+ "SelfManagedKafkaParameters": {
+ "TopicName": "orders",
+ "ConsumerGroupID": "custom-group"
+ }
+ }
+ """));
+
+ Pipe fallback = new Pipe();
+ fallback.setName("orders-pipe");
+ fallback.setSource("arn:aws:kafka:us-east-1:000000000000:cluster/demo/uuid");
+ fallback.setSourceParameters(MAPPER.readTree("""
+ {
+ "ManagedStreamingKafkaParameters": {
+ "TopicName": "orders"
+ }
+ }
+ """));
+
+ assertEquals("custom-group", manager.resolveConsumerGroupId(configured));
+
+ String fallbackGroupId = manager.resolveConsumerGroupId(fallback);
+ assertEquals(fallbackGroupId, manager.resolveConsumerGroupId(fallback));
+ org.junit.jupiter.api.Assertions.assertTrue(
+ fallbackGroupId.startsWith("floci-pipes-orders-pipe-"),
+ "Expected generated consumer group id prefix, got: " + fallbackGroupId);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java
new file mode 100644
index 0000000000..ec91255438
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java
@@ -0,0 +1,224 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import io.github.hectorvent.floci.services.pipes.model.Pipe;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.vertx.core.Vertx;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class PipesPollerTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @Mock private Vertx vertx;
+ @Mock private SqsService sqsService;
+ @Mock private KinesisService kinesisService;
+ @Mock private DynamoDbStreamService dynamoDbStreamService;
+ @Mock private PipesKafkaConsumerManager kafkaConsumerManager;
+ @Mock private PipesTargetInvoker targetInvoker;
+ @Mock private EmulatorConfig config;
+
+ private PipesPoller poller;
+
+ @BeforeEach
+ void setUp() {
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ poller = new PipesPoller(vertx, sqsService, kinesisService, dynamoDbStreamService,
+ kafkaConsumerManager, targetInvoker, new PipesFilterMatcher(MAPPER), MAPPER, config);
+ }
+
+ @Test
+ void pollKafka_filtersUsingDecodedPayloadButDeliversOriginalRecord() throws Exception {
+ Pipe pipe = selfManagedKafkaPipe();
+ byte[] key = "customer-123".getBytes(StandardCharsets.UTF_8);
+ byte[] value = "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8);
+ ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>("orders", 0, 42L, key, value);
+ record.headers().add("traceId", "abc123".getBytes(StandardCharsets.UTF_8));
+
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));
+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
+
+ poller.pollKafka(pipe, "us-east-1");
+
+ ArgumentCaptor<String> payloadCaptor = ArgumentCaptor.forClass(String.class);
+ verify(targetInvoker).invoke(eq(pipe), payloadCaptor.capture(), eq("us-east-1"));
+ verify(kafkaConsumerManager).commit(eq(pipe), anyMap());
+
+ JsonNode delivered = MAPPER.readTree(payloadCaptor.getValue());
+ assertEquals("orders", delivered.path("topic").asText());
+ assertTrue(delivered.path("partition").isInt());
+ assertEquals("broker-1:9092", delivered.path("bootstrapServers").asText());
+ assertEquals("Y3VzdG9tZXItMTIz", delivered.path("key").asText());
+ assertEquals("eyJzdGF0dXMiOiJhY3RpdmUiLCJpZCI6Im9yZGVyLTEifQ==", delivered.path("value").asText());
+ assertTrue(delivered.path("headers").get(0).path("traceId").isArray());
+ assertEquals(List.of(97, 98, 99, 49, 50, 51),
+ MAPPER.convertValue(delivered.path("headers").get(0).path("traceId"), List.class));
+ assertFalse(delivered.has("eventSourceKey"));
+ assertTrue(delivered.path("value").isTextual());
+ }
+
+ @Test
+ void pollKafka_doesNotCommitWhenDeliveryFails() throws Exception {
+ Pipe pipe = selfManagedKafkaPipe();
+ ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>(
+ "orders", 0, 7L, null, "{\"status\":\"active\"}".getBytes(StandardCharsets.UTF_8));
+
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));
+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
+ org.mockito.Mockito.doThrow(new RuntimeException("boom"))
+ .when(targetInvoker).invoke(eq(pipe), anyString(), eq("us-east-1"));
+
+ poller.pollKafka(pipe, "us-east-1");
+
+ verify(kafkaConsumerManager, never()).commit(pipe);
+ verify(kafkaConsumerManager, never()).commit(eq(pipe), anyMap());
+ }
+
+ @Test
+ void pollKafka_commitsDeliveredPrefixWhenLaterRecordFails() throws Exception {
+ Pipe pipe = selfManagedKafkaPipe();
+ ConsumerRecord<byte[], byte[]> first = new ConsumerRecord<>(
+ "orders", 0, 0L, null, "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8));
+ ConsumerRecord<byte[], byte[]> second = new ConsumerRecord<>(
+ "orders", 0, 1L, null, "{\"status\":\"active\",\"id\":\"order-2\"}".getBytes(StandardCharsets.UTF_8));
+
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(records(first, second));
+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
+ org.mockito.Mockito.doNothing()
+ .doThrow(new RuntimeException("boom"))
+ .when(targetInvoker).invoke(eq(pipe), anyString(), eq("us-east-1"));
+
+ poller.pollKafka(pipe, "us-east-1");
+
+ ArgumentCaptor<Map<TopicPartition, OffsetAndMetadata>> offsetsCaptor = ArgumentCaptor.forClass(Map.class);
+ verify(kafkaConsumerManager).commit(eq(pipe), offsetsCaptor.capture());
+ assertEquals(1L, offsetsCaptor.getValue().get(new TopicPartition("orders", 0)).offset());
+ }
+
+ @Test
+ void pollKafka_representsNullKeyAndValueAsJsonNull() throws Exception {
+ Pipe pipe = nullableSelfManagedKafkaPipe();
+ ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>("orders", 0, 3L, null, null);
+
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));
+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
+
+ poller.pollKafka(pipe, "us-east-1");
+
+ ArgumentCaptor<String> payloadCaptor = ArgumentCaptor.forClass(String.class);
+ verify(targetInvoker).invoke(eq(pipe), payloadCaptor.capture(), eq("us-east-1"));
+ JsonNode delivered = MAPPER.readTree(payloadCaptor.getValue());
+ assertTrue(delivered.path("key").isNull());
+ assertTrue(delivered.path("value").isNull());
+ }
+
+ @Test
+ void pollKafka_lambdaCommitsSuccessfulPrefixBeforeLaterFailure() throws Exception {
+ Pipe pipe = lambdaSelfManagedKafkaPipe();
+ ConsumerRecord<byte[], byte[]> first = new ConsumerRecord<>(
+ "orders", 0, 0L, null, "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8));
+ ConsumerRecord<byte[], byte[]> skipped = new ConsumerRecord<>(
+ "orders", 0, 1L, null, "{\"status\":\"inactive\",\"id\":\"order-2\"}".getBytes(StandardCharsets.UTF_8));
+ ConsumerRecord<byte[], byte[]> failing = new ConsumerRecord<>(
+ "orders", 0, 2L, null, "{\"status\":\"active\",\"id\":\"order-3\"}".getBytes(StandardCharsets.UTF_8));
+
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(records(first, skipped, failing));
+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
+ org.mockito.Mockito.doNothing()
+ .doThrow(new RuntimeException("boom"))
+ .when(targetInvoker).invoke(eq(pipe), anyString(), eq("us-east-1"));
+
+ poller.pollKafka(pipe, "us-east-1");
+
+ ArgumentCaptor<Map<TopicPartition, OffsetAndMetadata>> offsetsCaptor = ArgumentCaptor.forClass(Map.class);
+ verify(kafkaConsumerManager).commit(eq(pipe), offsetsCaptor.capture());
+ assertEquals(2L, offsetsCaptor.getValue().get(new TopicPartition("orders", 0)).offset());
+ }
+
+ private Pipe selfManagedKafkaPipe() throws Exception {
+ Pipe pipe = new Pipe();
+ pipe.setName("orders-pipe");
+ pipe.setArn("arn:aws:pipes:us-east-1:000000000000:pipe/orders-pipe");
+ pipe.setSource("smk://broker-1:9092");
+ pipe.setTarget("arn:aws:sqs:us-east-1:000000000000:orders-target");
+ pipe.setSourceParameters(MAPPER.readTree("""
+ {
+ "SelfManagedKafkaParameters": {
+ "TopicName": "orders"
+ },
+ "FilterCriteria": {
+ "Filters": [
+ {"Pattern": "{\\\"value\\\": {\\\"status\\\": [\\\"active\\\"]}}"}
+ ]
+ }
+ }
+ """));
+ return pipe;
+ }
+
+ private Pipe nullableSelfManagedKafkaPipe() throws Exception {
+ Pipe pipe = new Pipe();
+ pipe.setName("nullable-orders-pipe");
+ pipe.setArn("arn:aws:pipes:us-east-1:000000000000:pipe/nullable-orders-pipe");
+ pipe.setSource("smk://broker-1:9092");
+ pipe.setTarget("arn:aws:sqs:us-east-1:000000000000:orders-target");
+ pipe.setSourceParameters(MAPPER.readTree("""
+ {
+ "SelfManagedKafkaParameters": {
+ "TopicName": "orders"
+ },
+ "FilterCriteria": {
+ "Filters": [
+ {"Pattern": "{\\\"key\\\": [{\\\"exists\\\": false}]}"}
+ ]
+ }
+ }
+ """));
+ return pipe;
+ }
+
+ private Pipe lambdaSelfManagedKafkaPipe() throws Exception {
+ Pipe pipe = selfManagedKafkaPipe();
+ pipe.setTarget("arn:aws:lambda:us-east-1:000000000000:function:orders-target");
+ return pipe;
+ }
+
+ private ConsumerRecords<byte[], byte[]> records(ConsumerRecord<byte[], byte[]>... records) {
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> byPartition = new java.util.LinkedHashMap<>();
+ for (ConsumerRecord<byte[], byte[]> record : records) {
+ TopicPartition topicPartition = new TopicPartition(record.topic(), record.partition());
+ byPartition.computeIfAbsent(topicPartition, ignored -> new java.util.ArrayList<>()).add(record);
+ }
+ return new ConsumerRecords<>(byPartition);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java
index 36cd367122..619ecef9b3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java
@@ -107,6 +107,33 @@ class PipesServiceTest {
assertEquals("ValidationException", ex.getErrorCode());
}
+ @Test
+ void createPipeWithManagedKafkaSourceRequiresTopicParameters() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ pipesService.createPipe("msk-pipe",
+ "arn:aws:kafka:us-east-1:000000000000:cluster/test/uuid",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null, null, null, null, null, "us-east-1"));
+
+ assertEquals("ValidationException", ex.getErrorCode());
+ }
+
+ @Test
+ void createPipeWithSelfManagedKafkaSourceRequiresTopicParameters() throws Exception {
+ var mapper = new com.fasterxml.jackson.databind.ObjectMapper();
+ AwsException ex = assertThrows(AwsException.class, () ->
+ pipesService.createPipe("smk-pipe",
+ "smk://localhost:9092",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null,
+ mapper.readTree("{\"SelfManagedKafkaParameters\":{}}"),
+ null, null, null, "us-east-1"));
+
+ assertEquals("ValidationException", ex.getErrorCode());
+ }
+
@Test
void describePipe() {
pipesService.createPipe("my-pipe",