Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(pipes): replace kafka-clients with a Kafka REST Proxy client (#3160)

Priamo Germosencd2847dba0af
diff --git a/docs/services/pipes.md b/docs/services/pipes.md
index b1948b1c4d..c143c2b458 100644
--- a/docs/services/pipes.md
+++ b/docs/services/pipes.md
@@ -20,6 +20,11 @@
| Variable | Default | Description |
|---|---|---|
| `FLOCI_SERVICES_PIPES_ENABLED` | `true` | Enable or disable the service |
+| `FLOCI_SERVICES_PIPES_KAFKA_REST_BRIDGE_DEFAULT_IMAGE` | `ghcr.io/aiven-open/karapace:latest` | Docker image for the Karapace REST Proxy sidecar a Kafka-sourced pipe starts on demand |
+| `FLOCI_SERVICES_PIPES_KAFKA_REST_BRIDGE_HOST_PORT_BASE` | `9500` | Start of the host port range allocated to Karapace sidecars |
+| `FLOCI_SERVICES_PIPES_KAFKA_REST_BRIDGE_HOST_PORT_MAX` | `9599` | End of the host port range allocated to Karapace sidecars |
+
+A pipe with a Kafka source (MSK or self-managed via `smk://`) starts a Karapace REST Proxy sidecar container on first use, one per distinct `bootstrap.servers` target shared across every pipe reading it, so Docker is required for those pipes even though it is not for the others.
## Examples
diff --git a/pom.xml b/pom.xml
index 1e8c0dc08a..f1ee4dcd07 100644
--- a/pom.xml
+++ b/pom.xml
@@ -158,13 +158,6 @@
<version>9.2.1</version>
</dependency>
- <!-- Kafka client for MSK/self-managed Kafka pipe polling -->
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>4.3.1</version>
- </dependency>
-
<!-- Snappy for Firehose Snappy/HADOOP_SNAPPY delivery to S3: the only Java
library that writes the xerial stream format AWS uses. -->
<dependency>
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index fa837bcf8f..ee11ef6cd0 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -2192,6 +2192,25 @@ public interface EmulatorConfig {
interface PipesServiceConfig {
@WithDefault("true")
boolean enabled();
+
+ KafkaRestBridgeConfig kafkaRestBridge();
+ }
+
+ /**
+ * The Karapace sidecar Pipes launches, on demand, to poll a Kafka source over REST instead of
+ * embedding kafka-clients in Floci itself. One container is started per distinct target
+ * {@code bootstrap.servers} (a self-managed cluster, or an MSK cluster's Redpanda backing), and
+ * reused across pipes that share the same source.
+ */
+ interface KafkaRestBridgeConfig {
+ @WithDefault("ghcr.io/aiven-open/karapace:latest")
+ String defaultImage();
+
+ @WithDefault("9500")
+ int hostPortBase();
+
+ @WithDefault("9599")
+ int hostPortMax();
}
interface BedrockAgentCoreControlServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaConsumerHandle.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaConsumerHandle.java
new file mode 100644
index 0000000000..1aaed74cb9
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaConsumerHandle.java
@@ -0,0 +1,7 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import java.net.URI;
+
+/** A Kafka REST Proxy consumer instance: its own {@code base_uri} addresses every later call. */
+record KafkaConsumerHandle(String instanceId, URI baseUri) {
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaHeaderDto.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaHeaderDto.java
new file mode 100644
index 0000000000..7fb408c430
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaHeaderDto.java
@@ -0,0 +1,5 @@
+package io.github.hectorvent.floci.services.pipes;
+
+/** One Kafka record header, as decoded from a Kafka REST Proxy record response. */
+record KafkaHeaderDto(String key, byte[] value) {
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaOffsetDto.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaOffsetDto.java
new file mode 100644
index 0000000000..ffbcfecab1
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaOffsetDto.java
@@ -0,0 +1,10 @@
+package io.github.hectorvent.floci.services.pipes;
+
+/**
+ * A partition offset to commit through a Kafka REST Proxy, in place of
+ * {@code org.apache.kafka.common.TopicPartition} / {@code OffsetAndMetadata}. As with the native
+ * client, {@code offset} is the next offset to consume (last delivered offset + 1), not the last
+ * delivered offset itself.
+ */
+record KafkaOffsetDto(String topic, int partition, long offset) {
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaRecordDto.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaRecordDto.java
new file mode 100644
index 0000000000..49342a3b39
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaRecordDto.java
@@ -0,0 +1,26 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import java.util.List;
+
+/**
+ * One Kafka record as returned by a Kafka REST Proxy (Karapace, Pandaproxy) consumer poll, in
+ * place of {@code org.apache.kafka.clients.consumer.ConsumerRecord} so Floci's own process never
+ * needs {@code kafka-clients} on its classpath. See #2916.
+ *
+ * <p>{@code timestampType} is not part of the REST Proxy record schema (Karapace reads it
+ * internally but does not expose it), so it is always reported as {@code CreateTime}, the default
+ * for any topic that does not explicitly configure {@code message.timestamp.type}. This is a
+ * documented, intentional deviation from the native Kafka protocol's per-record fidelity.
+ */
+record KafkaRecordDto(
+ String topic,
+ int partition,
+ long offset,
+ long timestamp,
+ String timestampType,
+ byte[] key,
+ byte[] value,
+ List<KafkaHeaderDto> headers) {
+
+ static final String DEFAULT_TIMESTAMP_TYPE = "CreateTime";
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaTopicPartition.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaTopicPartition.java
new file mode 100644
index 0000000000..ac053058ad
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaTopicPartition.java
@@ -0,0 +1,5 @@
+package io.github.hectorvent.floci.services.pipes;
+
+/** A Kafka topic/partition pair, used as a map key when grouping records for delivery. */
+record KafkaTopicPartition(String topic, int partition) {
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KarapaceManager.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KarapaceManager.java
new file mode 100644
index 0000000000..3054d6bc78
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KarapaceManager.java
@@ -0,0 +1,228 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
+import io.github.hectorvent.floci.core.common.docker.PortAllocator;
+import jakarta.annotation.PreDestroy;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+/**
+ * Starts, on demand, a Karapace REST Proxy sidecar fronting a Kafka {@code bootstrap.servers}
+ * target, so Pipes can poll a Kafka source over plain HTTP ({@link PipesKafkaRestClient}) instead
+ * of embedding {@code kafka-clients} in Floci itself. See #2916.
+ *
+ * <p>One container is started per distinct target and reused for the life of this process:
+ * {@link #ensureStarted} is safe to call concurrently, including repeatedly for the same target
+ * (multiple Pipes reading the same cluster share one Karapace instance).
+ */
+@ApplicationScoped
+class KarapaceManager {
+
+ private static final Logger LOG = Logger.getLogger(KarapaceManager.class);
+ private static final int REST_PORT = 8082;
+ private static final Duration READY_TIMEOUT = Duration.ofSeconds(30);
+
+ private final ContainerBuilder containerBuilder;
+ private final ContainerLifecycleManager lifecycleManager;
+ private final ContainerDetector containerDetector;
+ private final EmulatorConfig config;
+ private final PortAllocator portAllocator;
+ private final DockerHostResolver dockerHostResolver;
+ private final Duration readyTimeout;
+ private final Map<String, URI> restBaseUriByTarget = new ConcurrentHashMap<>();
+ private final Map<String, String> containerIdByTarget = new ConcurrentHashMap<>();
+
+ @Inject
+ KarapaceManager(ContainerBuilder containerBuilder,
+ ContainerLifecycleManager lifecycleManager,
+ ContainerDetector containerDetector,
+ EmulatorConfig config,
+ PortAllocator portAllocator,
+ DockerHostResolver dockerHostResolver) {
+ this(containerBuilder, lifecycleManager, containerDetector, config, portAllocator, dockerHostResolver,
+ READY_TIMEOUT);
+ }
+
+ /** Test-only: overrides the readiness timeout so a failure case does not wait the full default. */
+ KarapaceManager(ContainerBuilder containerBuilder,
+ ContainerLifecycleManager lifecycleManager,
+ ContainerDetector containerDetector,
+ EmulatorConfig config,
+ PortAllocator portAllocator,
+ DockerHostResolver dockerHostResolver,
+ Duration readyTimeout) {
+ this.containerBuilder = containerBuilder;
+ this.lifecycleManager = lifecycleManager;
+ this.containerDetector = containerDetector;
+ this.config = config;
+ this.portAllocator = portAllocator;
+ this.dockerHostResolver = dockerHostResolver;
+ this.readyTimeout = readyTimeout;
+ }
+
+ URI ensureStarted(String bootstrapServers) {
+ return restBaseUriByTarget.computeIfAbsent(bootstrapServers, this::startContainer);
+ }
+
+ private URI startContainer(String bootstrapServers) {
+ EmulatorConfig.KafkaRestBridgeConfig bridgeConfig = config.services().pipes().kafkaRestBridge();
+ String targetKey = targetKey(bootstrapServers);
+ String containerName = ContainerStorageHelper.resourceName(config, "pipes-kafka-rest", targetKey, targetKey);
+
+ lifecycleManager.removeIfExists(containerName);
+
+ List<String> env = List.of(
+ "KARAPACE_HOST=0.0.0.0",
+ "KARAPACE_PORT=" + REST_PORT,
+ "KARAPACE_BOOTSTRAP_URI=" + resolveForContainer(bootstrapServers),
+ "KARAPACE_ADVERTISED_PROTOCOL=http");
+
+ ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(bridgeConfig.defaultImage())
+ .withName(containerName)
+ .withDockerNetwork(config.services().dockerNetwork())
+ // Native Linux Docker does not auto-inject host.docker.internal the way Docker
+ // Desktop does, and resolveForContainer rewrites a host-local broker to exactly
+ // that alias, so the sidecar needs the host-gateway mapping to resolve it.
+ .withHostDockerInternalOnLinux()
+ .withLogRotation()
+ // The image's default entrypoint (plain `python3`) does nothing; karapace_rest_proxy
+ // is the REST Proxy's own dedicated process, config-driven entirely by KARAPACE_* env.
+ .withEntrypoint(List.of("/venv/bin/karapace_rest_proxy"))
+ .withEnv(env)
+ .withLabels(Map.of("floci.component", "pipes-kafka-rest-bridge"));
+
+ int hostPort = 0;
+ if (!containerDetector.isRunningInContainer()) {
+ hostPort = portAllocator.allocate(bridgeConfig.hostPortBase(), bridgeConfig.hostPortMax());
+ specBuilder.withPortBinding(REST_PORT, hostPort);
+ } else {
+ specBuilder.withExposedPort(REST_PORT);
+ }
+
+ ContainerSpec spec = specBuilder.build();
+ ContainerInfo info;
+ try {
+ info = lifecycleManager.createAndStart(spec);
+ } catch (RuntimeException e) {
+ if (!containerDetector.isRunningInContainer()) {
+ portAllocator.release(hostPort);
+ }
+ throw e;
+ }
+
+ try {
+ containerIdByTarget.put(bootstrapServers, info.containerId());
+ EndpointInfo endpoint = info.getEndpoint(REST_PORT);
+ URI restBaseUri = URI.create("http://" + endpoint.host() + ":" + endpoint.port() + "/");
+ waitUntilReady(restBaseUri);
+ LOG.infov("Karapace REST Proxy {0} started for bootstrap {1}", info.containerId(), bootstrapServers);
+ return restBaseUri;
+ } catch (RuntimeException e) {
+ containerIdByTarget.remove(bootstrapServers);
+ lifecycleManager.stopAndRemove(info.containerId(), null);
+ if (!containerDetector.isRunningInContainer()) {
+ portAllocator.release(hostPort);
+ }
+ throw e;
+ }
+ }
+
+ /**
+ * Rewrites a host-local address (localhost/127.0.0.1) so the sidecar, in its own network
+ * namespace, can reach a broker on the Floci host instead of trying to reach itself. Applies
+ * to both self-managed sources given as {@code smk://localhost:9092} and, in native mode, an
+ * MSK-backed Redpanda cluster whose advertised address is also {@code localhost:<port>}.
+ */
+ private String resolveForContainer(String bootstrapServers) {
+ return Arrays.stream(bootstrapServers.split(","))
+ .map(this::resolveHostLocalAddress)
+ .collect(Collectors.joining(","));
+ }
+
+ private String resolveHostLocalAddress(String hostPort) {
+ int colonIndex = hostPort.lastIndexOf(':');
+ if (colonIndex < 0) {
+ return hostPort;
+ }
+ String host = hostPort.substring(0, colonIndex);
+ if (host.equals("localhost") || host.equals("127.0.0.1")) {
+ return dockerHostResolver.resolve() + hostPort.substring(colonIndex);
+ }
+ return hostPort;
+ }
+
+ private void waitUntilReady(URI restBaseUri) {
+ long deadline = System.nanoTime() + readyTimeout.toNanos();
+ while (System.nanoTime() < deadline) {
+ if (isReady(restBaseUri)) {
+ return;
+ }
+ try {
+ Thread.sleep(Math.min(200, Math.max(1, readyTimeout.toMillis() / 4)));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("Interrupted waiting for Karapace to become ready at " + restBaseUri, e);
+ }
+ }
+ throw new IllegalStateException("Karapace REST Proxy at " + restBaseUri + " did not become ready within "
+ + readyTimeout.toMillis() + "ms");
+ }
+
+ private boolean isReady(URI restBaseUri) {
+ try {
+ HttpURLConnection conn = (HttpURLConnection) restBaseUri.resolve("_health").toURL().openConnection();
+ conn.setRequestMethod("GET");
+ conn.setConnectTimeout(1000);
+ conn.setReadTimeout(1000);
+ return conn.getResponseCode() == 200;
+ } catch (Exception e) {
+ LOG.debugv("Karapace not ready yet at {0}: {1}", restBaseUri, e.getMessage());
+ return false;
+ }
+ }
+
+ private static String targetKey(String bootstrapServers) {
+ try {
+ MessageDigest digest = MessageDigest.getInstance("SHA-256");
+ byte[] hash = digest.digest(bootstrapServers.getBytes(StandardCharsets.UTF_8));
+ return HexFormat.of().formatHex(hash, 0, 8);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("SHA-256 is not available", e);
+ }
+ }
+
+ @PreDestroy
+ void shutdown() {
+ containerIdByTarget.values().forEach(containerId -> {
+ try {
+ lifecycleManager.stopAndRemove(containerId, null);
+ } catch (RuntimeException e) {
+ LOG.warnv("Failed to stop Karapace container {0}: {1}", containerId, e.getMessage());
+ }
+ });
+ containerIdByTarget.clear();
+ restBaseUriByTarget.clear();
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java
index 6962e7e12d..c30c092fcf 100644
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java
@@ -8,22 +8,21 @@ import io.github.hectorvent.floci.services.pipes.model.Pipe;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.serialization.ByteArrayDeserializer;
import org.jboss.logging.Logger;
+import java.net.URI;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+/**
+ * Polls Kafka pipe sources (MSK and self-managed, {@code smk://}) over a Kafka REST Proxy
+ * (Karapace) instead of embedding {@code kafka-clients} directly: see #2916, where kafka-clients
+ * and the zstd-jni/lz4-java it pulls in were reachable in every build regardless of whether a
+ * Pipe ever used a Kafka source. {@link KarapaceManager} starts a Karapace sidecar per distinct
+ * target on demand, and {@link PipesKafkaRestClient} speaks its REST API.
+ */
@ApplicationScoped
public class PipesKafkaConsumerManager {
@@ -32,11 +31,16 @@ public class PipesKafkaConsumerManager {
private static final String DEFAULT_STARTING_POSITION = "LATEST";
private final MskService mskService;
- private final ConcurrentHashMap<String, KafkaConsumer<byte[], byte[]>> consumers = new ConcurrentHashMap<>();
+ private final KarapaceManager karapaceManager;
+ private final PipesKafkaRestClient restClient;
+ private final ConcurrentHashMap<String, KafkaConsumerHandle> consumers = new ConcurrentHashMap<>();
@Inject
- public PipesKafkaConsumerManager(MskService mskService) {
+ public PipesKafkaConsumerManager(MskService mskService, KarapaceManager karapaceManager,
+ PipesKafkaRestClient restClient) {
this.mskService = mskService;
+ this.karapaceManager = karapaceManager;
+ this.restClient = restClient;
}
@PreDestroy
@@ -45,39 +49,26 @@ public class PipesKafkaConsumerManager {
consumers.clear();
}
- public ConsumerRecords<byte[], byte[]> poll(Pipe pipe) {
- KafkaConsumer<byte[], byte[]> consumer = consumers.computeIfAbsent(pipe.getArn(), ignored -> createConsumer(pipe));
+ public List<KafkaRecordDto> poll(Pipe pipe) {
+ KafkaConsumerHandle consumer = consumers.computeIfAbsent(pipe.getArn(), ignored -> createConsumer(pipe));
try {
- return consumer.poll(POLL_TIMEOUT);
+ return restClient.poll(consumer, POLL_TIMEOUT);
} catch (RuntimeException e) {
close(pipe);
throw e;
}
}
- public void commit(Pipe pipe) {
- KafkaConsumer<byte[], byte[]> consumer = consumers.get(pipe.getArn());
- if (consumer == null) {
- return;
- }
- try {
- consumer.commitSync();
- } catch (RuntimeException e) {
- close(pipe);
- throw e;
- }
- }
-
- public void commit(Pipe pipe, Map<TopicPartition, OffsetAndMetadata> offsets) {
+ public void commit(Pipe pipe, List<KafkaOffsetDto> offsets) {
if (offsets.isEmpty()) {
return;
}
- KafkaConsumer<byte[], byte[]> consumer = consumers.get(pipe.getArn());
+ KafkaConsumerHandle consumer = consumers.get(pipe.getArn());
if (consumer == null) {
return;
}
try {
- consumer.commitSync(offsets);
+ restClient.commit(consumer, offsets);
} catch (RuntimeException e) {
close(pipe);
throw e;
@@ -85,7 +76,7 @@ public class PipesKafkaConsumerManager {
}
public void close(Pipe pipe) {
- KafkaConsumer<byte[], byte[]> consumer = consumers.remove(pipe.getArn());
+ KafkaConsumerHandle consumer = consumers.remove(pipe.getArn());
if (consumer != null) {
closeQuietly(consumer);
}
@@ -131,34 +122,25 @@ public class PipesKafkaConsumerManager {
return resolveConsumerGroupId(pipe, kafkaParameters(pipe));
}
- /** Package-private so {@code PipesKafkaNativeSupportTest} guards the real settings, not a copy. */
- Properties consumerProperties(Pipe pipe) {
- JsonNode params = kafkaParameters(pipe);
-
- Properties properties = new Properties();
- properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, resolveBootstrapServers(pipe));
- properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
- properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
- properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
- properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, resolveOffsetReset(params));
- properties.put(ConsumerConfig.GROUP_ID_CONFIG, resolveConsumerGroupId(pipe, params));
- properties.put(ConsumerConfig.CLIENT_ID_CONFIG, "floci-pipes-" + UUID.randomUUID());
- properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, Integer.toString(resolveBatchSize(pipe, 100)));
- // On by default (KIP-714). Floci has no reason to push metrics to a user's broker, and it
- // makes the shaded protobuf serializer reflectively reachable on a path no broker we test
- // against exercises.
- properties.put(ConsumerConfig.ENABLE_METRICS_PUSH_CONFIG, "false");
- return properties;
- }
-
- private KafkaConsumer<byte[], byte[]> createConsumer(Pipe pipe) {
- Properties properties = consumerProperties(pipe);
+ private KafkaConsumerHandle createConsumer(Pipe pipe) {
+ String bootstrapServers = resolveBootstrapServers(pipe);
String topicName = resolveTopicName(pipe);
+ String groupId = resolveConsumerGroupId(pipe);
+ String offsetReset = resolveOffsetReset(kafkaParameters(pipe));
- KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(properties);
- consumer.subscribe(List.of(topicName));
- LOG.infov("Pipe {0}: subscribed Kafka consumer to topic {1} via {2}",
- pipe.getName(), topicName, properties.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG));
+ URI restBaseUri = karapaceManager.ensureStarted(bootstrapServers);
+ KafkaConsumerHandle consumer = restClient.createConsumer(restBaseUri, groupId, offsetReset);
+ try {
+ restClient.subscribe(consumer, List.of(topicName));
+ } catch (RuntimeException e) {
+ // The consumer already exists server-side; this method's caller only learns about it
+ // through the returned handle, so a failure here must delete it itself or it is never
+ // reachable again to close, and every retry creates another orphaned server consumer.
+ closeQuietly(consumer);
+ throw e;
+ }
+ LOG.infov("Pipe {0}: subscribed Kafka REST consumer to topic {1} via {2}",
+ pipe.getName(), topicName, bootstrapServers);
return consumer;
}
@@ -210,11 +192,11 @@ public class PipesKafkaConsumerManager {
throw new AwsException("ValidationException", "Unsupported Kafka source: " + pipe.getSource(), 400);
}
- private void closeQuietly(KafkaConsumer<byte[], byte[]> consumer) {
+ private void closeQuietly(KafkaConsumerHandle consumer) {
try {
- consumer.close();
+ restClient.close(consumer);
} catch (Exception e) {
- LOG.debugv("Ignoring Kafka consumer close error: {0}", e.getMessage());
+ LOG.debugv("Ignoring Kafka REST consumer close error: {0}", e.getMessage());
}
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.java
deleted file mode 100644
index 3fcf99168a..0000000000
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package io.github.hectorvent.floci.services.pipes;
-
-import io.quarkus.runtime.annotations.RegisterForReflection;
-import org.apache.kafka.clients.consumer.CooperativeStickyAssignor;
-import org.apache.kafka.clients.consumer.RangeAssignor;
-import org.apache.kafka.common.metrics.JmxReporter;
-import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-
-/**
- * Registers the kafka-clients classes that GraalVM cannot discover statically.
- *
- * <p>kafka-clients ships no native-image metadata and Floci uses it directly rather than through
- * the Quarkus Kafka extension. {@code AbstractConfig} resolves {@code CLASS}-typed settings by
- * name, so without these a native build cannot construct a consumer — which {@link PipesPoller}
- * swallows per poll cycle, leaving the pipe RUNNING and delivering nothing.
- *
- * <p>Entries are exactly what {@link PipesKafkaConsumerManager#consumerProperties} resolves, and
- * {@code PipesKafkaNativeSupportTest} reads that same method, so the two cannot drift.
- *
- * <p>SASL and SSL defaults are excluded: Floci is PLAINTEXT-only, and {@code DefaultJwtValidator}
- * fails the native build by referencing jose4j, an optional dependency Floci omits.
- */
-@RegisterForReflection(targets = {
- // key.deserializer / value.deserializer — set explicitly by PipesKafkaConsumerManager
- ByteArrayDeserializer.class,
- // partition.assignment.strategy — ConsumerConfig defaults
- RangeAssignor.class,
- CooperativeStickyAssignor.class,
- // metric.reporters — ConsumerConfig default
- JmxReporter.class
-})
-public class PipesKafkaNativeSupport {
-}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClient.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClient.java
new file mode 100644
index 0000000000..6e4cbf14b8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClient.java
@@ -0,0 +1,186 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.net.URLEncoder;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+import java.util.concurrent.Executors;
+
+/**
+ * Thin REST client for the subset of the Kafka REST Proxy v2 API (Karapace, Redpanda's
+ * Pandaproxy) that Pipes needs to consume a Kafka topic: create a consumer instance, subscribe,
+ * poll for records, commit offsets, close. Uses the JDK {@link HttpClient}, the same approach
+ * Floci already uses for other REST-shaped backends (e.g. {@code KubernetesApiClient}), instead
+ * of embedding {@code kafka-clients} directly: see #2916, where kafka-clients (and the
+ * zstd-jni/lz4-java it pulls in) was reachable in every native image regardless of whether a Pipe
+ * ever used a Kafka source.
+ *
+ * <p>Every consumer instance lives on exactly one REST Proxy process for its whole lifetime (the
+ * proxy holds the actual Kafka client state), so all calls for a given {@link KafkaConsumerHandle}
+ * must keep going to the {@code base_uri} that instance's creation call returned.
+ */
+@ApplicationScoped
+class PipesKafkaRestClient {
+
+ private static final String CONSUMER_CONTENT_TYPE = "application/vnd.kafka.v2+json";
+ private static final String BINARY_RECORDS_ACCEPT = "application/vnd.kafka.binary.v2+json";
+
+ private final ObjectMapper objectMapper;
+ private final HttpClient httpClient;
+
+ @Inject
+ PipesKafkaRestClient(ObjectMapper objectMapper) {
+ this(objectMapper, HttpClient.newBuilder()
+ .connectTimeout(Duration.ofSeconds(10))
+ .executor(Executors.newVirtualThreadPerTaskExecutor())
+ .build());
+ }
+
+ /** Test-only: injects an explicit client, e.g. one pointed at a fake REST Proxy server. */
+ PipesKafkaRestClient(ObjectMapper objectMapper, HttpClient httpClient) {
+ this.objectMapper = objectMapper;
+ this.httpClient = httpClient;
+ }
+
+ KafkaConsumerHandle createConsumer(URI restBaseUri, String groupId, String offsetReset) {
+ ObjectNode body = objectMapper.createObjectNode();
+ body.put("format", "binary");
+ body.put("auto.offset.reset", offsetReset);
+ body.put("auto.commit.enable", "false");
+
+ JsonNode response = send("POST", restBaseUri.resolve("consumers/" + urlEncode(groupId)), body);
+ // Karapace returns base_uri as a path relative to the proxy root, not an absolute URL,
+ // unlike the Confluent REST Proxy reference spec; resolve() handles both correctly.
+ return new KafkaConsumerHandle(
+ response.path("instance_id").asText(),
+ restBaseUri.resolve(response.path("base_uri").asText()));
+ }
+
+ void subscribe(KafkaConsumerHandle consumer, List<String> topics) {
+ ObjectNode body = objectMapper.createObjectNode();
+ ArrayNode topicsNode = body.putArray("topics");
+ topics.forEach(topicsNode::add);
+ send("POST", instanceUri(consumer, "subscription"), body);
+ }
+
+ List<KafkaRecordDto> poll(KafkaConsumerHandle consumer, Duration timeout) {
+ URI uri = URI.create(instanceUri(consumer, "records") + "?timeout=" + timeout.toMillis());
+ JsonNode response = sendWithAccept("GET", uri, null, BINARY_RECORDS_ACCEPT);
+
+ List<KafkaRecordDto> records = new ArrayList<>();
+ response.forEach(node -> records.add(toRecord(node)));
+ return records;
+ }
+
+ /**
+ * Commits the given offsets. Karapace requires an explicit offsets list (unlike the Confluent
+ * REST Proxy reference spec, an empty body is rejected with 400 rather than committing
+ * everything consumed), so callers always compute what to commit themselves.
+ */
+ void commit(KafkaConsumerHandle consumer, List<KafkaOffsetDto> offsets) {
+ if (offsets.isEmpty()) {
+ return;
+ }
+ ObjectNode body = objectMapper.createObjectNode();
+ ArrayNode offsetsNode = body.putArray("offsets");
+ for (KafkaOffsetDto offset : offsets) {
+ ObjectNode offsetNode = offsetsNode.addObject();
+ offsetNode.put("topic", offset.topic());
+ offsetNode.put("partition", offset.partition());
+ offsetNode.put("offset", offset.offset());
+ }
+ send("POST", instanceUri(consumer, "offsets"), body);
+ }
+
+ /** No-op if the consumer instance is already gone (e.g. the REST Proxy container restarted). */
+ void close(KafkaConsumerHandle consumer) {
+ try {
+ send("DELETE", consumer.baseUri(), null);
+ } catch (PipesKafkaRestClientException e) {
+ if (e.getStatusCode() != 404) {
+ throw e;
+ }
+ }
+ }
+
+ private KafkaRecordDto toRecord(JsonNode node) {
+ List<KafkaHeaderDto> headers = new ArrayList<>();
+ for (JsonNode headerNode : node.path("headers")) {
+ headers.add(new KafkaHeaderDto(
+ headerNode.path("name").asText(),
+ base64Decode(headerNode.path("value"))));
+ }
+ return new KafkaRecordDto(
+ node.path("topic").asText(),
+ node.path("partition").asInt(),
+ node.path("offset").asLong(),
+ node.path("timestamp").asLong(0),
+ KafkaRecordDto.DEFAULT_TIMESTAMP_TYPE,
+ base64Decode(node.path("key")),
+ base64Decode(node.path("value")),
+ headers);
+ }
+
+ private static byte[] base64Decode(JsonNode value) {
+ if (value == null || value.isNull() || value.isMissingNode()) {
+ return null;
+ }
+ return Base64.getDecoder().decode(value.asText());
+ }
+
+ private URI instanceUri(KafkaConsumerHandle consumer, String segment) {
+ String base = consumer.baseUri().toString();
+ return URI.create((base.endsWith("/") ? base : base + "/") + segment);
+ }
+
+ private JsonNode send(String method, URI uri, JsonNode body) {
+ return sendWithAccept(method, uri, body, CONSUMER_CONTENT_TYPE);
+ }
+
+ private JsonNode sendWithAccept(String method, URI uri, JsonNode body, String accept) {
+ try {
+ HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
+ .timeout(Duration.ofSeconds(30))
+ .header("Accept", accept);
+ if (body != null) {
+ builder.header("Content-Type", CONSUMER_CONTENT_TYPE)
+ .method(method, HttpRequest.BodyPublishers.ofByteArray(objectMapper.writeValueAsBytes(body)));
+ } else {
+ builder.method(method, HttpRequest.BodyPublishers.noBody());
+ }
+
+ HttpResponse<byte[]> response = httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofByteArray());
+ if (response.statusCode() / 100 != 2) {
+ throw new PipesKafkaRestClientException(method, uri.toString(), response.statusCode(),
+ new String(response.body(), StandardCharsets.UTF_8));
+ }
+ return response.body().length == 0 ? NullNode.getInstance() : objectMapper.readTree(response.body());
+ } catch (IOException e) {
+ throw new UncheckedIOException(
+ "Kafka REST Proxy " + method + " " + uri + " failed: " + e.getMessage(), e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("Interrupted calling Kafka REST Proxy " + method + " " + uri, e);
+ }
+ }
+
+ private static String urlEncode(String value) {
+ return URLEncoder.encode(value, StandardCharsets.UTF_8);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientException.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientException.java
new file mode 100644
index 0000000000..fb499145f9
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientException.java
@@ -0,0 +1,17 @@
+package io.github.hectorvent.floci.services.pipes;
+
+/** A Kafka REST Proxy (Karapace) response outside the expected 2xx status for its call. */
+class PipesKafkaRestClientException extends RuntimeException {
+
+ private final int statusCode;
+
+ PipesKafkaRestClientException(String method, String path, int statusCode, String body) {
+ super("Kafka REST Proxy " + method + " " + path + " failed: HTTP " + statusCode
+ + (body == null || body.isBlank() ? "" : " " + body));
+ this.statusCode = statusCode;
+ }
+
+ int getStatusCode() {
+ return statusCode;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
index 7b9593ff00..c6e5c7e2a8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
@@ -19,11 +19,6 @@ import io.vertx.core.Vertx;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.header.Header;
import org.jboss.logging.Logger;
import java.nio.charset.StandardCharsets;
@@ -341,20 +336,41 @@ public class PipesPoller implements Resettable {
}
void pollKafka(Pipe pipe, String region) {
- ConsumerRecords<byte[], byte[]> records = kafkaConsumerManager.poll(pipe);
+ List<KafkaRecordDto> records = kafkaConsumerManager.poll(pipe);
if (records.isEmpty()) {
return;
}
- List<ConsumerRecord<byte[], byte[]>> batch = new ArrayList<>(records.count());
- records.forEach(batch::add);
+ LOG.infov("Pipe {0}: received {1} Kafka record(s)", pipe.getName(), records.size());
+ int batchSize = kafkaConsumerManager.resolveBatchSize(pipe, DEFAULT_BATCH_SIZE);
+ Set<KafkaTopicPartition> blockedPartitions = new HashSet<>();
+ for (List<KafkaRecordDto> batch : partition(records, batchSize)) {
+ deliverKafkaBatch(pipe, region, batch, blockedPartitions);
+ }
+ }
- LOG.infov("Pipe {0}: received {1} Kafka record(s)", pipe.getName(), batch.size());
- List<ObjectNode> deliveryNodes = buildKafkaRecordNodes(batch, pipe);
- List<ObjectNode> filterNodes = buildKafkaFilterNodes(deliveryNodes, batch);
+ /**
+ * A REST Proxy poll can return more records than one Pipe {@code BatchSize} worth (unlike the
+ * native consumer, it has no per-call record cap), so one poll cycle may deliver several
+ * batches in sequence. {@code blockedPartitions} is shared across that whole sequence: once a
+ * partition fails delivery in one batch, its records in every later batch this cycle are left
+ * uncommitted too, so a later batch's success can never commit past an earlier record that is
+ * still awaiting redelivery.
+ */
+ private void deliverKafkaBatch(Pipe pipe, String region, List<KafkaRecordDto> batch,
+ Set<KafkaTopicPartition> blockedPartitions) {
+ List<KafkaRecordDto> deliverable = batch.stream()
+ .filter(record -> !blockedPartitions.contains(new KafkaTopicPartition(record.topic(), record.partition())))
+ .toList();
+ if (deliverable.isEmpty()) {
+ return;
+ }
+
+ List<ObjectNode> deliveryNodes = buildKafkaRecordNodes(deliverable, pipe);
+ List<ObjectNode> filterNodes = buildKafkaFilterNodes(deliveryNodes, deliverable);
List<JsonNode> filtered = filterMatcher.applyFilterCriteria(new ArrayList<>(filterNodes), pipe.getSourceParameters());
if (filtered.isEmpty()) {
- kafkaConsumerManager.commit(pipe);
+ commitWholeBatch(pipe, deliverable);
return;
}
@@ -371,13 +387,13 @@ public class PipesPoller implements Resettable {
}
}
if (deliveryRecords.isEmpty()) {
- kafkaConsumerManager.commit(pipe);
+ commitWholeBatch(pipe, deliverable);
return;
}
int failed = isLambdaTarget(pipe)
- ? deliverKafkaLambdaRecords(pipe, region, batch, deliveryRecordsByIdentity, filtered)
- : deliverKafkaRecords(pipe, region, batch, deliveryRecordsByIdentity, filtered);
+ ? deliverKafkaLambdaRecords(pipe, region, deliverable, deliveryRecordsByIdentity, filtered, blockedPartitions)
+ : deliverKafkaRecords(pipe, region, deliverable, deliveryRecordsByIdentity, filtered, blockedPartitions);
if (failed == 0) {
return;
}
@@ -388,48 +404,51 @@ public class PipesPoller implements Resettable {
private int deliverKafkaLambdaRecords(Pipe pipe,
String region,
- List<ConsumerRecord<byte[], byte[]>> batch,
+ List<KafkaRecordDto> batch,
Map<String, JsonNode> deliveryRecordsByIdentity,
- List<JsonNode> filtered) {
+ List<JsonNode> filtered,
+ Set<KafkaTopicPartition> blockedPartitions) {
Set<String> matchedIdentities = new HashSet<>(filtered.size());
filtered.forEach(record -> matchedIdentities.add(kafkaRecordIdentity(record)));
- Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = groupKafkaRecordsByPartition(batch);
+ Map<KafkaTopicPartition, Long> offsetsToCommit = new HashMap<>();
+ Map<KafkaTopicPartition, List<KafkaRecordDto>> recordsByPartition = groupKafkaRecordsByPartition(batch);
int failed = 0;
- for (Map.Entry<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> entry : recordsByPartition.entrySet()) {
- failed += deliverKafkaLambdaPartition(pipe, region, entry.getKey(), entry.getValue(),
+ for (Map.Entry<KafkaTopicPartition, List<KafkaRecordDto>> entry : recordsByPartition.entrySet()) {
+ int partitionFailed = deliverKafkaLambdaPartition(pipe, region, entry.getKey(), entry.getValue(),
deliveryRecordsByIdentity, matchedIdentities, offsetsToCommit);
+ if (partitionFailed > 0) {
+ blockedPartitions.add(entry.getKey());
+ }
+ failed += partitionFailed;
}
- if (!offsetsToCommit.isEmpty()) {
- kafkaConsumerManager.commit(pipe, offsetsToCommit);
- }
+ commitOffsets(pipe, offsetsToCommit);
return failed;
}
private int deliverKafkaLambdaPartition(Pipe pipe,
String region,
- TopicPartition partition,
- List<ConsumerRecord<byte[], byte[]>> partitionRecords,
+ KafkaTopicPartition partition,
+ List<KafkaRecordDto> partitionRecords,
Map<String, JsonNode> deliveryRecordsByIdentity,
Set<String> matchedIdentities,
- Map<TopicPartition, OffsetAndMetadata> offsetsToCommit) {
+ Map<KafkaTopicPartition, Long> offsetsToCommit) {
List<JsonNode> pendingBatch = new ArrayList<>();
long pendingOffset = -1L;
- for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {
+ for (KafkaRecordDto record : partitionRecords) {
String identity = kafkaRecordIdentity(record);
if (!matchedIdentities.contains(identity)) {
if (!pendingBatch.isEmpty()) {
if (!invokeWithDlq(pipe, wrapRecords(pendingBatch), region)) {
return pendingBatch.size();
}
- offsetsToCommit.put(partition, new OffsetAndMetadata(pendingOffset));
+ offsetsToCommit.put(partition, pendingOffset);
pendingBatch.clear();
}
- offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));
+ offsetsToCommit.put(partition, record.offset() + 1);
continue;
}
@@ -444,38 +463,38 @@ public class PipesPoller implements Resettable {
if (!invokeWithDlq(pipe, wrapRecords(pendingBatch), region)) {
return pendingBatch.size();
}
- offsetsToCommit.put(partition, new OffsetAndMetadata(pendingOffset));
+ offsetsToCommit.put(partition, pendingOffset);
}
return 0;
}
private int deliverKafkaRecords(Pipe pipe,
String region,
- List<ConsumerRecord<byte[], byte[]>> batch,
+ List<KafkaRecordDto> batch,
Map<String, JsonNode> deliveryRecordsByIdentity,
- List<JsonNode> filtered) {
+ List<JsonNode> filtered,
+ Set<KafkaTopicPartition> blockedPartitions) {
Set<String> matchedIdentities = new HashSet<>(filtered.size());
filtered.forEach(record -> matchedIdentities.add(kafkaRecordIdentity(record)));
- Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();
- Set<TopicPartition> blockedPartitions = new HashSet<>();
+ Map<KafkaTopicPartition, Long> offsetsToCommit = new HashMap<>();
int failed = 0;
- for (ConsumerRecord<byte[], byte[]> record : batch) {
- TopicPartition partition = new TopicPartition(record.topic(), record.partition());
+ for (KafkaRecordDto record : batch) {
+ KafkaTopicPartition partition = new KafkaTopicPartition(record.topic(), record.partition());
if (blockedPartitions.contains(partition)) {
continue;
}
String identity = kafkaRecordIdentity(record);
if (!matchedIdentities.contains(identity)) {
- offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));
+ offsetsToCommit.put(partition, record.offset() + 1);
continue;
}
JsonNode deliveryRecord = deliveryRecordsByIdentity.get(identity);
if (deliveryRecord != null && invokeWithDlq(pipe, deliveryRecord.toString(), region)) {
- offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));
+ offsetsToCommit.put(partition, record.offset() + 1);
continue;
}
@@ -483,9 +502,7 @@ public class PipesPoller implements Resettable {
failed++;
}
- if (!offsetsToCommit.isEmpty()) {
- kafkaConsumerManager.commit(pipe, offsetsToCommit);
- }
+ commitOffsets(pipe, offsetsToCommit);
return failed;
}
@@ -722,11 +739,11 @@ public class PipesPoller implements Resettable {
return nodes;
}
- private List<ObjectNode> buildKafkaRecordNodes(List<ConsumerRecord<byte[], byte[]>> records, Pipe pipe) {
+ private List<ObjectNode> buildKafkaRecordNodes(List<KafkaRecordDto> records, Pipe pipe) {
List<ObjectNode> nodes = new ArrayList<>();
String eventSource = pipe.getSource().contains(":kafka:") ? "aws:kafka" : "SelfManagedKafka";
String bootstrapServers = kafkaConsumerManager.resolveBootstrapServers(pipe);
- for (ConsumerRecord<byte[], byte[]> record : records) {
+ for (KafkaRecordDto record : records) {
ObjectNode node = objectMapper.createObjectNode();
node.put("eventSource", eventSource);
if (pipe.getSource().contains(":kafka:")) {
@@ -738,12 +755,12 @@ public class PipesPoller implements Resettable {
node.put("partition", record.partition());
node.put("offset", record.offset());
node.put("timestamp", record.timestamp());
- node.put("timestampType", record.timestampType().name());
+ node.put("timestampType", record.timestampType());
putKafkaBinaryField(node, "key", record.key());
putKafkaBinaryField(node, "value", record.value());
var headersNode = node.putArray("headers");
- for (Header header : record.headers()) {
+ for (KafkaHeaderDto header : record.headers()) {
ObjectNode headerNode = objectMapper.createObjectNode();
byte[] headerValue = header.value();
if (headerValue == null) {
@@ -761,12 +778,11 @@ public class PipesPoller implements Resettable {
return nodes;
}
- private List<ObjectNode> buildKafkaFilterNodes(List<ObjectNode> deliveryNodes,
- List<ConsumerRecord<byte[], byte[]>> records) {
+ private List<ObjectNode> buildKafkaFilterNodes(List<ObjectNode> deliveryNodes, List<KafkaRecordDto> records) {
List<ObjectNode> nodes = new ArrayList<>(deliveryNodes.size());
for (int i = 0; i < deliveryNodes.size(); i++) {
ObjectNode node = deliveryNodes.get(i).deepCopy();
- ConsumerRecord<byte[], byte[]> record = records.get(i);
+ KafkaRecordDto record = records.get(i);
applyDecodedKafkaField(node, "key", record.key());
applyDecodedKafkaField(node, "value", record.value());
nodes.add(node);
@@ -795,7 +811,7 @@ public class PipesPoller implements Resettable {
+ record.path("offset").asText();
}
- private static String kafkaRecordIdentity(ConsumerRecord<byte[], byte[]> record) {
+ private static String kafkaRecordIdentity(KafkaRecordDto record) {
return record.topic() + ":" + record.partition() + ":" + record.offset();
}
@@ -822,16 +838,49 @@ public class PipesPoller implements Resettable {
node.put(fieldName, base64(value));
}
- private Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> groupKafkaRecordsByPartition(
- List<ConsumerRecord<byte[], byte[]>> records) {
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = new HashMap<>();
- for (ConsumerRecord<byte[], byte[]> record : records) {
- TopicPartition partition = new TopicPartition(record.topic(), record.partition());
+ private Map<KafkaTopicPartition, List<KafkaRecordDto>> groupKafkaRecordsByPartition(List<KafkaRecordDto> records) {
+ Map<KafkaTopicPartition, List<KafkaRecordDto>> recordsByPartition = new HashMap<>();
+ for (KafkaRecordDto record : records) {
+ KafkaTopicPartition partition = new KafkaTopicPartition(record.topic(), record.partition());
recordsByPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);
}
return recordsByPartition;
}
+ /**
+ * Commits past every record in {@code batch}, for when none of them are being delivered (no
+ * filter match). The REST Proxy has no "commit everything consumed" shortcut (unlike the
+ * native consumer's no-args {@code commitSync()}), so this computes the equivalent explicitly:
+ * the highest offset seen per partition, plus one.
+ */
+ private void commitWholeBatch(Pipe pipe, List<KafkaRecordDto> batch) {
+ Map<KafkaTopicPartition, Long> offsetsToCommit = new HashMap<>();
+ for (KafkaRecordDto record : batch) {
+ KafkaTopicPartition partition = new KafkaTopicPartition(record.topic(), record.partition());
+ offsetsToCommit.merge(partition, record.offset() + 1, Math::max);
+ }
+ commitOffsets(pipe, offsetsToCommit);
+ }
+
+ private void commitOffsets(Pipe pipe, Map<KafkaTopicPartition, Long> offsetsToCommit) {
+ if (offsetsToCommit.isEmpty()) {
+ return;
+ }
+ List<KafkaOffsetDto> offsets = new ArrayList<>(offsetsToCommit.size());
+ offsetsToCommit.forEach((partition, offset) ->
+ offsets.add(new KafkaOffsetDto(partition.topic(), partition.partition(), offset)));
+ kafkaConsumerManager.commit(pipe, offsets);
+ }
+
+ private static <T> List<List<T>> partition(List<T> list, int size) {
+ int step = Math.max(1, size);
+ List<List<T>> partitions = new ArrayList<>();
+ for (int i = 0; i < list.size(); i += step) {
+ partitions.add(list.subList(i, Math.min(i + step, list.size())));
+ }
+ return partitions;
+ }
+
private static String pipeKey(Pipe pipe) {
return pipe.getArn();
}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index d09352c188..efa358b2f3 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -638,6 +638,10 @@ floci:
enabled: true
pipes:
enabled: true
+ kafka-rest-bridge:
+ default-image: "ghcr.io/aiven-open/karapace:latest"
+ host-port-base: 9500
+ host-port-max: 9599
configservice:
enabled: true
docdb:
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/KarapaceManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/KarapaceManagerTest.java
new file mode 100644
index 0000000000..fce9a62c9b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/KarapaceManagerTest.java
@@ -0,0 +1,193 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import com.sun.net.httpserver.HttpServer;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
+import io.github.hectorvent.floci.core.common.docker.PortAllocator;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class KarapaceManagerTest {
+
+ private static final int REST_PORT = 8082;
+
+ @Mock private ContainerLifecycleManager lifecycleManager;
+ @Mock private ContainerDetector containerDetector;
+ @Mock private PortAllocator portAllocator;
+ @Mock private EmulatorConfig config;
+ @Mock private DockerHostResolver dockerHostResolver;
+ @Mock private EmbeddedDnsServer embeddedDnsServer;
+
+ private KarapaceManager manager;
+ private HttpServer fakeRestServer;
+
+ @BeforeEach
+ void setUp() {
+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);
+ EmulatorConfig.PipesServiceConfig pipes = mock(EmulatorConfig.PipesServiceConfig.class);
+ EmulatorConfig.KafkaRestBridgeConfig bridge = mock(EmulatorConfig.KafkaRestBridgeConfig.class);
+ EmulatorConfig.DockerConfig docker = mock(EmulatorConfig.DockerConfig.class);
+
+ lenient().when(config.services()).thenReturn(services);
+ lenient().when(services.pipes()).thenReturn(pipes);
+ lenient().when(services.dockerNetwork()).thenReturn(Optional.empty());
+ lenient().when(pipes.kafkaRestBridge()).thenReturn(bridge);
+ lenient().when(bridge.defaultImage()).thenReturn("ghcr.io/aiven-open/karapace:latest");
+ lenient().when(bridge.hostPortBase()).thenReturn(9500);
+ lenient().when(bridge.hostPortMax()).thenReturn(9599);
+ lenient().when(config.docker()).thenReturn(docker);
+ lenient().when(docker.logMaxSize()).thenReturn("10m");
+ lenient().when(docker.logMaxFile()).thenReturn("3");
+
+ lenient().when(embeddedDnsServer.getServerIp()).thenReturn(Optional.empty());
+
+ ContainerBuilder containerBuilder = new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer);
+ manager = new KarapaceManager(containerBuilder, lifecycleManager, containerDetector, config, portAllocator,
+ dockerHostResolver, java.time.Duration.ofMillis(500));
+ }
+
+ @AfterEach
+ void tearDown() {
+ if (fakeRestServer != null) {
+ fakeRestServer.stop(0);
+ }
+ }
+
+ /** Stands in for Karapace's own {@code GET /_health}, which {@code isReady} polls. */
+ private int startFakeRestServer() throws Exception {
+ fakeRestServer = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+ fakeRestServer.createContext("/_health", exchange -> {
+ byte[] body = "{}".getBytes(StandardCharsets.UTF_8);
+ exchange.sendResponseHeaders(200, body.length);
+ exchange.getResponseBody().write(body);
+ exchange.close();
+ });
+ fakeRestServer.start();
+ return fakeRestServer.getAddress().getPort();
+ }
+
+ @Test
+ void ensureStartedConfiguresBootstrapUriAndPublishesRestPort() throws Exception {
+ int hostPort = startFakeRestServer();
+ when(containerDetector.isRunningInContainer()).thenReturn(false);
+ when(portAllocator.allocate(9500, 9599)).thenReturn(hostPort);
+
+ ContainerInfo info = new ContainerInfo("container-1",
+ Map.of(REST_PORT, new EndpointInfo("localhost", hostPort)));
+ when(lifecycleManager.createAndStart(any())).thenReturn(info);
+
+ URI restBaseUri = manager.ensureStarted("broker-1:9092");
+
+ assertEquals("http://localhost:" + hostPort + "/", restBaseUri.toString());
+
+ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
+ verify(lifecycleManager).createAndStart(specCaptor.capture());
+ ContainerSpec spec = specCaptor.getValue();
+ assertTrue(spec.env().contains("KARAPACE_BOOTSTRAP_URI=broker-1:9092"),
+ "Karapace must be pointed at the target cluster's bootstrap servers");
+ assertEquals(Integer.valueOf(hostPort), spec.portBindings().get(REST_PORT));
+ }
+
+ @Test
+ void ensureStartedReusesTheSameContainerForTheSameTarget() throws Exception {
+ int hostPort = startFakeRestServer();
+ when(containerDetector.isRunningInContainer()).thenReturn(false);
+ when(portAllocator.allocate(9500, 9599)).thenReturn(hostPort);
+
+ ContainerInfo info = new ContainerInfo("container-1",
+ Map.of(REST_PORT, new EndpointInfo("localhost", hostPort)));
+ when(lifecycleManager.createAndStart(any())).thenReturn(info);
+
+ URI first = manager.ensureStarted("broker-1:9092");
+ URI second = manager.ensureStarted("broker-1:9092");
+
+ assertSame(first, second, "multiple pipes reading the same broker must share one Karapace instance");
+ verify(lifecycleManager, times(1)).createAndStart(any());
+ }
+
+ @Test
+ void ensureStartedRewritesHostLocalBrokerForTheSidecar() throws Exception {
+ int hostPort = startFakeRestServer();
+ when(containerDetector.isRunningInContainer()).thenReturn(false);
+ when(portAllocator.allocate(9500, 9599)).thenReturn(hostPort);
+ when(dockerHostResolver.resolve()).thenReturn("host.docker.internal");
+
+ ContainerInfo info = new ContainerInfo("container-1",
+ Map.of(REST_PORT, new EndpointInfo("localhost", hostPort)));
+ when(lifecycleManager.createAndStart(any())).thenReturn(info);
+
+ manager.ensureStarted("localhost:9092");
+
+ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
+ verify(lifecycleManager).createAndStart(specCaptor.capture());
+ assertTrue(specCaptor.getValue().env().contains("KARAPACE_BOOTSTRAP_URI=host.docker.internal:9092"),
+ "a host-local broker address must be rewritten so the sidecar, in its own network "
+ + "namespace, reaches the Floci host instead of itself");
+ }
+
+ @Test
+ void ensureStartedAddsTheLinuxHostGatewayAliasSoRewrittenAddressesResolve() throws Exception {
+ int hostPort = startFakeRestServer();
+ when(containerDetector.isRunningInContainer()).thenReturn(false);
+ when(portAllocator.allocate(9500, 9599)).thenReturn(hostPort);
+ when(dockerHostResolver.resolve()).thenReturn("host.docker.internal");
+ when(dockerHostResolver.isLinuxHost()).thenReturn(true);
+
+ ContainerInfo info = new ContainerInfo("container-1",
+ Map.of(REST_PORT, new EndpointInfo("localhost", hostPort)));
+ when(lifecycleManager.createAndStart(any())).thenReturn(info);
+
+ manager.ensureStarted("localhost:9092");
+
+ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
+ verify(lifecycleManager).createAndStart(specCaptor.capture());
+ assertTrue(specCaptor.getValue().extraHosts().contains("host.docker.internal:host-gateway"),
+ "native Linux Docker does not auto-inject host.docker.internal, so the sidecar needs "
+ + "the host-gateway mapping to resolve the address resolveForContainer rewrote to it");
+ }
+
+ @Test
+ void ensureStartedCleansUpWhenReadinessNeverArrives() {
+ when(containerDetector.isRunningInContainer()).thenReturn(false);
+ when(portAllocator.allocate(9500, 9599)).thenReturn(54321);
+
+ ContainerInfo info = new ContainerInfo("container-1",
+ Map.of(REST_PORT, new EndpointInfo("localhost", 1)));
+ when(lifecycleManager.createAndStart(any())).thenReturn(info);
+
+ org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class,
+ () -> manager.ensureStarted("broker-1:9092"));
+
+ verify(lifecycleManager).stopAndRemove("container-1", null);
+ verify(portAllocator).release(54321);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java
index 125a87775f..3cdde4f8cc 100644
--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java
@@ -8,8 +8,15 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+import java.net.URI;
+
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class PipesKafkaConsumerManagerTest {
@@ -17,12 +24,45 @@ class PipesKafkaConsumerManagerTest {
private static final ObjectMapper MAPPER = new ObjectMapper();
private MskService mskService;
+ private KarapaceManager karapaceManager;
+ private PipesKafkaRestClient restClient;
private PipesKafkaConsumerManager manager;
@BeforeEach
void setUp() {
mskService = Mockito.mock(MskService.class);
- manager = new PipesKafkaConsumerManager(mskService);
+ karapaceManager = Mockito.mock(KarapaceManager.class);
+ restClient = Mockito.mock(PipesKafkaRestClient.class);
+ manager = new PipesKafkaConsumerManager(mskService, karapaceManager, restClient);
+ }
+
+ @Test
+ void pollClosesOrphanedConsumerWhenSubscriptionFails() throws Exception {
+ Pipe pipe = new Pipe();
+ pipe.setArn("arn:aws:pipes:us-east-1:000000000000:pipe/orders-pipe");
+ pipe.setName("orders-pipe");
+ pipe.setSource("smk://broker-1:9092");
+ pipe.setSourceParameters(MAPPER.readTree("""
+ {
+ "SelfManagedKafkaParameters": {
+ "TopicName": "orders"
+ }
+ }
+ """));
+
+ URI restBaseUri = URI.create("http://localhost:8082/");
+ KafkaConsumerHandle handle = new KafkaConsumerHandle("instance-1",
+ URI.create("http://localhost:8082/consumers/g/instances/instance-1"));
+ when(karapaceManager.ensureStarted("broker-1:9092")).thenReturn(restBaseUri);
+ when(restClient.createConsumer(eq(restBaseUri), anyString(), anyString())).thenReturn(handle);
+ Mockito.doThrow(new RuntimeException("subscribe failed"))
+ .when(restClient).subscribe(eq(handle), anyList());
+
+ assertThrows(RuntimeException.class, () -> manager.poll(pipe));
+
+ // The consumer already exists server-side once createConsumer succeeds; a failed
+ // subscribe must delete it here, since the map never learns its handle to close it later.
+ verify(restClient).close(handle);
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.java
deleted file mode 100644
index d73542e0b9..0000000000
--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-package io.github.hectorvent.floci.services.pipes;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import io.quarkus.runtime.annotations.RegisterForReflection;
-import io.github.hectorvent.floci.services.msk.MskService;
-import io.github.hectorvent.floci.services.pipes.model.Pipe;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.junit.jupiter.api.DisplayName;
-import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
-
-import java.util.Collection;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.function.Predicate;
-
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-/**
- * Pins {@link PipesKafkaNativeSupport} to what {@code ConsumerConfig} resolves from
- * {@link PipesKafkaConsumerManager#consumerProperties}, both ways: a newly resolved class must be
- * registered, and nothing else may be. Reading that method rather than copying it means a new
- * setting there is covered here too.
- *
- * <p>Scope: config-resolved classes only. Reflection always succeeds on the JVM, so a non-config
- * lookup from a future kafka-clients needs a Kafka pipe run against a native binary. The
- * compatibility suite already runs against the native image; it has no Kafka source case yet.
- */
-class PipesKafkaNativeSupportTest {
-
- private static final ObjectMapper MAPPER = new ObjectMapper();
-
- private static final Predicate<String> TRANSPORT_SECURITY =
- key -> key.startsWith("sasl.") || key.startsWith("ssl.");
-
- @Test
- @DisplayName("classes ConsumerConfig resolves reflectively are registered")
- void registrationCoversReflectivelyResolvedClasses() throws Exception {
- Set<String> missing = new TreeSet<>(expectedClassNames());
- missing.removeAll(registeredClassNames());
-
- assertTrue(missing.isEmpty(),
- "kafka-clients resolves these reflectively but PipesKafkaNativeSupport omits them, "
- + "so a native build cannot construct a consumer: " + missing);
- }
-
- @Test
- @DisplayName("nothing is registered that ConsumerConfig does not resolve")
- void registrationContainsNothingUnjustified() throws Exception {
- Set<String> unexpected = new TreeSet<>(registeredClassNames());
- unexpected.removeAll(expectedClassNames());
-
- assertTrue(unexpected.isEmpty(),
- "these are not config-resolved, so this test cannot guard them and they need a "
- + "native run to justify. SASL/SSL defaults in particular break the build — "
- + "DefaultJwtValidator drags in optional jose4j: " + unexpected);
- }
-
- /** Class-valued settings kafka-clients resolves, minus the transport security Floci never uses. */
- private static Set<String> expectedClassNames() throws Exception {
- Set<String> classes = new TreeSet<>();
- consumerConfigValues().forEach((key, value) -> {
- if (TRANSPORT_SECURITY.negate().test(key)) {
- collectClassNames(value, classes);
- }
- });
- return classes;
- }
-
- private static Set<String> registeredClassNames() {
- RegisterForReflection annotation =
- PipesKafkaNativeSupport.class.getAnnotation(RegisterForReflection.class);
- assertNotNull(annotation, "PipesKafkaNativeSupport must carry @RegisterForReflection");
-
- Set<String> names = new TreeSet<>(Set.of(annotation.classNames()));
- for (Class<?> target : annotation.targets()) {
- names.add(target.getName());
- }
- return names;
- }
-
- /**
- * Mirrors {@code AbstractConfig.getConfiguredInstances}: a setting may be a {@code Class} or a
- * class name — {@code metric.reporters} is the latter. Values naming no class are not candidates.
- */
- private static void collectClassNames(Object value, Set<String> into) {
- switch (value) {
- case Class<?> type -> into.add(type.getName());
- case Collection<?> collection -> collection.forEach(element -> collectClassNames(element, into));
- case String name -> loadable(name).ifPresent(into::add);
- case null, default -> { }
- }
- }
-
- private static Optional<String> loadable(String className) {
- try {
- return Optional.of(Class.forName(className, false, ConsumerConfig.class.getClassLoader()).getName());
- } catch (ClassNotFoundException | LinkageError e) {
- return Optional.empty();
- }
- }
-
- private static Map<String, ?> consumerConfigValues() throws Exception {
- PipesKafkaConsumerManager manager = new PipesKafkaConsumerManager(Mockito.mock(MskService.class));
-
- Pipe pipe = new Pipe();
- pipe.setName("native-support");
- pipe.setSource("smk://localhost:9092");
- pipe.setSourceParameters(MAPPER.readTree("""
- {
- "SelfManagedKafkaParameters": {
- "TopicName": "orders"
- }
- }
- """));
-
- return new ConsumerConfig(manager.consumerProperties(pipe)).values();
- }
-}
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientTest.java
new file mode 100644
index 0000000000..8072d23354
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientTest.java
@@ -0,0 +1,179 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.Base64;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/** Exercises {@link PipesKafkaRestClient} against a fake server speaking Karapace's REST Proxy API. */
+class PipesKafkaRestClientTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private HttpServer fakeServer;
+ private PipesKafkaRestClient client;
+ private final AtomicReference<String> lastRequestBody = new AtomicReference<>();
+ private final AtomicReference<String> lastRequestMethod = new AtomicReference<>();
+ private final AtomicReference<String> lastRequestPath = new AtomicReference<>();
+
+ @BeforeEach
+ void setUp() throws Exception {
+ fakeServer = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+
+ fakeServer.createContext("/consumers/orders-group", exchange -> {
+ record(exchange);
+ // Karapace returns base_uri as a path relative to the proxy root (not an absolute URL
+ // as the Confluent REST Proxy reference spec shows) - this fixture matches that.
+ String response = "{\"instance_id\":\"rest-consumer-1\","
+ + "\"base_uri\":\"consumers/orders-group/instances/rest-consumer-1\"}";
+ respond(exchange, 200, response);
+ });
+
+ fakeServer.createContext("/consumers/orders-group/instances/rest-consumer-1/subscription", exchange -> {
+ record(exchange);
+ respond(exchange, 204, "");
+ });
+
+ fakeServer.createContext("/consumers/orders-group/instances/rest-consumer-1/records", exchange -> {
+ record(exchange);
+ byte[] key = "customer-1".getBytes(StandardCharsets.UTF_8);
+ byte[] value = "{\"status\":\"active\"}".getBytes(StandardCharsets.UTF_8);
+ byte[] headerValue = "trace-1".getBytes(StandardCharsets.UTF_8);
+ String response = """
+ [
+ {
+ "topic": "orders",
+ "partition": 0,
+ "offset": 42,
+ "timestamp": 1700000000000,
+ "key": "%s",
+ "value": "%s",
+ "headers": [{"name": "traceId", "value": "%s"}]
+ }
+ ]
+ """.formatted(base64(key), base64(value), base64(headerValue));
+ respond(exchange, 200, response);
+ });
+
+ fakeServer.createContext("/consumers/orders-group/instances/rest-consumer-1/offsets", exchange -> {
+ record(exchange);
+ respond(exchange, 200, "");
+ });
+
+ fakeServer.createContext("/consumers/orders-group/instances/rest-consumer-1", exchange -> {
+ if ("DELETE".equals(exchange.getRequestMethod())) {
+ record(exchange);
+ respond(exchange, 204, "");
+ } else {
+ exchange.sendResponseHeaders(404, -1);
+ exchange.close();
+ }
+ });
+
+ fakeServer.start();
+ client = new PipesKafkaRestClient(MAPPER, HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build());
+ }
+
+ @AfterEach
+ void tearDown() {
+ fakeServer.stop(0);
+ }
+
+ private URI restBaseUri() {
+ return URI.create("http://localhost:" + fakeServer.getAddress().getPort() + "/");
+ }
+
+ private void record(com.sun.net.httpserver.HttpExchange exchange) throws java.io.IOException {
+ lastRequestMethod.set(exchange.getRequestMethod());
+ lastRequestPath.set(exchange.getRequestURI().toString());
+ lastRequestBody.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
+ }
+
+ private static void respond(com.sun.net.httpserver.HttpExchange exchange, int status, String body) throws java.io.IOException {
+ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
+ exchange.sendResponseHeaders(status, bytes.length == 0 ? -1 : bytes.length);
+ if (bytes.length > 0) {
+ exchange.getResponseBody().write(bytes);
+ }
+ exchange.close();
+ }
+
+ private static String base64(byte[] value) {
+ return Base64.getEncoder().encodeToString(value);
+ }
+
+ @Test
+ void createConsumerParsesInstanceIdAndBaseUri() {
+ KafkaConsumerHandle consumer = client.createConsumer(restBaseUri(), "orders-group", "latest");
+
+ assertEquals("rest-consumer-1", consumer.instanceId());
+ assertEquals("POST", lastRequestMethod.get());
+ assertTrue(consumer.baseUri().toString().endsWith("/consumers/orders-group/instances/rest-consumer-1"));
+ }
+
+ @Test
+ void subscribeSendsTopicList() throws Exception {
+ KafkaConsumerHandle consumer = client.createConsumer(restBaseUri(), "orders-group", "earliest");
+
+ client.subscribe(consumer, List.of("orders"));
+
+ JsonNode body = MAPPER.readTree(lastRequestBody.get());
+ assertEquals("orders", body.path("topics").get(0).asText());
+ }
+
+ @Test
+ void pollDecodesBinaryKeyValueAndHeaders() {
+ KafkaConsumerHandle consumer = client.createConsumer(restBaseUri(), "orders-group", "latest");
+
+ List<KafkaRecordDto> records = client.poll(consumer, Duration.ofMillis(100));
+
+ assertEquals(1, records.size());
+ KafkaRecordDto record = records.get(0);
+ assertEquals("orders", record.topic());
+ assertEquals(0, record.partition());
+ assertEquals(42L, record.offset());
+ assertEquals(1700000000000L, record.timestamp());
+ assertArrayEquals("customer-1".getBytes(StandardCharsets.UTF_8), record.key());
+ assertArrayEquals("{\"status\":\"active\"}".getBytes(StandardCharsets.UTF_8), record.value());
+ assertEquals(1, record.headers().size());
+ assertEquals("traceId", record.headers().get(0).key());
+ assertArrayEquals("trace-1".getBytes(StandardCharsets.UTF_8), record.headers().get(0).value());
+ }
+
+ @Test
+ void commitSpecificOffsetsSendsOffsetsArray() throws Exception {
+ KafkaConsumerHandle consumer = client.createConsumer(restBaseUri(), "orders-group", "latest");
+
+ client.commit(consumer, List.of(new KafkaOffsetDto("orders", 0, 43L)));
+
+ JsonNode body = MAPPER.readTree(lastRequestBody.get());
+ JsonNode offset = body.path("offsets").get(0);
+ assertEquals("orders", offset.path("topic").asText());
+ assertEquals(0, offset.path("partition").asInt());
+ assertEquals(43L, offset.path("offset").asLong());
+ }
+
+ @Test
+ void closeDeletesTheConsumerInstance() {
+ KafkaConsumerHandle consumer = client.createConsumer(restBaseUri(), "orders-group", "latest");
+
+ client.close(consumer);
+
+ assertEquals("DELETE", lastRequestMethod.get());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java
index c9cc6abdc0..9524d2db6a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java
@@ -9,10 +9,6 @@ import io.github.hectorvent.floci.services.pipes.model.Pipe;
import io.github.hectorvent.floci.services.sqs.SqsService;
import io.github.hectorvent.floci.services.sqs.model.Message;
import io.vertx.core.Vertx;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.common.TopicPartition;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -20,20 +16,18 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
-import java.nio.charset.StandardCharsets;
import java.util.List;
-import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -55,6 +49,7 @@ class PipesPollerTest {
@BeforeEach
void setUp() {
when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ lenient().when(kafkaConsumerManager.resolveBatchSize(any(), anyInt())).thenReturn(10);
poller = new PipesPoller(vertx, sqsService, kinesisService, dynamoDbStreamService,
kafkaConsumerManager, targetInvoker, new PipesFilterMatcher(MAPPER), MAPPER, config);
}
@@ -147,19 +142,19 @@ class PipesPollerTest {
@Test
void pollKafka_filtersUsingDecodedPayloadButDeliversOriginalRecord() throws Exception {
Pipe pipe = selfManagedKafkaPipe();
- byte[] key = "customer-123".getBytes(StandardCharsets.UTF_8);
- byte[] value = "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8);
- ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>("orders", 0, 42L, key, value);
- record.headers().add("traceId", "abc123".getBytes(StandardCharsets.UTF_8));
+ byte[] key = "customer-123".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ byte[] value = "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ byte[] traceId = "abc123".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ KafkaRecordDto record = kafkaRecord("orders", 0, 42L, key, value, new KafkaHeaderDto("traceId", traceId));
- when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(record));
when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
poller.pollKafka(pipe, "us-east-1");
ArgumentCaptor<String> payloadCaptor = ArgumentCaptor.forClass(String.class);
verify(targetInvoker).invoke(eq(pipe), payloadCaptor.capture(), eq("us-east-1"));
- verify(kafkaConsumerManager).commit(eq(pipe), anyMap());
+ verify(kafkaConsumerManager).commit(eq(pipe), anyList());
JsonNode delivered = MAPPER.readTree(payloadCaptor.getValue());
assertEquals("orders", delivered.path("topic").asText());
@@ -177,29 +172,28 @@ class PipesPollerTest {
@Test
void pollKafka_doesNotCommitWhenDeliveryFails() throws Exception {
Pipe pipe = selfManagedKafkaPipe();
- ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>(
- "orders", 0, 7L, null, "{\"status\":\"active\"}".getBytes(StandardCharsets.UTF_8));
+ KafkaRecordDto record = kafkaRecord("orders", 0, 7L, null,
+ "{\"status\":\"active\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));
- when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(record));
when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
org.mockito.Mockito.doThrow(new RuntimeException("boom"))
.when(targetInvoker).invoke(eq(pipe), anyString(), eq("us-east-1"));
poller.pollKafka(pipe, "us-east-1");
- verify(kafkaConsumerManager, never()).commit(pipe);
- verify(kafkaConsumerManager, never()).commit(eq(pipe), anyMap());
+ verify(kafkaConsumerManager, never()).commit(eq(pipe), anyList());
}
@Test
void pollKafka_commitsDeliveredPrefixWhenLaterRecordFails() throws Exception {
Pipe pipe = selfManagedKafkaPipe();
- ConsumerRecord<byte[], byte[]> first = new ConsumerRecord<>(
- "orders", 0, 0L, null, "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8));
- ConsumerRecord<byte[], byte[]> second = new ConsumerRecord<>(
- "orders", 0, 1L, null, "{\"status\":\"active\",\"id\":\"order-2\"}".getBytes(StandardCharsets.UTF_8));
+ KafkaRecordDto first = kafkaRecord("orders", 0, 0L, null,
+ "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));
+ KafkaRecordDto second = kafkaRecord("orders", 0, 1L, null,
+ "{\"status\":\"active\",\"id\":\"order-2\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));
- when(kafkaConsumerManager.poll(pipe)).thenReturn(records(first, second));
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(first, second));
when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
org.mockito.Mockito.doNothing()
.doThrow(new RuntimeException("boom"))
@@ -207,17 +201,42 @@ class PipesPollerTest {
poller.pollKafka(pipe, "us-east-1");
- ArgumentCaptor<Map<TopicPartition, OffsetAndMetadata>> offsetsCaptor = ArgumentCaptor.forClass(Map.class);
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<List<KafkaOffsetDto>> offsetsCaptor = ArgumentCaptor.forClass(List.class);
verify(kafkaConsumerManager).commit(eq(pipe), offsetsCaptor.capture());
- assertEquals(1L, offsetsCaptor.getValue().get(new TopicPartition("orders", 0)).offset());
+ assertEquals(1L, singleOffset(offsetsCaptor.getValue(), "orders", 0));
+ }
+
+ @Test
+ void pollKafka_laterGeneratedBatchDoesNotCommitPastAnEarlierFailureInTheSamePartition() throws Exception {
+ // A REST Proxy poll can return more than one BatchSize worth of records, so pollKafka
+ // splits them into several generated batches. If the first batch fails to deliver record 0
+ // and a later batch then succeeds delivering record 1 from the same partition, the second
+ // batch's commit must not advance past record 0, or it is lost forever.
+ Pipe pipe = selfManagedKafkaPipe();
+ when(kafkaConsumerManager.resolveBatchSize(eq(pipe), anyInt())).thenReturn(1);
+ KafkaRecordDto first = kafkaRecord("orders", 0, 0L, null,
+ "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));
+ KafkaRecordDto second = kafkaRecord("orders", 0, 1L, null,
+ "{\"status\":\"active\",\"id\":\"order-2\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));
+
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(first, second));
+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
+ org.mockito.Mockito.doThrow(new RuntimeException("boom"))
+ .doNothing()
+ .when(targetInvoker).invoke(eq(pipe), anyString(), eq("us-east-1"));
+
+ poller.pollKafka(pipe, "us-east-1");
+
+ verify(kafkaConsumerManager, never()).commit(eq(pipe), anyList());
}
@Test
void pollKafka_representsNullKeyAndValueAsJsonNull() throws Exception {
Pipe pipe = nullableSelfManagedKafkaPipe();
- ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>("orders", 0, 3L, null, null);
+ KafkaRecordDto record = kafkaRecord("orders", 0, 3L, null, null);
- when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(record));
when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
poller.pollKafka(pipe, "us-east-1");
@@ -232,14 +251,14 @@ class PipesPollerTest {
@Test
void pollKafka_lambdaCommitsSuccessfulPrefixBeforeLaterFailure() throws Exception {
Pipe pipe = lambdaSelfManagedKafkaPipe();
- ConsumerRecord<byte[], byte[]> first = new ConsumerRecord<>(
- "orders", 0, 0L, null, "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8));
- ConsumerRecord<byte[], byte[]> skipped = new ConsumerRecord<>(
- "orders", 0, 1L, null, "{\"status\":\"inactive\",\"id\":\"order-2\"}".getBytes(StandardCharsets.UTF_8));
- ConsumerRecord<byte[], byte[]> failing = new ConsumerRecord<>(
- "orders", 0, 2L, null, "{\"status\":\"active\",\"id\":\"order-3\"}".getBytes(StandardCharsets.UTF_8));
-
- when(kafkaConsumerManager.poll(pipe)).thenReturn(records(first, skipped, failing));
+ KafkaRecordDto first = kafkaRecord("orders", 0, 0L, null,
+ "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));
+ KafkaRecordDto skipped = kafkaRecord("orders", 0, 1L, null,
+ "{\"status\":\"inactive\",\"id\":\"order-2\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));
+ KafkaRecordDto failing = kafkaRecord("orders", 0, 2L, null,
+ "{\"status\":\"active\",\"id\":\"order-3\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));
+
+ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(first, skipped, failing));
when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");
org.mockito.Mockito.doNothing()
.doThrow(new RuntimeException("boom"))
@@ -247,9 +266,10 @@ class PipesPollerTest {
poller.pollKafka(pipe, "us-east-1");
- ArgumentCaptor<Map<TopicPartition, OffsetAndMetadata>> offsetsCaptor = ArgumentCaptor.forClass(Map.class);
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<List<KafkaOffsetDto>> offsetsCaptor = ArgumentCaptor.forClass(List.class);
verify(kafkaConsumerManager).commit(eq(pipe), offsetsCaptor.capture());
- assertEquals(2L, offsetsCaptor.getValue().get(new TopicPartition("orders", 0)).offset());
+ assertEquals(2L, singleOffset(offsetsCaptor.getValue(), "orders", 0));
}
private Pipe selfManagedKafkaPipe() throws Exception {
@@ -300,12 +320,17 @@ class PipesPollerTest {
return pipe;
}
- private ConsumerRecords<byte[], byte[]> records(ConsumerRecord<byte[], byte[]>... records) {
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> byPartition = new java.util.LinkedHashMap<>();
- for (ConsumerRecord<byte[], byte[]> record : records) {
- TopicPartition topicPartition = new TopicPartition(record.topic(), record.partition());
- byPartition.computeIfAbsent(topicPartition, ignored -> new java.util.ArrayList<>()).add(record);
- }
- return new ConsumerRecords<>(byPartition);
+ private static KafkaRecordDto kafkaRecord(String topic, int partition, long offset, byte[] key, byte[] value,
+ KafkaHeaderDto... headers) {
+ return new KafkaRecordDto(topic, partition, offset, System.currentTimeMillis(),
+ KafkaRecordDto.DEFAULT_TIMESTAMP_TYPE, key, value, List.of(headers));
+ }
+
+ private static long singleOffset(List<KafkaOffsetDto> offsets, String topic, int partition) {
+ return offsets.stream()
+ .filter(offset -> offset.topic().equals(topic) && offset.partition() == partition)
+ .mapToLong(KafkaOffsetDto::offset)
+ .findFirst()
+ .orElseThrow();
}
}