Light, fluffy, and always free - The AWS Local Emulator alternative
fix(pipes): replace kafka-clients with a Kafka REST Proxy client (#3160)
Priamo Germosencd2847dba0af
diff --git a/docs/services/pipes.md b/docs/services/pipes.mdindex b1948b1c4d..c143c2b458 100644--- a/docs/services/pipes.md+++ b/docs/services/pipes.md@@ -20,6 +20,11 @@| Variable | Default | Description ||---|---|---|| `FLOCI_SERVICES_PIPES_ENABLED` | `true` | Enable or disable the service |+| `FLOCI_SERVICES_PIPES_KAFKA_REST_BRIDGE_DEFAULT_IMAGE` | `ghcr.io/aiven-open/karapace:latest` | Docker image for the Karapace REST Proxy sidecar a Kafka-sourced pipe starts on demand |+| `FLOCI_SERVICES_PIPES_KAFKA_REST_BRIDGE_HOST_PORT_BASE` | `9500` | Start of the host port range allocated to Karapace sidecars |+| `FLOCI_SERVICES_PIPES_KAFKA_REST_BRIDGE_HOST_PORT_MAX` | `9599` | End of the host port range allocated to Karapace sidecars |++A pipe with a Kafka source (MSK or self-managed via `smk://`) starts a Karapace REST Proxy sidecar container on first use, one per distinct `bootstrap.servers` target shared across every pipe reading it, so Docker is required for those pipes even though it is not for the others.## Examplesdiff --git a/pom.xml b/pom.xmlindex 1e8c0dc08a..f1ee4dcd07 100644--- a/pom.xml+++ b/pom.xml@@ -158,13 +158,6 @@<version>9.2.1</version></dependency>- <!-- Kafka client for MSK/self-managed Kafka pipe polling -->- <dependency>- <groupId>org.apache.kafka</groupId>- <artifactId>kafka-clients</artifactId>- <version>4.3.1</version>- </dependency>-<!-- Snappy for Firehose Snappy/HADOOP_SNAPPY delivery to S3: the only Javalibrary that writes the xerial stream format AWS uses. --><dependency>diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex fa837bcf8f..ee11ef6cd0 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -2192,6 +2192,25 @@ public interface EmulatorConfig {interface PipesServiceConfig {@WithDefault("true")boolean enabled();++ KafkaRestBridgeConfig kafkaRestBridge();+ }++ /**+ * The Karapace sidecar Pipes launches, on demand, to poll a Kafka source over REST instead of+ * embedding kafka-clients in Floci itself. One container is started per distinct target+ * {@code bootstrap.servers} (a self-managed cluster, or an MSK cluster's Redpanda backing), and+ * reused across pipes that share the same source.+ */+ interface KafkaRestBridgeConfig {+ @WithDefault("ghcr.io/aiven-open/karapace:latest")+ String defaultImage();++ @WithDefault("9500")+ int hostPortBase();++ @WithDefault("9599")+ int hostPortMax();}interface BedrockAgentCoreControlServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaConsumerHandle.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaConsumerHandle.javanew file mode 100644index 0000000000..1aaed74cb9--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaConsumerHandle.java@@ -0,0 +1,7 @@+package io.github.hectorvent.floci.services.pipes;++import java.net.URI;++/** A Kafka REST Proxy consumer instance: its own {@code base_uri} addresses every later call. */+record KafkaConsumerHandle(String instanceId, URI baseUri) {+}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaHeaderDto.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaHeaderDto.javanew file mode 100644index 0000000000..7fb408c430--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaHeaderDto.java@@ -0,0 +1,5 @@+package io.github.hectorvent.floci.services.pipes;++/** One Kafka record header, as decoded from a Kafka REST Proxy record response. */+record KafkaHeaderDto(String key, byte[] value) {+}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaOffsetDto.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaOffsetDto.javanew file mode 100644index 0000000000..ffbcfecab1--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaOffsetDto.java@@ -0,0 +1,10 @@+package io.github.hectorvent.floci.services.pipes;++/**+ * A partition offset to commit through a Kafka REST Proxy, in place of+ * {@code org.apache.kafka.common.TopicPartition} / {@code OffsetAndMetadata}. As with the native+ * client, {@code offset} is the next offset to consume (last delivered offset + 1), not the last+ * delivered offset itself.+ */+record KafkaOffsetDto(String topic, int partition, long offset) {+}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaRecordDto.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaRecordDto.javanew file mode 100644index 0000000000..49342a3b39--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaRecordDto.java@@ -0,0 +1,26 @@+package io.github.hectorvent.floci.services.pipes;++import java.util.List;++/**+ * One Kafka record as returned by a Kafka REST Proxy (Karapace, Pandaproxy) consumer poll, in+ * place of {@code org.apache.kafka.clients.consumer.ConsumerRecord} so Floci's own process never+ * needs {@code kafka-clients} on its classpath. See #2916.+ *+ * <p>{@code timestampType} is not part of the REST Proxy record schema (Karapace reads it+ * internally but does not expose it), so it is always reported as {@code CreateTime}, the default+ * for any topic that does not explicitly configure {@code message.timestamp.type}. This is a+ * documented, intentional deviation from the native Kafka protocol's per-record fidelity.+ */+record KafkaRecordDto(+ String topic,+ int partition,+ long offset,+ long timestamp,+ String timestampType,+ byte[] key,+ byte[] value,+ List<KafkaHeaderDto> headers) {++ static final String DEFAULT_TIMESTAMP_TYPE = "CreateTime";+}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaTopicPartition.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaTopicPartition.javanew file mode 100644index 0000000000..ac053058ad--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KafkaTopicPartition.java@@ -0,0 +1,5 @@+package io.github.hectorvent.floci.services.pipes;++/** A Kafka topic/partition pair, used as a map key when grouping records for delivery. */+record KafkaTopicPartition(String topic, int partition) {+}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/KarapaceManager.java b/src/main/java/io/github/hectorvent/floci/services/pipes/KarapaceManager.javanew file mode 100644index 0000000000..3054d6bc78--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/KarapaceManager.java@@ -0,0 +1,228 @@+package io.github.hectorvent.floci.services.pipes;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import io.github.hectorvent.floci.core.common.docker.PortAllocator;+import jakarta.annotation.PreDestroy;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.net.HttpURLConnection;+import java.net.URI;+import java.nio.charset.StandardCharsets;+import java.security.MessageDigest;+import java.security.NoSuchAlgorithmException;+import java.time.Duration;+import java.util.Arrays;+import java.util.HexFormat;+import java.util.List;+import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.stream.Collectors;++/**+ * Starts, on demand, a Karapace REST Proxy sidecar fronting a Kafka {@code bootstrap.servers}+ * target, so Pipes can poll a Kafka source over plain HTTP ({@link PipesKafkaRestClient}) instead+ * of embedding {@code kafka-clients} in Floci itself. See #2916.+ *+ * <p>One container is started per distinct target and reused for the life of this process:+ * {@link #ensureStarted} is safe to call concurrently, including repeatedly for the same target+ * (multiple Pipes reading the same cluster share one Karapace instance).+ */+@ApplicationScoped+class KarapaceManager {++ private static final Logger LOG = Logger.getLogger(KarapaceManager.class);+ private static final int REST_PORT = 8082;+ private static final Duration READY_TIMEOUT = Duration.ofSeconds(30);++ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final ContainerDetector containerDetector;+ private final EmulatorConfig config;+ private final PortAllocator portAllocator;+ private final DockerHostResolver dockerHostResolver;+ private final Duration readyTimeout;+ private final Map<String, URI> restBaseUriByTarget = new ConcurrentHashMap<>();+ private final Map<String, String> containerIdByTarget = new ConcurrentHashMap<>();++ @Inject+ KarapaceManager(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerDetector containerDetector,+ EmulatorConfig config,+ PortAllocator portAllocator,+ DockerHostResolver dockerHostResolver) {+ this(containerBuilder, lifecycleManager, containerDetector, config, portAllocator, dockerHostResolver,+ READY_TIMEOUT);+ }++ /** Test-only: overrides the readiness timeout so a failure case does not wait the full default. */+ KarapaceManager(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerDetector containerDetector,+ EmulatorConfig config,+ PortAllocator portAllocator,+ DockerHostResolver dockerHostResolver,+ Duration readyTimeout) {+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.containerDetector = containerDetector;+ this.config = config;+ this.portAllocator = portAllocator;+ this.dockerHostResolver = dockerHostResolver;+ this.readyTimeout = readyTimeout;+ }++ URI ensureStarted(String bootstrapServers) {+ return restBaseUriByTarget.computeIfAbsent(bootstrapServers, this::startContainer);+ }++ private URI startContainer(String bootstrapServers) {+ EmulatorConfig.KafkaRestBridgeConfig bridgeConfig = config.services().pipes().kafkaRestBridge();+ String targetKey = targetKey(bootstrapServers);+ String containerName = ContainerStorageHelper.resourceName(config, "pipes-kafka-rest", targetKey, targetKey);++ lifecycleManager.removeIfExists(containerName);++ List<String> env = List.of(+ "KARAPACE_HOST=0.0.0.0",+ "KARAPACE_PORT=" + REST_PORT,+ "KARAPACE_BOOTSTRAP_URI=" + resolveForContainer(bootstrapServers),+ "KARAPACE_ADVERTISED_PROTOCOL=http");++ ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(bridgeConfig.defaultImage())+ .withName(containerName)+ .withDockerNetwork(config.services().dockerNetwork())+ // Native Linux Docker does not auto-inject host.docker.internal the way Docker+ // Desktop does, and resolveForContainer rewrites a host-local broker to exactly+ // that alias, so the sidecar needs the host-gateway mapping to resolve it.+ .withHostDockerInternalOnLinux()+ .withLogRotation()+ // The image's default entrypoint (plain `python3`) does nothing; karapace_rest_proxy+ // is the REST Proxy's own dedicated process, config-driven entirely by KARAPACE_* env.+ .withEntrypoint(List.of("/venv/bin/karapace_rest_proxy"))+ .withEnv(env)+ .withLabels(Map.of("floci.component", "pipes-kafka-rest-bridge"));++ int hostPort = 0;+ if (!containerDetector.isRunningInContainer()) {+ hostPort = portAllocator.allocate(bridgeConfig.hostPortBase(), bridgeConfig.hostPortMax());+ specBuilder.withPortBinding(REST_PORT, hostPort);+ } else {+ specBuilder.withExposedPort(REST_PORT);+ }++ ContainerSpec spec = specBuilder.build();+ ContainerInfo info;+ try {+ info = lifecycleManager.createAndStart(spec);+ } catch (RuntimeException e) {+ if (!containerDetector.isRunningInContainer()) {+ portAllocator.release(hostPort);+ }+ throw e;+ }++ try {+ containerIdByTarget.put(bootstrapServers, info.containerId());+ EndpointInfo endpoint = info.getEndpoint(REST_PORT);+ URI restBaseUri = URI.create("http://" + endpoint.host() + ":" + endpoint.port() + "/");+ waitUntilReady(restBaseUri);+ LOG.infov("Karapace REST Proxy {0} started for bootstrap {1}", info.containerId(), bootstrapServers);+ return restBaseUri;+ } catch (RuntimeException e) {+ containerIdByTarget.remove(bootstrapServers);+ lifecycleManager.stopAndRemove(info.containerId(), null);+ if (!containerDetector.isRunningInContainer()) {+ portAllocator.release(hostPort);+ }+ throw e;+ }+ }++ /**+ * Rewrites a host-local address (localhost/127.0.0.1) so the sidecar, in its own network+ * namespace, can reach a broker on the Floci host instead of trying to reach itself. Applies+ * to both self-managed sources given as {@code smk://localhost:9092} and, in native mode, an+ * MSK-backed Redpanda cluster whose advertised address is also {@code localhost:<port>}.+ */+ private String resolveForContainer(String bootstrapServers) {+ return Arrays.stream(bootstrapServers.split(","))+ .map(this::resolveHostLocalAddress)+ .collect(Collectors.joining(","));+ }++ private String resolveHostLocalAddress(String hostPort) {+ int colonIndex = hostPort.lastIndexOf(':');+ if (colonIndex < 0) {+ return hostPort;+ }+ String host = hostPort.substring(0, colonIndex);+ if (host.equals("localhost") || host.equals("127.0.0.1")) {+ return dockerHostResolver.resolve() + hostPort.substring(colonIndex);+ }+ return hostPort;+ }++ private void waitUntilReady(URI restBaseUri) {+ long deadline = System.nanoTime() + readyTimeout.toNanos();+ while (System.nanoTime() < deadline) {+ if (isReady(restBaseUri)) {+ return;+ }+ try {+ Thread.sleep(Math.min(200, Math.max(1, readyTimeout.toMillis() / 4)));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException("Interrupted waiting for Karapace to become ready at " + restBaseUri, e);+ }+ }+ throw new IllegalStateException("Karapace REST Proxy at " + restBaseUri + " did not become ready within "+ + readyTimeout.toMillis() + "ms");+ }++ private boolean isReady(URI restBaseUri) {+ try {+ HttpURLConnection conn = (HttpURLConnection) restBaseUri.resolve("_health").toURL().openConnection();+ conn.setRequestMethod("GET");+ conn.setConnectTimeout(1000);+ conn.setReadTimeout(1000);+ return conn.getResponseCode() == 200;+ } catch (Exception e) {+ LOG.debugv("Karapace not ready yet at {0}: {1}", restBaseUri, e.getMessage());+ return false;+ }+ }++ private static String targetKey(String bootstrapServers) {+ try {+ MessageDigest digest = MessageDigest.getInstance("SHA-256");+ byte[] hash = digest.digest(bootstrapServers.getBytes(StandardCharsets.UTF_8));+ return HexFormat.of().formatHex(hash, 0, 8);+ } catch (NoSuchAlgorithmException e) {+ throw new IllegalStateException("SHA-256 is not available", e);+ }+ }++ @PreDestroy+ void shutdown() {+ containerIdByTarget.values().forEach(containerId -> {+ try {+ lifecycleManager.stopAndRemove(containerId, null);+ } catch (RuntimeException e) {+ LOG.warnv("Failed to stop Karapace container {0}: {1}", containerId, e.getMessage());+ }+ });+ containerIdByTarget.clear();+ restBaseUriByTarget.clear();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.javaindex 6962e7e12d..c30c092fcf 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java@@ -8,22 +8,21 @@ import io.github.hectorvent.floci.services.pipes.model.Pipe;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;-import org.apache.kafka.clients.consumer.ConsumerConfig;-import org.apache.kafka.clients.consumer.ConsumerRecords;-import org.apache.kafka.clients.consumer.KafkaConsumer;-import org.apache.kafka.clients.consumer.OffsetAndMetadata;-import org.apache.kafka.common.TopicPartition;-import org.apache.kafka.common.serialization.ByteArrayDeserializer;import org.jboss.logging.Logger;+import java.net.URI;import java.time.Duration;import java.util.ArrayList;import java.util.List;-import java.util.Map;-import java.util.Properties;-import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;+/**+ * Polls Kafka pipe sources (MSK and self-managed, {@code smk://}) over a Kafka REST Proxy+ * (Karapace) instead of embedding {@code kafka-clients} directly: see #2916, where kafka-clients+ * and the zstd-jni/lz4-java it pulls in were reachable in every build regardless of whether a+ * Pipe ever used a Kafka source. {@link KarapaceManager} starts a Karapace sidecar per distinct+ * target on demand, and {@link PipesKafkaRestClient} speaks its REST API.+ */@ApplicationScopedpublic class PipesKafkaConsumerManager {@@ -32,11 +31,16 @@ public class PipesKafkaConsumerManager {private static final String DEFAULT_STARTING_POSITION = "LATEST";private final MskService mskService;- private final ConcurrentHashMap<String, KafkaConsumer<byte[], byte[]>> consumers = new ConcurrentHashMap<>();+ private final KarapaceManager karapaceManager;+ private final PipesKafkaRestClient restClient;+ private final ConcurrentHashMap<String, KafkaConsumerHandle> consumers = new ConcurrentHashMap<>();@Inject- public PipesKafkaConsumerManager(MskService mskService) {+ public PipesKafkaConsumerManager(MskService mskService, KarapaceManager karapaceManager,+ PipesKafkaRestClient restClient) {this.mskService = mskService;+ this.karapaceManager = karapaceManager;+ this.restClient = restClient;}@PreDestroy@@ -45,39 +49,26 @@ public class PipesKafkaConsumerManager {consumers.clear();}- public ConsumerRecords<byte[], byte[]> poll(Pipe pipe) {- KafkaConsumer<byte[], byte[]> consumer = consumers.computeIfAbsent(pipe.getArn(), ignored -> createConsumer(pipe));+ public List<KafkaRecordDto> poll(Pipe pipe) {+ KafkaConsumerHandle consumer = consumers.computeIfAbsent(pipe.getArn(), ignored -> createConsumer(pipe));try {- return consumer.poll(POLL_TIMEOUT);+ return restClient.poll(consumer, POLL_TIMEOUT);} catch (RuntimeException e) {close(pipe);throw e;}}- public void commit(Pipe pipe) {- KafkaConsumer<byte[], byte[]> consumer = consumers.get(pipe.getArn());- if (consumer == null) {- return;- }- try {- consumer.commitSync();- } catch (RuntimeException e) {- close(pipe);- throw e;- }- }-- public void commit(Pipe pipe, Map<TopicPartition, OffsetAndMetadata> offsets) {+ public void commit(Pipe pipe, List<KafkaOffsetDto> offsets) {if (offsets.isEmpty()) {return;}- KafkaConsumer<byte[], byte[]> consumer = consumers.get(pipe.getArn());+ KafkaConsumerHandle consumer = consumers.get(pipe.getArn());if (consumer == null) {return;}try {- consumer.commitSync(offsets);+ restClient.commit(consumer, offsets);} catch (RuntimeException e) {close(pipe);throw e;@@ -85,7 +76,7 @@ public class PipesKafkaConsumerManager {}public void close(Pipe pipe) {- KafkaConsumer<byte[], byte[]> consumer = consumers.remove(pipe.getArn());+ KafkaConsumerHandle consumer = consumers.remove(pipe.getArn());if (consumer != null) {closeQuietly(consumer);}@@ -131,34 +122,25 @@ public class PipesKafkaConsumerManager {return resolveConsumerGroupId(pipe, kafkaParameters(pipe));}- /** Package-private so {@code PipesKafkaNativeSupportTest} guards the real settings, not a copy. */- Properties consumerProperties(Pipe pipe) {- JsonNode params = kafkaParameters(pipe);-- Properties properties = new Properties();- properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, resolveBootstrapServers(pipe));- properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());- properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());- properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");- properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, resolveOffsetReset(params));- properties.put(ConsumerConfig.GROUP_ID_CONFIG, resolveConsumerGroupId(pipe, params));- properties.put(ConsumerConfig.CLIENT_ID_CONFIG, "floci-pipes-" + UUID.randomUUID());- properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, Integer.toString(resolveBatchSize(pipe, 100)));- // On by default (KIP-714). Floci has no reason to push metrics to a user's broker, and it- // makes the shaded protobuf serializer reflectively reachable on a path no broker we test- // against exercises.- properties.put(ConsumerConfig.ENABLE_METRICS_PUSH_CONFIG, "false");- return properties;- }-- private KafkaConsumer<byte[], byte[]> createConsumer(Pipe pipe) {- Properties properties = consumerProperties(pipe);+ private KafkaConsumerHandle createConsumer(Pipe pipe) {+ String bootstrapServers = resolveBootstrapServers(pipe);String topicName = resolveTopicName(pipe);+ String groupId = resolveConsumerGroupId(pipe);+ String offsetReset = resolveOffsetReset(kafkaParameters(pipe));- KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(properties);- consumer.subscribe(List.of(topicName));- LOG.infov("Pipe {0}: subscribed Kafka consumer to topic {1} via {2}",- pipe.getName(), topicName, properties.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG));+ URI restBaseUri = karapaceManager.ensureStarted(bootstrapServers);+ KafkaConsumerHandle consumer = restClient.createConsumer(restBaseUri, groupId, offsetReset);+ try {+ restClient.subscribe(consumer, List.of(topicName));+ } catch (RuntimeException e) {+ // The consumer already exists server-side; this method's caller only learns about it+ // through the returned handle, so a failure here must delete it itself or it is never+ // reachable again to close, and every retry creates another orphaned server consumer.+ closeQuietly(consumer);+ throw e;+ }+ LOG.infov("Pipe {0}: subscribed Kafka REST consumer to topic {1} via {2}",+ pipe.getName(), topicName, bootstrapServers);return consumer;}@@ -210,11 +192,11 @@ public class PipesKafkaConsumerManager {throw new AwsException("ValidationException", "Unsupported Kafka source: " + pipe.getSource(), 400);}- private void closeQuietly(KafkaConsumer<byte[], byte[]> consumer) {+ private void closeQuietly(KafkaConsumerHandle consumer) {try {- consumer.close();+ restClient.close(consumer);} catch (Exception e) {- LOG.debugv("Ignoring Kafka consumer close error: {0}", e.getMessage());+ LOG.debugv("Ignoring Kafka REST consumer close error: {0}", e.getMessage());}}}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.javadeleted file mode 100644index 3fcf99168a..0000000000--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.java+++ /dev/null@@ -1,33 +0,0 @@-package io.github.hectorvent.floci.services.pipes;--import io.quarkus.runtime.annotations.RegisterForReflection;-import org.apache.kafka.clients.consumer.CooperativeStickyAssignor;-import org.apache.kafka.clients.consumer.RangeAssignor;-import org.apache.kafka.common.metrics.JmxReporter;-import org.apache.kafka.common.serialization.ByteArrayDeserializer;--/**- * Registers the kafka-clients classes that GraalVM cannot discover statically.- *- * <p>kafka-clients ships no native-image metadata and Floci uses it directly rather than through- * the Quarkus Kafka extension. {@code AbstractConfig} resolves {@code CLASS}-typed settings by- * name, so without these a native build cannot construct a consumer — which {@link PipesPoller}- * swallows per poll cycle, leaving the pipe RUNNING and delivering nothing.- *- * <p>Entries are exactly what {@link PipesKafkaConsumerManager#consumerProperties} resolves, and- * {@code PipesKafkaNativeSupportTest} reads that same method, so the two cannot drift.- *- * <p>SASL and SSL defaults are excluded: Floci is PLAINTEXT-only, and {@code DefaultJwtValidator}- * fails the native build by referencing jose4j, an optional dependency Floci omits.- */-@RegisterForReflection(targets = {- // key.deserializer / value.deserializer — set explicitly by PipesKafkaConsumerManager- ByteArrayDeserializer.class,- // partition.assignment.strategy — ConsumerConfig defaults- RangeAssignor.class,- CooperativeStickyAssignor.class,- // metric.reporters — ConsumerConfig default- JmxReporter.class-})-public class PipesKafkaNativeSupport {-}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClient.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClient.javanew file mode 100644index 0000000000..6e4cbf14b8--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClient.java@@ -0,0 +1,186 @@+package io.github.hectorvent.floci.services.pipes;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.NullNode;+import com.fasterxml.jackson.databind.node.ObjectNode;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;++import java.io.IOException;+import java.io.UncheckedIOException;+import java.net.URI;+import java.net.URLEncoder;+import java.net.http.HttpClient;+import java.net.http.HttpRequest;+import java.net.http.HttpResponse;+import java.nio.charset.StandardCharsets;+import java.time.Duration;+import java.util.ArrayList;+import java.util.Base64;+import java.util.List;+import java.util.concurrent.Executors;++/**+ * Thin REST client for the subset of the Kafka REST Proxy v2 API (Karapace, Redpanda's+ * Pandaproxy) that Pipes needs to consume a Kafka topic: create a consumer instance, subscribe,+ * poll for records, commit offsets, close. Uses the JDK {@link HttpClient}, the same approach+ * Floci already uses for other REST-shaped backends (e.g. {@code KubernetesApiClient}), instead+ * of embedding {@code kafka-clients} directly: see #2916, where kafka-clients (and the+ * zstd-jni/lz4-java it pulls in) was reachable in every native image regardless of whether a Pipe+ * ever used a Kafka source.+ *+ * <p>Every consumer instance lives on exactly one REST Proxy process for its whole lifetime (the+ * proxy holds the actual Kafka client state), so all calls for a given {@link KafkaConsumerHandle}+ * must keep going to the {@code base_uri} that instance's creation call returned.+ */+@ApplicationScoped+class PipesKafkaRestClient {++ private static final String CONSUMER_CONTENT_TYPE = "application/vnd.kafka.v2+json";+ private static final String BINARY_RECORDS_ACCEPT = "application/vnd.kafka.binary.v2+json";++ private final ObjectMapper objectMapper;+ private final HttpClient httpClient;++ @Inject+ PipesKafkaRestClient(ObjectMapper objectMapper) {+ this(objectMapper, HttpClient.newBuilder()+ .connectTimeout(Duration.ofSeconds(10))+ .executor(Executors.newVirtualThreadPerTaskExecutor())+ .build());+ }++ /** Test-only: injects an explicit client, e.g. one pointed at a fake REST Proxy server. */+ PipesKafkaRestClient(ObjectMapper objectMapper, HttpClient httpClient) {+ this.objectMapper = objectMapper;+ this.httpClient = httpClient;+ }++ KafkaConsumerHandle createConsumer(URI restBaseUri, String groupId, String offsetReset) {+ ObjectNode body = objectMapper.createObjectNode();+ body.put("format", "binary");+ body.put("auto.offset.reset", offsetReset);+ body.put("auto.commit.enable", "false");++ JsonNode response = send("POST", restBaseUri.resolve("consumers/" + urlEncode(groupId)), body);+ // Karapace returns base_uri as a path relative to the proxy root, not an absolute URL,+ // unlike the Confluent REST Proxy reference spec; resolve() handles both correctly.+ return new KafkaConsumerHandle(+ response.path("instance_id").asText(),+ restBaseUri.resolve(response.path("base_uri").asText()));+ }++ void subscribe(KafkaConsumerHandle consumer, List<String> topics) {+ ObjectNode body = objectMapper.createObjectNode();+ ArrayNode topicsNode = body.putArray("topics");+ topics.forEach(topicsNode::add);+ send("POST", instanceUri(consumer, "subscription"), body);+ }++ List<KafkaRecordDto> poll(KafkaConsumerHandle consumer, Duration timeout) {+ URI uri = URI.create(instanceUri(consumer, "records") + "?timeout=" + timeout.toMillis());+ JsonNode response = sendWithAccept("GET", uri, null, BINARY_RECORDS_ACCEPT);++ List<KafkaRecordDto> records = new ArrayList<>();+ response.forEach(node -> records.add(toRecord(node)));+ return records;+ }++ /**+ * Commits the given offsets. Karapace requires an explicit offsets list (unlike the Confluent+ * REST Proxy reference spec, an empty body is rejected with 400 rather than committing+ * everything consumed), so callers always compute what to commit themselves.+ */+ void commit(KafkaConsumerHandle consumer, List<KafkaOffsetDto> offsets) {+ if (offsets.isEmpty()) {+ return;+ }+ ObjectNode body = objectMapper.createObjectNode();+ ArrayNode offsetsNode = body.putArray("offsets");+ for (KafkaOffsetDto offset : offsets) {+ ObjectNode offsetNode = offsetsNode.addObject();+ offsetNode.put("topic", offset.topic());+ offsetNode.put("partition", offset.partition());+ offsetNode.put("offset", offset.offset());+ }+ send("POST", instanceUri(consumer, "offsets"), body);+ }++ /** No-op if the consumer instance is already gone (e.g. the REST Proxy container restarted). */+ void close(KafkaConsumerHandle consumer) {+ try {+ send("DELETE", consumer.baseUri(), null);+ } catch (PipesKafkaRestClientException e) {+ if (e.getStatusCode() != 404) {+ throw e;+ }+ }+ }++ private KafkaRecordDto toRecord(JsonNode node) {+ List<KafkaHeaderDto> headers = new ArrayList<>();+ for (JsonNode headerNode : node.path("headers")) {+ headers.add(new KafkaHeaderDto(+ headerNode.path("name").asText(),+ base64Decode(headerNode.path("value"))));+ }+ return new KafkaRecordDto(+ node.path("topic").asText(),+ node.path("partition").asInt(),+ node.path("offset").asLong(),+ node.path("timestamp").asLong(0),+ KafkaRecordDto.DEFAULT_TIMESTAMP_TYPE,+ base64Decode(node.path("key")),+ base64Decode(node.path("value")),+ headers);+ }++ private static byte[] base64Decode(JsonNode value) {+ if (value == null || value.isNull() || value.isMissingNode()) {+ return null;+ }+ return Base64.getDecoder().decode(value.asText());+ }++ private URI instanceUri(KafkaConsumerHandle consumer, String segment) {+ String base = consumer.baseUri().toString();+ return URI.create((base.endsWith("/") ? base : base + "/") + segment);+ }++ private JsonNode send(String method, URI uri, JsonNode body) {+ return sendWithAccept(method, uri, body, CONSUMER_CONTENT_TYPE);+ }++ private JsonNode sendWithAccept(String method, URI uri, JsonNode body, String accept) {+ try {+ HttpRequest.Builder builder = HttpRequest.newBuilder(uri)+ .timeout(Duration.ofSeconds(30))+ .header("Accept", accept);+ if (body != null) {+ builder.header("Content-Type", CONSUMER_CONTENT_TYPE)+ .method(method, HttpRequest.BodyPublishers.ofByteArray(objectMapper.writeValueAsBytes(body)));+ } else {+ builder.method(method, HttpRequest.BodyPublishers.noBody());+ }++ HttpResponse<byte[]> response = httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofByteArray());+ if (response.statusCode() / 100 != 2) {+ throw new PipesKafkaRestClientException(method, uri.toString(), response.statusCode(),+ new String(response.body(), StandardCharsets.UTF_8));+ }+ return response.body().length == 0 ? NullNode.getInstance() : objectMapper.readTree(response.body());+ } catch (IOException e) {+ throw new UncheckedIOException(+ "Kafka REST Proxy " + method + " " + uri + " failed: " + e.getMessage(), e);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException("Interrupted calling Kafka REST Proxy " + method + " " + uri, e);+ }+ }++ private static String urlEncode(String value) {+ return URLEncoder.encode(value, StandardCharsets.UTF_8);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientException.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientException.javanew file mode 100644index 0000000000..fb499145f9--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientException.java@@ -0,0 +1,17 @@+package io.github.hectorvent.floci.services.pipes;++/** A Kafka REST Proxy (Karapace) response outside the expected 2xx status for its call. */+class PipesKafkaRestClientException extends RuntimeException {++ private final int statusCode;++ PipesKafkaRestClientException(String method, String path, int statusCode, String body) {+ super("Kafka REST Proxy " + method + " " + path + " failed: HTTP " + statusCode+ + (body == null || body.isBlank() ? "" : " " + body));+ this.statusCode = statusCode;+ }++ int getStatusCode() {+ return statusCode;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.javaindex 7b9593ff00..c6e5c7e2a8 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java@@ -19,11 +19,6 @@ import io.vertx.core.Vertx;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;-import org.apache.kafka.clients.consumer.ConsumerRecord;-import org.apache.kafka.clients.consumer.ConsumerRecords;-import org.apache.kafka.clients.consumer.OffsetAndMetadata;-import org.apache.kafka.common.TopicPartition;-import org.apache.kafka.common.header.Header;import org.jboss.logging.Logger;import java.nio.charset.StandardCharsets;@@ -341,20 +336,41 @@ public class PipesPoller implements Resettable {}void pollKafka(Pipe pipe, String region) {- ConsumerRecords<byte[], byte[]> records = kafkaConsumerManager.poll(pipe);+ List<KafkaRecordDto> records = kafkaConsumerManager.poll(pipe);if (records.isEmpty()) {return;}- List<ConsumerRecord<byte[], byte[]>> batch = new ArrayList<>(records.count());- records.forEach(batch::add);+ LOG.infov("Pipe {0}: received {1} Kafka record(s)", pipe.getName(), records.size());+ int batchSize = kafkaConsumerManager.resolveBatchSize(pipe, DEFAULT_BATCH_SIZE);+ Set<KafkaTopicPartition> blockedPartitions = new HashSet<>();+ for (List<KafkaRecordDto> batch : partition(records, batchSize)) {+ deliverKafkaBatch(pipe, region, batch, blockedPartitions);+ }+ }- LOG.infov("Pipe {0}: received {1} Kafka record(s)", pipe.getName(), batch.size());- List<ObjectNode> deliveryNodes = buildKafkaRecordNodes(batch, pipe);- List<ObjectNode> filterNodes = buildKafkaFilterNodes(deliveryNodes, batch);+ /**+ * A REST Proxy poll can return more records than one Pipe {@code BatchSize} worth (unlike the+ * native consumer, it has no per-call record cap), so one poll cycle may deliver several+ * batches in sequence. {@code blockedPartitions} is shared across that whole sequence: once a+ * partition fails delivery in one batch, its records in every later batch this cycle are left+ * uncommitted too, so a later batch's success can never commit past an earlier record that is+ * still awaiting redelivery.+ */+ private void deliverKafkaBatch(Pipe pipe, String region, List<KafkaRecordDto> batch,+ Set<KafkaTopicPartition> blockedPartitions) {+ List<KafkaRecordDto> deliverable = batch.stream()+ .filter(record -> !blockedPartitions.contains(new KafkaTopicPartition(record.topic(), record.partition())))+ .toList();+ if (deliverable.isEmpty()) {+ return;+ }++ List<ObjectNode> deliveryNodes = buildKafkaRecordNodes(deliverable, pipe);+ List<ObjectNode> filterNodes = buildKafkaFilterNodes(deliveryNodes, deliverable);List<JsonNode> filtered = filterMatcher.applyFilterCriteria(new ArrayList<>(filterNodes), pipe.getSourceParameters());if (filtered.isEmpty()) {- kafkaConsumerManager.commit(pipe);+ commitWholeBatch(pipe, deliverable);return;}@@ -371,13 +387,13 @@ public class PipesPoller implements Resettable {}}if (deliveryRecords.isEmpty()) {- kafkaConsumerManager.commit(pipe);+ commitWholeBatch(pipe, deliverable);return;}int failed = isLambdaTarget(pipe)- ? deliverKafkaLambdaRecords(pipe, region, batch, deliveryRecordsByIdentity, filtered)- : deliverKafkaRecords(pipe, region, batch, deliveryRecordsByIdentity, filtered);+ ? deliverKafkaLambdaRecords(pipe, region, deliverable, deliveryRecordsByIdentity, filtered, blockedPartitions)+ : deliverKafkaRecords(pipe, region, deliverable, deliveryRecordsByIdentity, filtered, blockedPartitions);if (failed == 0) {return;}@@ -388,48 +404,51 @@ public class PipesPoller implements Resettable {private int deliverKafkaLambdaRecords(Pipe pipe,String region,- List<ConsumerRecord<byte[], byte[]>> batch,+ List<KafkaRecordDto> batch,Map<String, JsonNode> deliveryRecordsByIdentity,- List<JsonNode> filtered) {+ List<JsonNode> filtered,+ Set<KafkaTopicPartition> blockedPartitions) {Set<String> matchedIdentities = new HashSet<>(filtered.size());filtered.forEach(record -> matchedIdentities.add(kafkaRecordIdentity(record)));- Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = groupKafkaRecordsByPartition(batch);+ Map<KafkaTopicPartition, Long> offsetsToCommit = new HashMap<>();+ Map<KafkaTopicPartition, List<KafkaRecordDto>> recordsByPartition = groupKafkaRecordsByPartition(batch);int failed = 0;- for (Map.Entry<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> entry : recordsByPartition.entrySet()) {- failed += deliverKafkaLambdaPartition(pipe, region, entry.getKey(), entry.getValue(),+ for (Map.Entry<KafkaTopicPartition, List<KafkaRecordDto>> entry : recordsByPartition.entrySet()) {+ int partitionFailed = deliverKafkaLambdaPartition(pipe, region, entry.getKey(), entry.getValue(),deliveryRecordsByIdentity, matchedIdentities, offsetsToCommit);+ if (partitionFailed > 0) {+ blockedPartitions.add(entry.getKey());+ }+ failed += partitionFailed;}- if (!offsetsToCommit.isEmpty()) {- kafkaConsumerManager.commit(pipe, offsetsToCommit);- }+ commitOffsets(pipe, offsetsToCommit);return failed;}private int deliverKafkaLambdaPartition(Pipe pipe,String region,- TopicPartition partition,- List<ConsumerRecord<byte[], byte[]>> partitionRecords,+ KafkaTopicPartition partition,+ List<KafkaRecordDto> partitionRecords,Map<String, JsonNode> deliveryRecordsByIdentity,Set<String> matchedIdentities,- Map<TopicPartition, OffsetAndMetadata> offsetsToCommit) {+ Map<KafkaTopicPartition, Long> offsetsToCommit) {List<JsonNode> pendingBatch = new ArrayList<>();long pendingOffset = -1L;- for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {+ for (KafkaRecordDto record : partitionRecords) {String identity = kafkaRecordIdentity(record);if (!matchedIdentities.contains(identity)) {if (!pendingBatch.isEmpty()) {if (!invokeWithDlq(pipe, wrapRecords(pendingBatch), region)) {return pendingBatch.size();}- offsetsToCommit.put(partition, new OffsetAndMetadata(pendingOffset));+ offsetsToCommit.put(partition, pendingOffset);pendingBatch.clear();}- offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));+ offsetsToCommit.put(partition, record.offset() + 1);continue;}@@ -444,38 +463,38 @@ public class PipesPoller implements Resettable {if (!invokeWithDlq(pipe, wrapRecords(pendingBatch), region)) {return pendingBatch.size();}- offsetsToCommit.put(partition, new OffsetAndMetadata(pendingOffset));+ offsetsToCommit.put(partition, pendingOffset);}return 0;}private int deliverKafkaRecords(Pipe pipe,String region,- List<ConsumerRecord<byte[], byte[]>> batch,+ List<KafkaRecordDto> batch,Map<String, JsonNode> deliveryRecordsByIdentity,- List<JsonNode> filtered) {+ List<JsonNode> filtered,+ Set<KafkaTopicPartition> blockedPartitions) {Set<String> matchedIdentities = new HashSet<>(filtered.size());filtered.forEach(record -> matchedIdentities.add(kafkaRecordIdentity(record)));- Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();- Set<TopicPartition> blockedPartitions = new HashSet<>();+ Map<KafkaTopicPartition, Long> offsetsToCommit = new HashMap<>();int failed = 0;- for (ConsumerRecord<byte[], byte[]> record : batch) {- TopicPartition partition = new TopicPartition(record.topic(), record.partition());+ for (KafkaRecordDto record : batch) {+ KafkaTopicPartition partition = new KafkaTopicPartition(record.topic(), record.partition());if (blockedPartitions.contains(partition)) {continue;}String identity = kafkaRecordIdentity(record);if (!matchedIdentities.contains(identity)) {- offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));+ offsetsToCommit.put(partition, record.offset() + 1);continue;}JsonNode deliveryRecord = deliveryRecordsByIdentity.get(identity);if (deliveryRecord != null && invokeWithDlq(pipe, deliveryRecord.toString(), region)) {- offsetsToCommit.put(partition, new OffsetAndMetadata(record.offset() + 1));+ offsetsToCommit.put(partition, record.offset() + 1);continue;}@@ -483,9 +502,7 @@ public class PipesPoller implements Resettable {failed++;}- if (!offsetsToCommit.isEmpty()) {- kafkaConsumerManager.commit(pipe, offsetsToCommit);- }+ commitOffsets(pipe, offsetsToCommit);return failed;}@@ -722,11 +739,11 @@ public class PipesPoller implements Resettable {return nodes;}- private List<ObjectNode> buildKafkaRecordNodes(List<ConsumerRecord<byte[], byte[]>> records, Pipe pipe) {+ private List<ObjectNode> buildKafkaRecordNodes(List<KafkaRecordDto> records, Pipe pipe) {List<ObjectNode> nodes = new ArrayList<>();String eventSource = pipe.getSource().contains(":kafka:") ? "aws:kafka" : "SelfManagedKafka";String bootstrapServers = kafkaConsumerManager.resolveBootstrapServers(pipe);- for (ConsumerRecord<byte[], byte[]> record : records) {+ for (KafkaRecordDto record : records) {ObjectNode node = objectMapper.createObjectNode();node.put("eventSource", eventSource);if (pipe.getSource().contains(":kafka:")) {@@ -738,12 +755,12 @@ public class PipesPoller implements Resettable {node.put("partition", record.partition());node.put("offset", record.offset());node.put("timestamp", record.timestamp());- node.put("timestampType", record.timestampType().name());+ node.put("timestampType", record.timestampType());putKafkaBinaryField(node, "key", record.key());putKafkaBinaryField(node, "value", record.value());var headersNode = node.putArray("headers");- for (Header header : record.headers()) {+ for (KafkaHeaderDto header : record.headers()) {ObjectNode headerNode = objectMapper.createObjectNode();byte[] headerValue = header.value();if (headerValue == null) {@@ -761,12 +778,11 @@ public class PipesPoller implements Resettable {return nodes;}- private List<ObjectNode> buildKafkaFilterNodes(List<ObjectNode> deliveryNodes,- List<ConsumerRecord<byte[], byte[]>> records) {+ private List<ObjectNode> buildKafkaFilterNodes(List<ObjectNode> deliveryNodes, List<KafkaRecordDto> records) {List<ObjectNode> nodes = new ArrayList<>(deliveryNodes.size());for (int i = 0; i < deliveryNodes.size(); i++) {ObjectNode node = deliveryNodes.get(i).deepCopy();- ConsumerRecord<byte[], byte[]> record = records.get(i);+ KafkaRecordDto record = records.get(i);applyDecodedKafkaField(node, "key", record.key());applyDecodedKafkaField(node, "value", record.value());nodes.add(node);@@ -795,7 +811,7 @@ public class PipesPoller implements Resettable {+ record.path("offset").asText();}- private static String kafkaRecordIdentity(ConsumerRecord<byte[], byte[]> record) {+ private static String kafkaRecordIdentity(KafkaRecordDto record) {return record.topic() + ":" + record.partition() + ":" + record.offset();}@@ -822,16 +838,49 @@ public class PipesPoller implements Resettable {node.put(fieldName, base64(value));}- private Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> groupKafkaRecordsByPartition(- List<ConsumerRecord<byte[], byte[]>> records) {- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = new HashMap<>();- for (ConsumerRecord<byte[], byte[]> record : records) {- TopicPartition partition = new TopicPartition(record.topic(), record.partition());+ private Map<KafkaTopicPartition, List<KafkaRecordDto>> groupKafkaRecordsByPartition(List<KafkaRecordDto> records) {+ Map<KafkaTopicPartition, List<KafkaRecordDto>> recordsByPartition = new HashMap<>();+ for (KafkaRecordDto record : records) {+ KafkaTopicPartition partition = new KafkaTopicPartition(record.topic(), record.partition());recordsByPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);}return recordsByPartition;}+ /**+ * Commits past every record in {@code batch}, for when none of them are being delivered (no+ * filter match). The REST Proxy has no "commit everything consumed" shortcut (unlike the+ * native consumer's no-args {@code commitSync()}), so this computes the equivalent explicitly:+ * the highest offset seen per partition, plus one.+ */+ private void commitWholeBatch(Pipe pipe, List<KafkaRecordDto> batch) {+ Map<KafkaTopicPartition, Long> offsetsToCommit = new HashMap<>();+ for (KafkaRecordDto record : batch) {+ KafkaTopicPartition partition = new KafkaTopicPartition(record.topic(), record.partition());+ offsetsToCommit.merge(partition, record.offset() + 1, Math::max);+ }+ commitOffsets(pipe, offsetsToCommit);+ }++ private void commitOffsets(Pipe pipe, Map<KafkaTopicPartition, Long> offsetsToCommit) {+ if (offsetsToCommit.isEmpty()) {+ return;+ }+ List<KafkaOffsetDto> offsets = new ArrayList<>(offsetsToCommit.size());+ offsetsToCommit.forEach((partition, offset) ->+ offsets.add(new KafkaOffsetDto(partition.topic(), partition.partition(), offset)));+ kafkaConsumerManager.commit(pipe, offsets);+ }++ private static <T> List<List<T>> partition(List<T> list, int size) {+ int step = Math.max(1, size);+ List<List<T>> partitions = new ArrayList<>();+ for (int i = 0; i < list.size(); i += step) {+ partitions.add(list.subList(i, Math.min(i + step, list.size())));+ }+ return partitions;+ }+private static String pipeKey(Pipe pipe) {return pipe.getArn();}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex d09352c188..efa358b2f3 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -638,6 +638,10 @@ floci:enabled: truepipes:enabled: true+ kafka-rest-bridge:+ default-image: "ghcr.io/aiven-open/karapace:latest"+ host-port-base: 9500+ host-port-max: 9599configservice:enabled: truedocdb:diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/KarapaceManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/KarapaceManagerTest.javanew file mode 100644index 0000000000..fce9a62c9b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/KarapaceManagerTest.java@@ -0,0 +1,193 @@+package io.github.hectorvent.floci.services.pipes;++import com.sun.net.httpserver.HttpServer;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import io.github.hectorvent.floci.core.common.docker.PortAllocator;+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.extension.ExtendWith;+import org.mockito.ArgumentCaptor;+import org.mockito.Mock;+import org.mockito.junit.jupiter.MockitoExtension;++import java.net.InetSocketAddress;+import java.net.URI;+import java.nio.charset.StandardCharsets;+import java.util.Map;+import java.util.Optional;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.lenient;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++@ExtendWith(MockitoExtension.class)+class KarapaceManagerTest {++ private static final int REST_PORT = 8082;++ @Mock private ContainerLifecycleManager lifecycleManager;+ @Mock private ContainerDetector containerDetector;+ @Mock private PortAllocator portAllocator;+ @Mock private EmulatorConfig config;+ @Mock private DockerHostResolver dockerHostResolver;+ @Mock private EmbeddedDnsServer embeddedDnsServer;++ private KarapaceManager manager;+ private HttpServer fakeRestServer;++ @BeforeEach+ void setUp() {+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.PipesServiceConfig pipes = mock(EmulatorConfig.PipesServiceConfig.class);+ EmulatorConfig.KafkaRestBridgeConfig bridge = mock(EmulatorConfig.KafkaRestBridgeConfig.class);+ EmulatorConfig.DockerConfig docker = mock(EmulatorConfig.DockerConfig.class);++ lenient().when(config.services()).thenReturn(services);+ lenient().when(services.pipes()).thenReturn(pipes);+ lenient().when(services.dockerNetwork()).thenReturn(Optional.empty());+ lenient().when(pipes.kafkaRestBridge()).thenReturn(bridge);+ lenient().when(bridge.defaultImage()).thenReturn("ghcr.io/aiven-open/karapace:latest");+ lenient().when(bridge.hostPortBase()).thenReturn(9500);+ lenient().when(bridge.hostPortMax()).thenReturn(9599);+ lenient().when(config.docker()).thenReturn(docker);+ lenient().when(docker.logMaxSize()).thenReturn("10m");+ lenient().when(docker.logMaxFile()).thenReturn("3");++ lenient().when(embeddedDnsServer.getServerIp()).thenReturn(Optional.empty());++ ContainerBuilder containerBuilder = new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer);+ manager = new KarapaceManager(containerBuilder, lifecycleManager, containerDetector, config, portAllocator,+ dockerHostResolver, java.time.Duration.ofMillis(500));+ }++ @AfterEach+ void tearDown() {+ if (fakeRestServer != null) {+ fakeRestServer.stop(0);+ }+ }++ /** Stands in for Karapace's own {@code GET /_health}, which {@code isReady} polls. */+ private int startFakeRestServer() throws Exception {+ fakeRestServer = HttpServer.create(new InetSocketAddress("localhost", 0), 0);+ fakeRestServer.createContext("/_health", exchange -> {+ byte[] body = "{}".getBytes(StandardCharsets.UTF_8);+ exchange.sendResponseHeaders(200, body.length);+ exchange.getResponseBody().write(body);+ exchange.close();+ });+ fakeRestServer.start();+ return fakeRestServer.getAddress().getPort();+ }++ @Test+ void ensureStartedConfiguresBootstrapUriAndPublishesRestPort() throws Exception {+ int hostPort = startFakeRestServer();+ when(containerDetector.isRunningInContainer()).thenReturn(false);+ when(portAllocator.allocate(9500, 9599)).thenReturn(hostPort);++ ContainerInfo info = new ContainerInfo("container-1",+ Map.of(REST_PORT, new EndpointInfo("localhost", hostPort)));+ when(lifecycleManager.createAndStart(any())).thenReturn(info);++ URI restBaseUri = manager.ensureStarted("broker-1:9092");++ assertEquals("http://localhost:" + hostPort + "/", restBaseUri.toString());++ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);+ verify(lifecycleManager).createAndStart(specCaptor.capture());+ ContainerSpec spec = specCaptor.getValue();+ assertTrue(spec.env().contains("KARAPACE_BOOTSTRAP_URI=broker-1:9092"),+ "Karapace must be pointed at the target cluster's bootstrap servers");+ assertEquals(Integer.valueOf(hostPort), spec.portBindings().get(REST_PORT));+ }++ @Test+ void ensureStartedReusesTheSameContainerForTheSameTarget() throws Exception {+ int hostPort = startFakeRestServer();+ when(containerDetector.isRunningInContainer()).thenReturn(false);+ when(portAllocator.allocate(9500, 9599)).thenReturn(hostPort);++ ContainerInfo info = new ContainerInfo("container-1",+ Map.of(REST_PORT, new EndpointInfo("localhost", hostPort)));+ when(lifecycleManager.createAndStart(any())).thenReturn(info);++ URI first = manager.ensureStarted("broker-1:9092");+ URI second = manager.ensureStarted("broker-1:9092");++ assertSame(first, second, "multiple pipes reading the same broker must share one Karapace instance");+ verify(lifecycleManager, times(1)).createAndStart(any());+ }++ @Test+ void ensureStartedRewritesHostLocalBrokerForTheSidecar() throws Exception {+ int hostPort = startFakeRestServer();+ when(containerDetector.isRunningInContainer()).thenReturn(false);+ when(portAllocator.allocate(9500, 9599)).thenReturn(hostPort);+ when(dockerHostResolver.resolve()).thenReturn("host.docker.internal");++ ContainerInfo info = new ContainerInfo("container-1",+ Map.of(REST_PORT, new EndpointInfo("localhost", hostPort)));+ when(lifecycleManager.createAndStart(any())).thenReturn(info);++ manager.ensureStarted("localhost:9092");++ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);+ verify(lifecycleManager).createAndStart(specCaptor.capture());+ assertTrue(specCaptor.getValue().env().contains("KARAPACE_BOOTSTRAP_URI=host.docker.internal:9092"),+ "a host-local broker address must be rewritten so the sidecar, in its own network "+ + "namespace, reaches the Floci host instead of itself");+ }++ @Test+ void ensureStartedAddsTheLinuxHostGatewayAliasSoRewrittenAddressesResolve() throws Exception {+ int hostPort = startFakeRestServer();+ when(containerDetector.isRunningInContainer()).thenReturn(false);+ when(portAllocator.allocate(9500, 9599)).thenReturn(hostPort);+ when(dockerHostResolver.resolve()).thenReturn("host.docker.internal");+ when(dockerHostResolver.isLinuxHost()).thenReturn(true);++ ContainerInfo info = new ContainerInfo("container-1",+ Map.of(REST_PORT, new EndpointInfo("localhost", hostPort)));+ when(lifecycleManager.createAndStart(any())).thenReturn(info);++ manager.ensureStarted("localhost:9092");++ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);+ verify(lifecycleManager).createAndStart(specCaptor.capture());+ assertTrue(specCaptor.getValue().extraHosts().contains("host.docker.internal:host-gateway"),+ "native Linux Docker does not auto-inject host.docker.internal, so the sidecar needs "+ + "the host-gateway mapping to resolve the address resolveForContainer rewrote to it");+ }++ @Test+ void ensureStartedCleansUpWhenReadinessNeverArrives() {+ when(containerDetector.isRunningInContainer()).thenReturn(false);+ when(portAllocator.allocate(9500, 9599)).thenReturn(54321);++ ContainerInfo info = new ContainerInfo("container-1",+ Map.of(REST_PORT, new EndpointInfo("localhost", 1)));+ when(lifecycleManager.createAndStart(any())).thenReturn(info);++ org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class,+ () -> manager.ensureStarted("broker-1:9092"));++ verify(lifecycleManager).stopAndRemove("container-1", null);+ verify(portAllocator).release(54321);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.javaindex 125a87775f..3cdde4f8cc 100644--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManagerTest.java@@ -8,8 +8,15 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.mockito.Mockito;+import java.net.URI;+import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyList;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;class PipesKafkaConsumerManagerTest {@@ -17,12 +24,45 @@ class PipesKafkaConsumerManagerTest {private static final ObjectMapper MAPPER = new ObjectMapper();private MskService mskService;+ private KarapaceManager karapaceManager;+ private PipesKafkaRestClient restClient;private PipesKafkaConsumerManager manager;@BeforeEachvoid setUp() {mskService = Mockito.mock(MskService.class);- manager = new PipesKafkaConsumerManager(mskService);+ karapaceManager = Mockito.mock(KarapaceManager.class);+ restClient = Mockito.mock(PipesKafkaRestClient.class);+ manager = new PipesKafkaConsumerManager(mskService, karapaceManager, restClient);+ }++ @Test+ void pollClosesOrphanedConsumerWhenSubscriptionFails() throws Exception {+ Pipe pipe = new Pipe();+ pipe.setArn("arn:aws:pipes:us-east-1:000000000000:pipe/orders-pipe");+ pipe.setName("orders-pipe");+ pipe.setSource("smk://broker-1:9092");+ pipe.setSourceParameters(MAPPER.readTree("""+ {+ "SelfManagedKafkaParameters": {+ "TopicName": "orders"+ }+ }+ """));++ URI restBaseUri = URI.create("http://localhost:8082/");+ KafkaConsumerHandle handle = new KafkaConsumerHandle("instance-1",+ URI.create("http://localhost:8082/consumers/g/instances/instance-1"));+ when(karapaceManager.ensureStarted("broker-1:9092")).thenReturn(restBaseUri);+ when(restClient.createConsumer(eq(restBaseUri), anyString(), anyString())).thenReturn(handle);+ Mockito.doThrow(new RuntimeException("subscribe failed"))+ .when(restClient).subscribe(eq(handle), anyList());++ assertThrows(RuntimeException.class, () -> manager.poll(pipe));++ // The consumer already exists server-side once createConsumer succeeds; a failed+ // subscribe must delete it here, since the map never learns its handle to close it later.+ verify(restClient).close(handle);}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.javadeleted file mode 100644index d73542e0b9..0000000000--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.java+++ /dev/null@@ -1,122 +0,0 @@-package io.github.hectorvent.floci.services.pipes;--import com.fasterxml.jackson.databind.ObjectMapper;-import io.quarkus.runtime.annotations.RegisterForReflection;-import io.github.hectorvent.floci.services.msk.MskService;-import io.github.hectorvent.floci.services.pipes.model.Pipe;-import org.apache.kafka.clients.consumer.ConsumerConfig;-import org.junit.jupiter.api.DisplayName;-import org.junit.jupiter.api.Test;-import org.mockito.Mockito;--import java.util.Collection;-import java.util.Map;-import java.util.Optional;-import java.util.Set;-import java.util.TreeSet;-import java.util.function.Predicate;--import static org.junit.jupiter.api.Assertions.assertNotNull;-import static org.junit.jupiter.api.Assertions.assertTrue;--/**- * Pins {@link PipesKafkaNativeSupport} to what {@code ConsumerConfig} resolves from- * {@link PipesKafkaConsumerManager#consumerProperties}, both ways: a newly resolved class must be- * registered, and nothing else may be. Reading that method rather than copying it means a new- * setting there is covered here too.- *- * <p>Scope: config-resolved classes only. Reflection always succeeds on the JVM, so a non-config- * lookup from a future kafka-clients needs a Kafka pipe run against a native binary. The- * compatibility suite already runs against the native image; it has no Kafka source case yet.- */-class PipesKafkaNativeSupportTest {-- private static final ObjectMapper MAPPER = new ObjectMapper();-- private static final Predicate<String> TRANSPORT_SECURITY =- key -> key.startsWith("sasl.") || key.startsWith("ssl.");-- @Test- @DisplayName("classes ConsumerConfig resolves reflectively are registered")- void registrationCoversReflectivelyResolvedClasses() throws Exception {- Set<String> missing = new TreeSet<>(expectedClassNames());- missing.removeAll(registeredClassNames());-- assertTrue(missing.isEmpty(),- "kafka-clients resolves these reflectively but PipesKafkaNativeSupport omits them, "- + "so a native build cannot construct a consumer: " + missing);- }-- @Test- @DisplayName("nothing is registered that ConsumerConfig does not resolve")- void registrationContainsNothingUnjustified() throws Exception {- Set<String> unexpected = new TreeSet<>(registeredClassNames());- unexpected.removeAll(expectedClassNames());-- assertTrue(unexpected.isEmpty(),- "these are not config-resolved, so this test cannot guard them and they need a "- + "native run to justify. SASL/SSL defaults in particular break the build — "- + "DefaultJwtValidator drags in optional jose4j: " + unexpected);- }-- /** Class-valued settings kafka-clients resolves, minus the transport security Floci never uses. */- private static Set<String> expectedClassNames() throws Exception {- Set<String> classes = new TreeSet<>();- consumerConfigValues().forEach((key, value) -> {- if (TRANSPORT_SECURITY.negate().test(key)) {- collectClassNames(value, classes);- }- });- return classes;- }-- private static Set<String> registeredClassNames() {- RegisterForReflection annotation =- PipesKafkaNativeSupport.class.getAnnotation(RegisterForReflection.class);- assertNotNull(annotation, "PipesKafkaNativeSupport must carry @RegisterForReflection");-- Set<String> names = new TreeSet<>(Set.of(annotation.classNames()));- for (Class<?> target : annotation.targets()) {- names.add(target.getName());- }- return names;- }-- /**- * Mirrors {@code AbstractConfig.getConfiguredInstances}: a setting may be a {@code Class} or a- * class name — {@code metric.reporters} is the latter. Values naming no class are not candidates.- */- private static void collectClassNames(Object value, Set<String> into) {- switch (value) {- case Class<?> type -> into.add(type.getName());- case Collection<?> collection -> collection.forEach(element -> collectClassNames(element, into));- case String name -> loadable(name).ifPresent(into::add);- case null, default -> { }- }- }-- private static Optional<String> loadable(String className) {- try {- return Optional.of(Class.forName(className, false, ConsumerConfig.class.getClassLoader()).getName());- } catch (ClassNotFoundException | LinkageError e) {- return Optional.empty();- }- }-- private static Map<String, ?> consumerConfigValues() throws Exception {- PipesKafkaConsumerManager manager = new PipesKafkaConsumerManager(Mockito.mock(MskService.class));-- Pipe pipe = new Pipe();- pipe.setName("native-support");- pipe.setSource("smk://localhost:9092");- pipe.setSourceParameters(MAPPER.readTree("""- {- "SelfManagedKafkaParameters": {- "TopicName": "orders"- }- }- """));-- return new ConsumerConfig(manager.consumerProperties(pipe)).values();- }-}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientTest.javanew file mode 100644index 0000000000..8072d23354--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaRestClientTest.java@@ -0,0 +1,179 @@+package io.github.hectorvent.floci.services.pipes;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.sun.net.httpserver.HttpServer;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.net.InetSocketAddress;+import java.net.URI;+import java.net.http.HttpClient;+import java.nio.charset.StandardCharsets;+import java.time.Duration;+import java.util.Base64;+import java.util.List;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/** Exercises {@link PipesKafkaRestClient} against a fake server speaking Karapace's REST Proxy API. */+class PipesKafkaRestClientTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();++ private HttpServer fakeServer;+ private PipesKafkaRestClient client;+ private final AtomicReference<String> lastRequestBody = new AtomicReference<>();+ private final AtomicReference<String> lastRequestMethod = new AtomicReference<>();+ private final AtomicReference<String> lastRequestPath = new AtomicReference<>();++ @BeforeEach+ void setUp() throws Exception {+ fakeServer = HttpServer.create(new InetSocketAddress("localhost", 0), 0);++ fakeServer.createContext("/consumers/orders-group", exchange -> {+ record(exchange);+ // Karapace returns base_uri as a path relative to the proxy root (not an absolute URL+ // as the Confluent REST Proxy reference spec shows) - this fixture matches that.+ String response = "{\"instance_id\":\"rest-consumer-1\","+ + "\"base_uri\":\"consumers/orders-group/instances/rest-consumer-1\"}";+ respond(exchange, 200, response);+ });++ fakeServer.createContext("/consumers/orders-group/instances/rest-consumer-1/subscription", exchange -> {+ record(exchange);+ respond(exchange, 204, "");+ });++ fakeServer.createContext("/consumers/orders-group/instances/rest-consumer-1/records", exchange -> {+ record(exchange);+ byte[] key = "customer-1".getBytes(StandardCharsets.UTF_8);+ byte[] value = "{\"status\":\"active\"}".getBytes(StandardCharsets.UTF_8);+ byte[] headerValue = "trace-1".getBytes(StandardCharsets.UTF_8);+ String response = """+ [+ {+ "topic": "orders",+ "partition": 0,+ "offset": 42,+ "timestamp": 1700000000000,+ "key": "%s",+ "value": "%s",+ "headers": [{"name": "traceId", "value": "%s"}]+ }+ ]+ """.formatted(base64(key), base64(value), base64(headerValue));+ respond(exchange, 200, response);+ });++ fakeServer.createContext("/consumers/orders-group/instances/rest-consumer-1/offsets", exchange -> {+ record(exchange);+ respond(exchange, 200, "");+ });++ fakeServer.createContext("/consumers/orders-group/instances/rest-consumer-1", exchange -> {+ if ("DELETE".equals(exchange.getRequestMethod())) {+ record(exchange);+ respond(exchange, 204, "");+ } else {+ exchange.sendResponseHeaders(404, -1);+ exchange.close();+ }+ });++ fakeServer.start();+ client = new PipesKafkaRestClient(MAPPER, HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build());+ }++ @AfterEach+ void tearDown() {+ fakeServer.stop(0);+ }++ private URI restBaseUri() {+ return URI.create("http://localhost:" + fakeServer.getAddress().getPort() + "/");+ }++ private void record(com.sun.net.httpserver.HttpExchange exchange) throws java.io.IOException {+ lastRequestMethod.set(exchange.getRequestMethod());+ lastRequestPath.set(exchange.getRequestURI().toString());+ lastRequestBody.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));+ }++ private static void respond(com.sun.net.httpserver.HttpExchange exchange, int status, String body) throws java.io.IOException {+ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);+ exchange.sendResponseHeaders(status, bytes.length == 0 ? -1 : bytes.length);+ if (bytes.length > 0) {+ exchange.getResponseBody().write(bytes);+ }+ exchange.close();+ }++ private static String base64(byte[] value) {+ return Base64.getEncoder().encodeToString(value);+ }++ @Test+ void createConsumerParsesInstanceIdAndBaseUri() {+ KafkaConsumerHandle consumer = client.createConsumer(restBaseUri(), "orders-group", "latest");++ assertEquals("rest-consumer-1", consumer.instanceId());+ assertEquals("POST", lastRequestMethod.get());+ assertTrue(consumer.baseUri().toString().endsWith("/consumers/orders-group/instances/rest-consumer-1"));+ }++ @Test+ void subscribeSendsTopicList() throws Exception {+ KafkaConsumerHandle consumer = client.createConsumer(restBaseUri(), "orders-group", "earliest");++ client.subscribe(consumer, List.of("orders"));++ JsonNode body = MAPPER.readTree(lastRequestBody.get());+ assertEquals("orders", body.path("topics").get(0).asText());+ }++ @Test+ void pollDecodesBinaryKeyValueAndHeaders() {+ KafkaConsumerHandle consumer = client.createConsumer(restBaseUri(), "orders-group", "latest");++ List<KafkaRecordDto> records = client.poll(consumer, Duration.ofMillis(100));++ assertEquals(1, records.size());+ KafkaRecordDto record = records.get(0);+ assertEquals("orders", record.topic());+ assertEquals(0, record.partition());+ assertEquals(42L, record.offset());+ assertEquals(1700000000000L, record.timestamp());+ assertArrayEquals("customer-1".getBytes(StandardCharsets.UTF_8), record.key());+ assertArrayEquals("{\"status\":\"active\"}".getBytes(StandardCharsets.UTF_8), record.value());+ assertEquals(1, record.headers().size());+ assertEquals("traceId", record.headers().get(0).key());+ assertArrayEquals("trace-1".getBytes(StandardCharsets.UTF_8), record.headers().get(0).value());+ }++ @Test+ void commitSpecificOffsetsSendsOffsetsArray() throws Exception {+ KafkaConsumerHandle consumer = client.createConsumer(restBaseUri(), "orders-group", "latest");++ client.commit(consumer, List.of(new KafkaOffsetDto("orders", 0, 43L)));++ JsonNode body = MAPPER.readTree(lastRequestBody.get());+ JsonNode offset = body.path("offsets").get(0);+ assertEquals("orders", offset.path("topic").asText());+ assertEquals(0, offset.path("partition").asInt());+ assertEquals(43L, offset.path("offset").asLong());+ }++ @Test+ void closeDeletesTheConsumerInstance() {+ KafkaConsumerHandle consumer = client.createConsumer(restBaseUri(), "orders-group", "latest");++ client.close(consumer);++ assertEquals("DELETE", lastRequestMethod.get());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.javaindex c9cc6abdc0..9524d2db6a 100644--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java@@ -9,10 +9,6 @@ import io.github.hectorvent.floci.services.pipes.model.Pipe;import io.github.hectorvent.floci.services.sqs.SqsService;import io.github.hectorvent.floci.services.sqs.model.Message;import io.vertx.core.Vertx;-import org.apache.kafka.clients.consumer.ConsumerRecord;-import org.apache.kafka.clients.consumer.ConsumerRecords;-import org.apache.kafka.clients.consumer.OffsetAndMetadata;-import org.apache.kafka.common.TopicPartition;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;@@ -20,20 +16,18 @@ import org.mockito.ArgumentCaptor;import org.mockito.Mock;import org.mockito.junit.jupiter.MockitoExtension;-import java.nio.charset.StandardCharsets;import java.util.List;-import java.util.Map;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;-import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyInt;-import static org.mockito.ArgumentMatchers.anyMap;+import static org.mockito.ArgumentMatchers.anyList;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.lenient;import static org.mockito.Mockito.never;-import static org.mockito.Mockito.verifyNoMoreInteractions;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@@ -55,6 +49,7 @@ class PipesPollerTest {@BeforeEachvoid setUp() {when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ lenient().when(kafkaConsumerManager.resolveBatchSize(any(), anyInt())).thenReturn(10);poller = new PipesPoller(vertx, sqsService, kinesisService, dynamoDbStreamService,kafkaConsumerManager, targetInvoker, new PipesFilterMatcher(MAPPER), MAPPER, config);}@@ -147,19 +142,19 @@ class PipesPollerTest {@Testvoid pollKafka_filtersUsingDecodedPayloadButDeliversOriginalRecord() throws Exception {Pipe pipe = selfManagedKafkaPipe();- byte[] key = "customer-123".getBytes(StandardCharsets.UTF_8);- byte[] value = "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8);- ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>("orders", 0, 42L, key, value);- record.headers().add("traceId", "abc123".getBytes(StandardCharsets.UTF_8));+ byte[] key = "customer-123".getBytes(java.nio.charset.StandardCharsets.UTF_8);+ byte[] value = "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8);+ byte[] traceId = "abc123".getBytes(java.nio.charset.StandardCharsets.UTF_8);+ KafkaRecordDto record = kafkaRecord("orders", 0, 42L, key, value, new KafkaHeaderDto("traceId", traceId));- when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));+ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(record));when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");poller.pollKafka(pipe, "us-east-1");ArgumentCaptor<String> payloadCaptor = ArgumentCaptor.forClass(String.class);verify(targetInvoker).invoke(eq(pipe), payloadCaptor.capture(), eq("us-east-1"));- verify(kafkaConsumerManager).commit(eq(pipe), anyMap());+ verify(kafkaConsumerManager).commit(eq(pipe), anyList());JsonNode delivered = MAPPER.readTree(payloadCaptor.getValue());assertEquals("orders", delivered.path("topic").asText());@@ -177,29 +172,28 @@ class PipesPollerTest {@Testvoid pollKafka_doesNotCommitWhenDeliveryFails() throws Exception {Pipe pipe = selfManagedKafkaPipe();- ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>(- "orders", 0, 7L, null, "{\"status\":\"active\"}".getBytes(StandardCharsets.UTF_8));+ KafkaRecordDto record = kafkaRecord("orders", 0, 7L, null,+ "{\"status\":\"active\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));- when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));+ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(record));when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");org.mockito.Mockito.doThrow(new RuntimeException("boom")).when(targetInvoker).invoke(eq(pipe), anyString(), eq("us-east-1"));poller.pollKafka(pipe, "us-east-1");- verify(kafkaConsumerManager, never()).commit(pipe);- verify(kafkaConsumerManager, never()).commit(eq(pipe), anyMap());+ verify(kafkaConsumerManager, never()).commit(eq(pipe), anyList());}@Testvoid pollKafka_commitsDeliveredPrefixWhenLaterRecordFails() throws Exception {Pipe pipe = selfManagedKafkaPipe();- ConsumerRecord<byte[], byte[]> first = new ConsumerRecord<>(- "orders", 0, 0L, null, "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8));- ConsumerRecord<byte[], byte[]> second = new ConsumerRecord<>(- "orders", 0, 1L, null, "{\"status\":\"active\",\"id\":\"order-2\"}".getBytes(StandardCharsets.UTF_8));+ KafkaRecordDto first = kafkaRecord("orders", 0, 0L, null,+ "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));+ KafkaRecordDto second = kafkaRecord("orders", 0, 1L, null,+ "{\"status\":\"active\",\"id\":\"order-2\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));- when(kafkaConsumerManager.poll(pipe)).thenReturn(records(first, second));+ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(first, second));when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");org.mockito.Mockito.doNothing().doThrow(new RuntimeException("boom"))@@ -207,17 +201,42 @@ class PipesPollerTest {poller.pollKafka(pipe, "us-east-1");- ArgumentCaptor<Map<TopicPartition, OffsetAndMetadata>> offsetsCaptor = ArgumentCaptor.forClass(Map.class);+ @SuppressWarnings("unchecked")+ ArgumentCaptor<List<KafkaOffsetDto>> offsetsCaptor = ArgumentCaptor.forClass(List.class);verify(kafkaConsumerManager).commit(eq(pipe), offsetsCaptor.capture());- assertEquals(1L, offsetsCaptor.getValue().get(new TopicPartition("orders", 0)).offset());+ assertEquals(1L, singleOffset(offsetsCaptor.getValue(), "orders", 0));+ }++ @Test+ void pollKafka_laterGeneratedBatchDoesNotCommitPastAnEarlierFailureInTheSamePartition() throws Exception {+ // A REST Proxy poll can return more than one BatchSize worth of records, so pollKafka+ // splits them into several generated batches. If the first batch fails to deliver record 0+ // and a later batch then succeeds delivering record 1 from the same partition, the second+ // batch's commit must not advance past record 0, or it is lost forever.+ Pipe pipe = selfManagedKafkaPipe();+ when(kafkaConsumerManager.resolveBatchSize(eq(pipe), anyInt())).thenReturn(1);+ KafkaRecordDto first = kafkaRecord("orders", 0, 0L, null,+ "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));+ KafkaRecordDto second = kafkaRecord("orders", 0, 1L, null,+ "{\"status\":\"active\",\"id\":\"order-2\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));++ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(first, second));+ when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");+ org.mockito.Mockito.doThrow(new RuntimeException("boom"))+ .doNothing()+ .when(targetInvoker).invoke(eq(pipe), anyString(), eq("us-east-1"));++ poller.pollKafka(pipe, "us-east-1");++ verify(kafkaConsumerManager, never()).commit(eq(pipe), anyList());}@Testvoid pollKafka_representsNullKeyAndValueAsJsonNull() throws Exception {Pipe pipe = nullableSelfManagedKafkaPipe();- ConsumerRecord<byte[], byte[]> record = new ConsumerRecord<>("orders", 0, 3L, null, null);+ KafkaRecordDto record = kafkaRecord("orders", 0, 3L, null, null);- when(kafkaConsumerManager.poll(pipe)).thenReturn(records(record));+ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(record));when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");poller.pollKafka(pipe, "us-east-1");@@ -232,14 +251,14 @@ class PipesPollerTest {@Testvoid pollKafka_lambdaCommitsSuccessfulPrefixBeforeLaterFailure() throws Exception {Pipe pipe = lambdaSelfManagedKafkaPipe();- ConsumerRecord<byte[], byte[]> first = new ConsumerRecord<>(- "orders", 0, 0L, null, "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(StandardCharsets.UTF_8));- ConsumerRecord<byte[], byte[]> skipped = new ConsumerRecord<>(- "orders", 0, 1L, null, "{\"status\":\"inactive\",\"id\":\"order-2\"}".getBytes(StandardCharsets.UTF_8));- ConsumerRecord<byte[], byte[]> failing = new ConsumerRecord<>(- "orders", 0, 2L, null, "{\"status\":\"active\",\"id\":\"order-3\"}".getBytes(StandardCharsets.UTF_8));-- when(kafkaConsumerManager.poll(pipe)).thenReturn(records(first, skipped, failing));+ KafkaRecordDto first = kafkaRecord("orders", 0, 0L, null,+ "{\"status\":\"active\",\"id\":\"order-1\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));+ KafkaRecordDto skipped = kafkaRecord("orders", 0, 1L, null,+ "{\"status\":\"inactive\",\"id\":\"order-2\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));+ KafkaRecordDto failing = kafkaRecord("orders", 0, 2L, null,+ "{\"status\":\"active\",\"id\":\"order-3\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8));++ when(kafkaConsumerManager.poll(pipe)).thenReturn(List.of(first, skipped, failing));when(kafkaConsumerManager.resolveBootstrapServers(pipe)).thenReturn("broker-1:9092");org.mockito.Mockito.doNothing().doThrow(new RuntimeException("boom"))@@ -247,9 +266,10 @@ class PipesPollerTest {poller.pollKafka(pipe, "us-east-1");- ArgumentCaptor<Map<TopicPartition, OffsetAndMetadata>> offsetsCaptor = ArgumentCaptor.forClass(Map.class);+ @SuppressWarnings("unchecked")+ ArgumentCaptor<List<KafkaOffsetDto>> offsetsCaptor = ArgumentCaptor.forClass(List.class);verify(kafkaConsumerManager).commit(eq(pipe), offsetsCaptor.capture());- assertEquals(2L, offsetsCaptor.getValue().get(new TopicPartition("orders", 0)).offset());+ assertEquals(2L, singleOffset(offsetsCaptor.getValue(), "orders", 0));}private Pipe selfManagedKafkaPipe() throws Exception {@@ -300,12 +320,17 @@ class PipesPollerTest {return pipe;}- private ConsumerRecords<byte[], byte[]> records(ConsumerRecord<byte[], byte[]>... records) {- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> byPartition = new java.util.LinkedHashMap<>();- for (ConsumerRecord<byte[], byte[]> record : records) {- TopicPartition topicPartition = new TopicPartition(record.topic(), record.partition());- byPartition.computeIfAbsent(topicPartition, ignored -> new java.util.ArrayList<>()).add(record);- }- return new ConsumerRecords<>(byPartition);+ private static KafkaRecordDto kafkaRecord(String topic, int partition, long offset, byte[] key, byte[] value,+ KafkaHeaderDto... headers) {+ return new KafkaRecordDto(topic, partition, offset, System.currentTimeMillis(),+ KafkaRecordDto.DEFAULT_TIMESTAMP_TYPE, key, value, List.of(headers));+ }++ private static long singleOffset(List<KafkaOffsetDto> offsets, String topic, int partition) {+ return offsets.stream()+ .filter(offset -> offset.topic().equals(topic) && offset.partition() == partition)+ .mapToLong(KafkaOffsetDto::offset)+ .findFirst()+ .orElseThrow();}}