Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(pipes): register kafka-clients reflection metadata for native image (#2068)

Liam MacPhersonab733798a38d
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java
index a9c80cdddd..6962e7e12d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java
@@ -131,13 +131,12 @@ public class PipesKafkaConsumerManager {
return resolveConsumerGroupId(pipe, kafkaParameters(pipe));
}
- private KafkaConsumer<byte[], byte[]> createConsumer(Pipe pipe) {
- String bootstrapServers = resolveBootstrapServers(pipe);
- String topicName = resolveTopicName(pipe);
+ /** Package-private so {@code PipesKafkaNativeSupportTest} guards the real settings, not a copy. */
+ Properties consumerProperties(Pipe pipe) {
JsonNode params = kafkaParameters(pipe);
Properties properties = new Properties();
- properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, resolveBootstrapServers(pipe));
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
@@ -145,11 +144,21 @@ public class PipesKafkaConsumerManager {
properties.put(ConsumerConfig.GROUP_ID_CONFIG, resolveConsumerGroupId(pipe, params));
properties.put(ConsumerConfig.CLIENT_ID_CONFIG, "floci-pipes-" + UUID.randomUUID());
properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, Integer.toString(resolveBatchSize(pipe, 100)));
+ // On by default (KIP-714). Floci has no reason to push metrics to a user's broker, and it
+ // makes the shaded protobuf serializer reflectively reachable on a path no broker we test
+ // against exercises.
+ properties.put(ConsumerConfig.ENABLE_METRICS_PUSH_CONFIG, "false");
+ return properties;
+ }
+
+ private KafkaConsumer<byte[], byte[]> createConsumer(Pipe pipe) {
+ Properties properties = consumerProperties(pipe);
+ String topicName = resolveTopicName(pipe);
KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(properties);
consumer.subscribe(List.of(topicName));
LOG.infov("Pipe {0}: subscribed Kafka consumer to topic {1} via {2}",
- pipe.getName(), topicName, bootstrapServers);
+ pipe.getName(), topicName, properties.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG));
return consumer;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.java
new file mode 100644
index 0000000000..3fcf99168a
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.java
@@ -0,0 +1,33 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import org.apache.kafka.clients.consumer.CooperativeStickyAssignor;
+import org.apache.kafka.clients.consumer.RangeAssignor;
+import org.apache.kafka.common.metrics.JmxReporter;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+
+/**
+ * Registers the kafka-clients classes that GraalVM cannot discover statically.
+ *
+ * <p>kafka-clients ships no native-image metadata and Floci uses it directly rather than through
+ * the Quarkus Kafka extension. {@code AbstractConfig} resolves {@code CLASS}-typed settings by
+ * name, so without these a native build cannot construct a consumer — which {@link PipesPoller}
+ * swallows per poll cycle, leaving the pipe RUNNING and delivering nothing.
+ *
+ * <p>Entries are exactly what {@link PipesKafkaConsumerManager#consumerProperties} resolves, and
+ * {@code PipesKafkaNativeSupportTest} reads that same method, so the two cannot drift.
+ *
+ * <p>SASL and SSL defaults are excluded: Floci is PLAINTEXT-only, and {@code DefaultJwtValidator}
+ * fails the native build by referencing jose4j, an optional dependency Floci omits.
+ */
+@RegisterForReflection(targets = {
+ // key.deserializer / value.deserializer — set explicitly by PipesKafkaConsumerManager
+ ByteArrayDeserializer.class,
+ // partition.assignment.strategy — ConsumerConfig defaults
+ RangeAssignor.class,
+ CooperativeStickyAssignor.class,
+ // metric.reporters — ConsumerConfig default
+ JmxReporter.class
+})
+public class PipesKafkaNativeSupport {
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.java
new file mode 100644
index 0000000000..d73542e0b9
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.java
@@ -0,0 +1,122 @@
+package io.github.hectorvent.floci.services.pipes;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import io.github.hectorvent.floci.services.msk.MskService;
+import io.github.hectorvent.floci.services.pipes.model.Pipe;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.function.Predicate;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Pins {@link PipesKafkaNativeSupport} to what {@code ConsumerConfig} resolves from
+ * {@link PipesKafkaConsumerManager#consumerProperties}, both ways: a newly resolved class must be
+ * registered, and nothing else may be. Reading that method rather than copying it means a new
+ * setting there is covered here too.
+ *
+ * <p>Scope: config-resolved classes only. Reflection always succeeds on the JVM, so a non-config
+ * lookup from a future kafka-clients needs a Kafka pipe run against a native binary. The
+ * compatibility suite already runs against the native image; it has no Kafka source case yet.
+ */
+class PipesKafkaNativeSupportTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private static final Predicate<String> TRANSPORT_SECURITY =
+ key -> key.startsWith("sasl.") || key.startsWith("ssl.");
+
+ @Test
+ @DisplayName("classes ConsumerConfig resolves reflectively are registered")
+ void registrationCoversReflectivelyResolvedClasses() throws Exception {
+ Set<String> missing = new TreeSet<>(expectedClassNames());
+ missing.removeAll(registeredClassNames());
+
+ assertTrue(missing.isEmpty(),
+ "kafka-clients resolves these reflectively but PipesKafkaNativeSupport omits them, "
+ + "so a native build cannot construct a consumer: " + missing);
+ }
+
+ @Test
+ @DisplayName("nothing is registered that ConsumerConfig does not resolve")
+ void registrationContainsNothingUnjustified() throws Exception {
+ Set<String> unexpected = new TreeSet<>(registeredClassNames());
+ unexpected.removeAll(expectedClassNames());
+
+ assertTrue(unexpected.isEmpty(),
+ "these are not config-resolved, so this test cannot guard them and they need a "
+ + "native run to justify. SASL/SSL defaults in particular break the build — "
+ + "DefaultJwtValidator drags in optional jose4j: " + unexpected);
+ }
+
+ /** Class-valued settings kafka-clients resolves, minus the transport security Floci never uses. */
+ private static Set<String> expectedClassNames() throws Exception {
+ Set<String> classes = new TreeSet<>();
+ consumerConfigValues().forEach((key, value) -> {
+ if (TRANSPORT_SECURITY.negate().test(key)) {
+ collectClassNames(value, classes);
+ }
+ });
+ return classes;
+ }
+
+ private static Set<String> registeredClassNames() {
+ RegisterForReflection annotation =
+ PipesKafkaNativeSupport.class.getAnnotation(RegisterForReflection.class);
+ assertNotNull(annotation, "PipesKafkaNativeSupport must carry @RegisterForReflection");
+
+ Set<String> names = new TreeSet<>(Set.of(annotation.classNames()));
+ for (Class<?> target : annotation.targets()) {
+ names.add(target.getName());
+ }
+ return names;
+ }
+
+ /**
+ * Mirrors {@code AbstractConfig.getConfiguredInstances}: a setting may be a {@code Class} or a
+ * class name — {@code metric.reporters} is the latter. Values naming no class are not candidates.
+ */
+ private static void collectClassNames(Object value, Set<String> into) {
+ switch (value) {
+ case Class<?> type -> into.add(type.getName());
+ case Collection<?> collection -> collection.forEach(element -> collectClassNames(element, into));
+ case String name -> loadable(name).ifPresent(into::add);
+ case null, default -> { }
+ }
+ }
+
+ private static Optional<String> loadable(String className) {
+ try {
+ return Optional.of(Class.forName(className, false, ConsumerConfig.class.getClassLoader()).getName());
+ } catch (ClassNotFoundException | LinkageError e) {
+ return Optional.empty();
+ }
+ }
+
+ private static Map<String, ?> consumerConfigValues() throws Exception {
+ PipesKafkaConsumerManager manager = new PipesKafkaConsumerManager(Mockito.mock(MskService.class));
+
+ Pipe pipe = new Pipe();
+ pipe.setName("native-support");
+ pipe.setSource("smk://localhost:9092");
+ pipe.setSourceParameters(MAPPER.readTree("""
+ {
+ "SelfManagedKafkaParameters": {
+ "TopicName": "orders"
+ }
+ }
+ """));
+
+ return new ConsumerConfig(manager.consumerProperties(pipe)).values();
+ }
+}