Light, fluffy, and always free - The AWS Local Emulator alternative
fix(pipes): register kafka-clients reflection metadata for native image (#2068)
Liam MacPhersonab733798a38d
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.javaindex a9c80cdddd..6962e7e12d 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaConsumerManager.java@@ -131,13 +131,12 @@ public class PipesKafkaConsumerManager {return resolveConsumerGroupId(pipe, kafkaParameters(pipe));}- private KafkaConsumer<byte[], byte[]> createConsumer(Pipe pipe) {- String bootstrapServers = resolveBootstrapServers(pipe);- String topicName = resolveTopicName(pipe);+ /** Package-private so {@code PipesKafkaNativeSupportTest} guards the real settings, not a copy. */+ Properties consumerProperties(Pipe pipe) {JsonNode params = kafkaParameters(pipe);Properties properties = new Properties();- properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);+ properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, resolveBootstrapServers(pipe));properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");@@ -145,11 +144,21 @@ public class PipesKafkaConsumerManager {properties.put(ConsumerConfig.GROUP_ID_CONFIG, resolveConsumerGroupId(pipe, params));properties.put(ConsumerConfig.CLIENT_ID_CONFIG, "floci-pipes-" + UUID.randomUUID());properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, Integer.toString(resolveBatchSize(pipe, 100)));+ // On by default (KIP-714). Floci has no reason to push metrics to a user's broker, and it+ // makes the shaded protobuf serializer reflectively reachable on a path no broker we test+ // against exercises.+ properties.put(ConsumerConfig.ENABLE_METRICS_PUSH_CONFIG, "false");+ return properties;+ }++ private KafkaConsumer<byte[], byte[]> createConsumer(Pipe pipe) {+ Properties properties = consumerProperties(pipe);+ String topicName = resolveTopicName(pipe);KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(properties);consumer.subscribe(List.of(topicName));LOG.infov("Pipe {0}: subscribed Kafka consumer to topic {1} via {2}",- pipe.getName(), topicName, bootstrapServers);+ pipe.getName(), topicName, properties.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG));return consumer;}diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.javanew file mode 100644index 0000000000..3fcf99168a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupport.java@@ -0,0 +1,33 @@+package io.github.hectorvent.floci.services.pipes;++import io.quarkus.runtime.annotations.RegisterForReflection;+import org.apache.kafka.clients.consumer.CooperativeStickyAssignor;+import org.apache.kafka.clients.consumer.RangeAssignor;+import org.apache.kafka.common.metrics.JmxReporter;+import org.apache.kafka.common.serialization.ByteArrayDeserializer;++/**+ * Registers the kafka-clients classes that GraalVM cannot discover statically.+ *+ * <p>kafka-clients ships no native-image metadata and Floci uses it directly rather than through+ * the Quarkus Kafka extension. {@code AbstractConfig} resolves {@code CLASS}-typed settings by+ * name, so without these a native build cannot construct a consumer — which {@link PipesPoller}+ * swallows per poll cycle, leaving the pipe RUNNING and delivering nothing.+ *+ * <p>Entries are exactly what {@link PipesKafkaConsumerManager#consumerProperties} resolves, and+ * {@code PipesKafkaNativeSupportTest} reads that same method, so the two cannot drift.+ *+ * <p>SASL and SSL defaults are excluded: Floci is PLAINTEXT-only, and {@code DefaultJwtValidator}+ * fails the native build by referencing jose4j, an optional dependency Floci omits.+ */+@RegisterForReflection(targets = {+ // key.deserializer / value.deserializer — set explicitly by PipesKafkaConsumerManager+ ByteArrayDeserializer.class,+ // partition.assignment.strategy — ConsumerConfig defaults+ RangeAssignor.class,+ CooperativeStickyAssignor.class,+ // metric.reporters — ConsumerConfig default+ JmxReporter.class+})+public class PipesKafkaNativeSupport {+}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.javanew file mode 100644index 0000000000..d73542e0b9--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesKafkaNativeSupportTest.java@@ -0,0 +1,122 @@+package io.github.hectorvent.floci.services.pipes;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.quarkus.runtime.annotations.RegisterForReflection;+import io.github.hectorvent.floci.services.msk.MskService;+import io.github.hectorvent.floci.services.pipes.model.Pipe;+import org.apache.kafka.clients.consumer.ConsumerConfig;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import org.mockito.Mockito;++import java.util.Collection;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.TreeSet;+import java.util.function.Predicate;++import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Pins {@link PipesKafkaNativeSupport} to what {@code ConsumerConfig} resolves from+ * {@link PipesKafkaConsumerManager#consumerProperties}, both ways: a newly resolved class must be+ * registered, and nothing else may be. Reading that method rather than copying it means a new+ * setting there is covered here too.+ *+ * <p>Scope: config-resolved classes only. Reflection always succeeds on the JVM, so a non-config+ * lookup from a future kafka-clients needs a Kafka pipe run against a native binary. The+ * compatibility suite already runs against the native image; it has no Kafka source case yet.+ */+class PipesKafkaNativeSupportTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();++ private static final Predicate<String> TRANSPORT_SECURITY =+ key -> key.startsWith("sasl.") || key.startsWith("ssl.");++ @Test+ @DisplayName("classes ConsumerConfig resolves reflectively are registered")+ void registrationCoversReflectivelyResolvedClasses() throws Exception {+ Set<String> missing = new TreeSet<>(expectedClassNames());+ missing.removeAll(registeredClassNames());++ assertTrue(missing.isEmpty(),+ "kafka-clients resolves these reflectively but PipesKafkaNativeSupport omits them, "+ + "so a native build cannot construct a consumer: " + missing);+ }++ @Test+ @DisplayName("nothing is registered that ConsumerConfig does not resolve")+ void registrationContainsNothingUnjustified() throws Exception {+ Set<String> unexpected = new TreeSet<>(registeredClassNames());+ unexpected.removeAll(expectedClassNames());++ assertTrue(unexpected.isEmpty(),+ "these are not config-resolved, so this test cannot guard them and they need a "+ + "native run to justify. SASL/SSL defaults in particular break the build — "+ + "DefaultJwtValidator drags in optional jose4j: " + unexpected);+ }++ /** Class-valued settings kafka-clients resolves, minus the transport security Floci never uses. */+ private static Set<String> expectedClassNames() throws Exception {+ Set<String> classes = new TreeSet<>();+ consumerConfigValues().forEach((key, value) -> {+ if (TRANSPORT_SECURITY.negate().test(key)) {+ collectClassNames(value, classes);+ }+ });+ return classes;+ }++ private static Set<String> registeredClassNames() {+ RegisterForReflection annotation =+ PipesKafkaNativeSupport.class.getAnnotation(RegisterForReflection.class);+ assertNotNull(annotation, "PipesKafkaNativeSupport must carry @RegisterForReflection");++ Set<String> names = new TreeSet<>(Set.of(annotation.classNames()));+ for (Class<?> target : annotation.targets()) {+ names.add(target.getName());+ }+ return names;+ }++ /**+ * Mirrors {@code AbstractConfig.getConfiguredInstances}: a setting may be a {@code Class} or a+ * class name — {@code metric.reporters} is the latter. Values naming no class are not candidates.+ */+ private static void collectClassNames(Object value, Set<String> into) {+ switch (value) {+ case Class<?> type -> into.add(type.getName());+ case Collection<?> collection -> collection.forEach(element -> collectClassNames(element, into));+ case String name -> loadable(name).ifPresent(into::add);+ case null, default -> { }+ }+ }++ private static Optional<String> loadable(String className) {+ try {+ return Optional.of(Class.forName(className, false, ConsumerConfig.class.getClassLoader()).getName());+ } catch (ClassNotFoundException | LinkageError e) {+ return Optional.empty();+ }+ }++ private static Map<String, ?> consumerConfigValues() throws Exception {+ PipesKafkaConsumerManager manager = new PipesKafkaConsumerManager(Mockito.mock(MskService.class));++ Pipe pipe = new Pipe();+ pipe.setName("native-support");+ pipe.setSource("smk://localhost:9092");+ pipe.setSourceParameters(MAPPER.readTree("""+ {+ "SelfManagedKafkaParameters": {+ "TopicName": "orders"+ }+ }+ """));++ return new ConsumerConfig(manager.consumerProperties(pipe)).values();+ }+}