Apache Solr open-source search software
SOLR-18412: CrossDC - ZooKeeper /crossdc.properties no longer overrides solr.crossdc.kafka.* sysprops or env vars (#4884)
Andrzej Białecki62aace94a13f
diff --git a/changelog/unreleased/SOLR-18412.yml b/changelog/unreleased/SOLR-18412.ymlnew file mode 100644index 00000000000..6fdc63a806f--- /dev/null+++ b/changelog/unreleased/SOLR-18412.yml@@ -0,0 +1,8 @@+title: CrossDC - ZooKeeper /crossdc.properties no longer overrides solr.crossdc.kafka.* sysprops or env vars+type: fixed+authors:+ - name: Andrzej Bialecki+ nick: ab+links:+ - name: SOLR-18412+ url: https://issues.apache.org/jira/browse/SOLR-18412diff --git a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.javaindex 88db372683e..64dfc2cae7a 100644--- a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java+++ b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java@@ -295,8 +295,17 @@ public class KafkaCrossDcConf extends CrossDcConf {}zkPropsUnprocessed.forEach((key, val) -> {- if (properties.get(key) == null) {- properties.put((String) key, val);+ String strKey = (String) key;+ String targetKey = strKey;+ if (strKey.startsWith(ConfUtil.KAFKA_ENV_PREFIX)) {+ targetKey = ConfUtil.normalizeKafkaEnvKey(strKey);+ } else if (strKey.startsWith(ConfUtil.KAFKA_PROP_PREFIX)) {+ targetKey = ConfUtil.normalizeKafkaSysPropKey(strKey);+ }+ // A JVM system property or env var pass-through override always takes precedence+ // over the same property coming from ZooKeeper's /crossdc.properties.+ if (properties.get(targetKey) == null) {+ properties.put(targetKey, val);}});}diff --git a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.javaindex 6afcc35ae2a..05591e8f8d5 100644--- a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java+++ b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java@@ -426,6 +426,87 @@ public class ConfUtilTest extends SolrTestCaseJ4 {assertEquals("lz4", conf.getAdditionalProperties().get("compression.type"));}+ @Test+ public void testFillProperties_ZkOnlyKafkaPrefixPropIsNormalized() throws Exception {+ Map<String, Object> properties = new HashMap<>();++ System.setProperty(KafkaCrossDcConf.BOOTSTRAP_SERVERS, "sys-kafka:9092");+ System.setProperty(KafkaCrossDcConf.TOPIC_NAME, "sys-topic");++ Properties zkProps = new Properties();+ zkProps.setProperty("solr.crossdc.kafka.compression.type", "zk-value");++ ByteArrayOutputStream baos = new ByteArrayOutputStream();+ OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8);+ zkProps.store(writer, null);+ writer.close();+ byte[] zkData = baos.toByteArray();++ when(mockZkClient.exists(anyString())).thenReturn(true);+ when(mockZkClient.getData(anyString(), isNull(), isNull())).thenReturn(zkData);++ ConfUtil.fillProperties(mockZkClient, properties);++ // ZK value lands under the normalized key, not the raw solr.crossdc.kafka.* key.+ assertEquals("zk-value", properties.get("compression.type"));+ assertNull(properties.get("solr.crossdc.kafka.compression.type"));+ }++ @Test+ public void testFillProperties_KafkaPrefixSysPropPrecedenceOverZk() throws Exception {+ Map<String, Object> properties = new HashMap<>();++ System.setProperty(KafkaCrossDcConf.BOOTSTRAP_SERVERS, "sys-kafka:9092");+ System.setProperty(KafkaCrossDcConf.TOPIC_NAME, "sys-topic");+ System.setProperty("solr.crossdc.kafka.compression.type", "sys-value");++ Properties zkProps = new Properties();+ zkProps.setProperty("solr.crossdc.kafka.compression.type", "zk-value");++ ByteArrayOutputStream baos = new ByteArrayOutputStream();+ OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8);+ zkProps.store(writer, null);+ writer.close();+ byte[] zkData = baos.toByteArray();++ when(mockZkClient.exists(anyString())).thenReturn(true);+ when(mockZkClient.getData(anyString(), isNull(), isNull())).thenReturn(zkData);++ ConfUtil.fillProperties(mockZkClient, properties);++ assertEquals("sys-value", properties.get("compression.type"));+ }++ @Test+ public void testFillProperties_SysPropPrecedenceOverDoubleZk() throws Exception {+ Map<String, Object> properties = new HashMap<>();++ System.setProperty(KafkaCrossDcConf.BOOTSTRAP_SERVERS, "sys-kafka:9092");+ System.setProperty(KafkaCrossDcConf.TOPIC_NAME, "sys-topic");+ System.setProperty("solr.crossdc.kafka.compression.type", "sys-value");++ // First call, as Consumer.start() does, with no ZK client yet.+ ConfUtil.fillProperties(null, properties);+ assertEquals("sys-value", properties.get("compression.type"));++ Properties zkProps = new Properties();+ zkProps.setProperty("solr.crossdc.kafka.compression.type", "zk-value");++ ByteArrayOutputStream baos = new ByteArrayOutputStream();+ OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8);+ zkProps.store(writer, null);+ writer.close();+ byte[] zkData = baos.toByteArray();++ when(mockZkClient.exists(anyString())).thenReturn(true);+ when(mockZkClient.getData(anyString(), isNull(), isNull())).thenReturn(zkData);++ // Second call, now with the real ZK client, as Consumer.start() does.+ ConfUtil.fillProperties(mockZkClient, properties);++ assertEquals("sys-value", properties.get("compression.type"));+ }+// we can't easily modify envvars, test just the key conversion in properties@Testpublic void testUnderscoreToDotsConversion() {