Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-18412: CrossDC - ZooKeeper /crossdc.properties no longer overrides solr.crossdc.kafka.* sysprops or env vars (#4884)

Andrzej Białecki62aace94a13f
diff --git a/changelog/unreleased/SOLR-18412.yml b/changelog/unreleased/SOLR-18412.yml
new file mode 100644
index 00000000000..6fdc63a806f
--- /dev/null
+++ b/changelog/unreleased/SOLR-18412.yml
@@ -0,0 +1,8 @@
+title: CrossDC - ZooKeeper /crossdc.properties no longer overrides solr.crossdc.kafka.* sysprops or env vars
+type: fixed
+authors:
+ - name: Andrzej Bialecki
+ nick: ab
+links:
+ - name: SOLR-18412
+ url: https://issues.apache.org/jira/browse/SOLR-18412
diff --git a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java
index 88db372683e..64dfc2cae7a 100644
--- a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java
+++ b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java
@@ -295,8 +295,17 @@ public class KafkaCrossDcConf extends CrossDcConf {
}
zkPropsUnprocessed.forEach(
(key, val) -> {
- if (properties.get(key) == null) {
- properties.put((String) key, val);
+ String strKey = (String) key;
+ String targetKey = strKey;
+ if (strKey.startsWith(ConfUtil.KAFKA_ENV_PREFIX)) {
+ targetKey = ConfUtil.normalizeKafkaEnvKey(strKey);
+ } else if (strKey.startsWith(ConfUtil.KAFKA_PROP_PREFIX)) {
+ targetKey = ConfUtil.normalizeKafkaSysPropKey(strKey);
+ }
+ // A JVM system property or env var pass-through override always takes precedence
+ // over the same property coming from ZooKeeper's /crossdc.properties.
+ if (properties.get(targetKey) == null) {
+ properties.put(targetKey, val);
}
});
}
diff --git a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java
index 6afcc35ae2a..05591e8f8d5 100644
--- a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java
+++ b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java
@@ -426,6 +426,87 @@ public class ConfUtilTest extends SolrTestCaseJ4 {
assertEquals("lz4", conf.getAdditionalProperties().get("compression.type"));
}
+ @Test
+ public void testFillProperties_ZkOnlyKafkaPrefixPropIsNormalized() throws Exception {
+ Map<String, Object> properties = new HashMap<>();
+
+ System.setProperty(KafkaCrossDcConf.BOOTSTRAP_SERVERS, "sys-kafka:9092");
+ System.setProperty(KafkaCrossDcConf.TOPIC_NAME, "sys-topic");
+
+ Properties zkProps = new Properties();
+ zkProps.setProperty("solr.crossdc.kafka.compression.type", "zk-value");
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8);
+ zkProps.store(writer, null);
+ writer.close();
+ byte[] zkData = baos.toByteArray();
+
+ when(mockZkClient.exists(anyString())).thenReturn(true);
+ when(mockZkClient.getData(anyString(), isNull(), isNull())).thenReturn(zkData);
+
+ ConfUtil.fillProperties(mockZkClient, properties);
+
+ // ZK value lands under the normalized key, not the raw solr.crossdc.kafka.* key.
+ assertEquals("zk-value", properties.get("compression.type"));
+ assertNull(properties.get("solr.crossdc.kafka.compression.type"));
+ }
+
+ @Test
+ public void testFillProperties_KafkaPrefixSysPropPrecedenceOverZk() throws Exception {
+ Map<String, Object> properties = new HashMap<>();
+
+ System.setProperty(KafkaCrossDcConf.BOOTSTRAP_SERVERS, "sys-kafka:9092");
+ System.setProperty(KafkaCrossDcConf.TOPIC_NAME, "sys-topic");
+ System.setProperty("solr.crossdc.kafka.compression.type", "sys-value");
+
+ Properties zkProps = new Properties();
+ zkProps.setProperty("solr.crossdc.kafka.compression.type", "zk-value");
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8);
+ zkProps.store(writer, null);
+ writer.close();
+ byte[] zkData = baos.toByteArray();
+
+ when(mockZkClient.exists(anyString())).thenReturn(true);
+ when(mockZkClient.getData(anyString(), isNull(), isNull())).thenReturn(zkData);
+
+ ConfUtil.fillProperties(mockZkClient, properties);
+
+ assertEquals("sys-value", properties.get("compression.type"));
+ }
+
+ @Test
+ public void testFillProperties_SysPropPrecedenceOverDoubleZk() throws Exception {
+ Map<String, Object> properties = new HashMap<>();
+
+ System.setProperty(KafkaCrossDcConf.BOOTSTRAP_SERVERS, "sys-kafka:9092");
+ System.setProperty(KafkaCrossDcConf.TOPIC_NAME, "sys-topic");
+ System.setProperty("solr.crossdc.kafka.compression.type", "sys-value");
+
+ // First call, as Consumer.start() does, with no ZK client yet.
+ ConfUtil.fillProperties(null, properties);
+ assertEquals("sys-value", properties.get("compression.type"));
+
+ Properties zkProps = new Properties();
+ zkProps.setProperty("solr.crossdc.kafka.compression.type", "zk-value");
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8);
+ zkProps.store(writer, null);
+ writer.close();
+ byte[] zkData = baos.toByteArray();
+
+ when(mockZkClient.exists(anyString())).thenReturn(true);
+ when(mockZkClient.getData(anyString(), isNull(), isNull())).thenReturn(zkData);
+
+ // Second call, now with the real ZK client, as Consumer.start() does.
+ ConfUtil.fillProperties(mockZkClient, properties);
+
+ assertEquals("sys-value", properties.get("compression.type"));
+ }
+
// we can't easily modify envvars, test just the key conversion in properties
@Test
public void testUnderscoreToDotsConversion() {