Apache Solr open-source search software
SOLR-16487: Improve Pull Replica Interval (#1437)
Justin Sweeneyc6706dfdbaa4
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex d1c54405070..8d0bfaeb853 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -183,6 +183,8 @@ Optimizations* SOLR-16615: Jersey 'ApplicationHandlers' are now shared by compatible cores where possible (Jason Gerlowski, Houston Putman)+* SOLR-16487: Replication pooling is optimized based on hard and soft commit settings for a given collection (Justin Sweeney)+Bug Fixes---------------------diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.javaindex 2755102ad21..7abe46fe0c3 100644--- a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java+++ b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java@@ -80,10 +80,10 @@ public class ReplicateFromLeader {if (System.getProperty("jetty.testMode") != null) {pollIntervalStr = "00:00:01";}- if (uinfo.autoCommmitMaxTime != -1) {- pollIntervalStr = toPollIntervalStr(uinfo.autoCommmitMaxTime / 2);- } else if (uinfo.autoSoftCommmitMaxTime != -1) {- pollIntervalStr = toPollIntervalStr(uinfo.autoSoftCommmitMaxTime / 2);++ String calculatedPollIntervalString = determinePollInterval(uinfo);+ if (calculatedPollIntervalString != null) {+ pollIntervalStr = calculatedPollIntervalString;}log.info("Will start replication from leader with poll interval: {}", pollIntervalStr);@@ -155,6 +155,44 @@ public class ReplicateFromLeader {}}+ /**+ * Determine the poll interval for replicas based on the auto soft/hard commit schedule+ *+ * @param uinfo the update handler info containing soft/hard commit configuration+ * @return a poll interval string representing a cadence of polling frequency in the form of+ * hh:mm:ss+ */+ public static String determinePollInterval(SolrConfig.UpdateHandlerInfo uinfo) {+ int hardCommitMaxTime = uinfo.autoCommmitMaxTime;+ int softCommitMaxTime = uinfo.autoSoftCommmitMaxTime;+ boolean hardCommitNewSearcher = uinfo.openSearcher;+ String pollIntervalStr = null;+ if (hardCommitMaxTime != -1) {+ // configured hardCommit places a ceiling on the interval at which new segments will be+ // available to replicate+ if (softCommitMaxTime != -1+ && (!hardCommitNewSearcher || softCommitMaxTime <= hardCommitMaxTime)) {+ /*+ * softCommit is configured.+ * Usually if softCommit is configured, `hardCommitNewSearcher==false`,+ * in which case you want to calculate poll interval wrt the max of hardCommitTime+ * (when segments are available to replicate) and softCommitTime (when changes are visible).+ * But in the unusual case that hardCommit _does_ open a new searcher and+ * `hardCommitMaxTime < softCommitMaxTime`, then fallback to `else` clause,+ * setting poll interval wrt `hardCommitMaxTime` alone.+ */+ pollIntervalStr = toPollIntervalStr(Math.max(hardCommitMaxTime, softCommitMaxTime) / 2);+ } else {+ pollIntervalStr = toPollIntervalStr(hardCommitMaxTime / 2);+ }+ } else if (softCommitMaxTime != -1) {+ // visibility of changes places a ceiling on polling frequency+ pollIntervalStr = toPollIntervalStr(softCommitMaxTime / 2);+ }++ return pollIntervalStr;+ }+private static String toPollIntervalStr(int ms) {int sec = ms / 1000;int hour = sec / 3600;diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.javanew file mode 100644index 00000000000..723eda5874d--- /dev/null+++ b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java@@ -0,0 +1,71 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import static org.junit.Assert.assertEquals;++import org.apache.solr.core.SolrConfig;+import org.junit.Test;++public class ReplicateFromLeaderTest {++ @Test+ public void determinePollIntervalString() {+ SolrConfig.UpdateHandlerInfo updateHandlerInfo =+ new SolrConfig.UpdateHandlerInfo(+ "solr.DirectUpdateHandler2", -1, 15000, -1, true, -1, 60000, false);+ String pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);+ assertEquals("0:0:7", pollInterval);++ updateHandlerInfo =+ new SolrConfig.UpdateHandlerInfo(+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, 15000, false);+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);+ assertEquals("0:0:30", pollInterval);++ updateHandlerInfo =+ new SolrConfig.UpdateHandlerInfo(+ "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, 60000, false);+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);+ assertEquals("0:0:30", pollInterval);++ updateHandlerInfo =+ new SolrConfig.UpdateHandlerInfo(+ "solr.DirectUpdateHandler2", -1, 60000, -1, false, -1, 15000, false);+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);+ assertEquals("0:0:30", pollInterval);++ updateHandlerInfo =+ new SolrConfig.UpdateHandlerInfo(+ "solr.DirectUpdateHandler2", -1, -1, -1, false, -1, 60000, false);+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);+ assertEquals("0:0:30", pollInterval);++ updateHandlerInfo =+ new SolrConfig.UpdateHandlerInfo(+ "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, -1, false);+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);+ assertEquals("0:0:7", pollInterval);++ updateHandlerInfo =+ new SolrConfig.UpdateHandlerInfo(+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false);+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);+ assertEquals("0:0:30", pollInterval);+ }+}