Apache Solr open-source search software
SOLR-17187: make replica poll interval configurable (#3144)
Christine Poerschke2b1fbc137670
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 51e38d4dfd7..7b969f90990 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -238,6 +238,9 @@ Improvements* SOLR-17744: Solr now enables Jetty's Graceful Shutdown features to prevent client connections from being abruptly terminated on orderly shutdown (hossman)+* SOLR-17187: The polling interval for PULL and TLOG replicas can now be overridden using the `commitPollInterval` setting, which takes a String+ formatted as "HH:mm:ss" (Torsten Koster via Christine Poerschke, Jason Gerlowski)+Optimizations---------------------* SOLR-17578: Remove ZkController internal core supplier, for slightly faster reconnection after Zookeeper session loss. (Pierre Salagnac)diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.javaindex 06bbbefedbe..7cc932365e1 100644--- a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java+++ b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java@@ -75,16 +75,9 @@ public class ReplicateFromLeader {"SolrCore not found:" + coreName + " in " + CloudUtil.getLoadedCoreNamesAsString(cc));}}- SolrConfig.UpdateHandlerInfo uinfo = core.getSolrConfig().getUpdateHandlerInfo();- String pollIntervalStr = "00:00:03";- if (System.getProperty("jetty.testMode") != null) {- pollIntervalStr = "00:00:01";- }- String calculatedPollIntervalString = determinePollInterval(uinfo);- if (calculatedPollIntervalString != null) {- pollIntervalStr = calculatedPollIntervalString;- }+ final SolrConfig.UpdateHandlerInfo uinfo = core.getSolrConfig().getUpdateHandlerInfo();+ final String pollIntervalStr = determinePollInterval(uinfo);log.info("Will start replication from leader with poll interval: {}", pollIntervalStr);NamedList<Object> followerConfig = new NamedList<>();@@ -134,18 +127,25 @@ public class ReplicateFromLeader {}/**- * Determine the poll interval for replicas based on the auto soft/hard commit schedule+ * Determine the poll interval for replicas based on the auto soft/hard commit schedule or+ * configured commit poll interval** @param uinfo the update handler info containing soft/hard commit configuration* @return a poll interval string representing a cadence of polling frequency in the form of- * hh:mm:ss+ * hh:mm:ss, never <code>null</code>*/public static String determinePollInterval(SolrConfig.UpdateHandlerInfo uinfo) {int hardCommitMaxTime = uinfo.autoCommmitMaxTime;int softCommitMaxTime = uinfo.autoSoftCommmitMaxTime;boolean hardCommitNewSearcher = uinfo.openSearcher;- String pollIntervalStr = null;- if (hardCommitMaxTime != -1) {+ String customCommitPollInterval = uinfo.commitPollInterval;+ String pollIntervalStr = "00:00:03";++ if (System.getProperty("jetty.testMode") != null) {+ pollIntervalStr = "00:00:01";+ } else if (customCommitPollInterval != null) {+ pollIntervalStr = customCommitPollInterval;+ } else if (hardCommitMaxTime != -1) {// configured hardCommit places a ceiling on the interval at which new segments will be// available to replicateif (softCommitMaxTime != -1@@ -168,6 +168,9 @@ public class ReplicateFromLeader {pollIntervalStr = toPollIntervalStr(softCommitMaxTime / 2);}+ // validate poll interval and fail early+ ReplicationHandler.readIntervalNs(pollIntervalStr);+ assert pollIntervalStr != null;return pollIntervalStr;}diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.javaindex 4034fb42e2a..4d7962a5ea5 100644--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java@@ -800,6 +800,7 @@ public class SolrConfig implements MapSerializable {public final long autoCommitMaxSizeBytes;public final boolean openSearcher; // is opening a new searcher part of hard autocommit?public final boolean commitWithinSoftCommit;+ public final String commitPollInterval;public final boolean aggregateNodeLevelMetricsEnabled;/**@@ -815,7 +816,8 @@ public class SolrConfig implements MapSerializable {boolean openSearcher,int autoSoftCommmitMaxDocs,int autoSoftCommmitMaxTime,- boolean commitWithinSoftCommit) {+ boolean commitWithinSoftCommit,+ String commitPollInterval) {this.className = className;this.autoCommmitMaxDocs = autoCommmitMaxDocs;this.autoCommmitMaxTime = autoCommmitMaxTime;@@ -826,6 +828,7 @@ public class SolrConfig implements MapSerializable {this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime;this.commitWithinSoftCommit = commitWithinSoftCommit;+ this.commitPollInterval = commitPollInterval;this.aggregateNodeLevelMetricsEnabled = false;}@@ -841,6 +844,7 @@ public class SolrConfig implements MapSerializable {this.autoSoftCommmitMaxTime = updateHandler.get("autoSoftCommit").get("maxTime").intVal(-1);this.commitWithinSoftCommit =updateHandler.get("commitWithin").get("softCommit").boolVal(true);+ this.commitPollInterval = updateHandler.get("commitPollInterval").txt();this.aggregateNodeLevelMetricsEnabled =updateHandler.boolAttr("aggregateNodeLevelMetricsEnabled", false);}@@ -857,6 +861,7 @@ public class SolrConfig implements MapSerializable {map.put("autoSoftCommit",Map.of("maxDocs", autoSoftCommmitMaxDocs, "maxTime", autoSoftCommmitMaxTime));+ map.put("commitPollInterval", commitPollInterval);return map;}}diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.javaindex c506a875294..8367e9166f9 100644--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java@@ -1490,7 +1490,7 @@ public class ReplicationHandler extends RequestHandlerBasereturn TimeUnit.MILLISECONDS.convert(readIntervalNs(interval), TimeUnit.NANOSECONDS);}- private Long readIntervalNs(String interval) {+ public static Long readIntervalNs(String interval) {if (interval == null) return null;int result = 0;Matcher m = INTERVAL_PATTERN.matcher(interval.trim());diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.javaindex 723eda5874d..aa4ae37e1c5 100644--- a/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java@@ -17,55 +17,88 @@package org.apache.solr.cloud;-import static org.junit.Assert.assertEquals;-+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.common.SolrException;import org.apache.solr.core.SolrConfig;import org.junit.Test;-public class ReplicateFromLeaderTest {+public class ReplicateFromLeaderTest extends SolrTestCaseJ4 {++ @Test+ public void determineTestPollIntervalString() {+ SolrConfig.UpdateHandlerInfo updateHandlerInfo =+ new SolrConfig.UpdateHandlerInfo(+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false, "0:0:56");+ String pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);+ assertEquals("00:00:01", pollInterval);+ }@Testpublic void determinePollIntervalString() {+ // disable jetty test mode+ System.clearProperty("jetty.testMode");+SolrConfig.UpdateHandlerInfo updateHandlerInfo =new SolrConfig.UpdateHandlerInfo(- "solr.DirectUpdateHandler2", -1, 15000, -1, true, -1, 60000, false);+ "solr.DirectUpdateHandler2", -1, 15000, -1, true, -1, 60000, false, null);String pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);assertEquals("0:0:7", pollInterval);updateHandlerInfo =new SolrConfig.UpdateHandlerInfo(- "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, 15000, false);+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, 15000, false, null);pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);assertEquals("0:0:30", pollInterval);updateHandlerInfo =new SolrConfig.UpdateHandlerInfo(- "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, 60000, false);+ "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, 60000, false, null);pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);assertEquals("0:0:30", pollInterval);updateHandlerInfo =new SolrConfig.UpdateHandlerInfo(- "solr.DirectUpdateHandler2", -1, 60000, -1, false, -1, 15000, false);+ "solr.DirectUpdateHandler2", -1, 60000, -1, false, -1, 15000, false, null);pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);assertEquals("0:0:30", pollInterval);updateHandlerInfo =new SolrConfig.UpdateHandlerInfo(- "solr.DirectUpdateHandler2", -1, -1, -1, false, -1, 60000, false);+ "solr.DirectUpdateHandler2", -1, -1, -1, false, -1, 60000, false, null);pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);assertEquals("0:0:30", pollInterval);updateHandlerInfo =new SolrConfig.UpdateHandlerInfo(- "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, -1, false);+ "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, -1, false, null);pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);assertEquals("0:0:7", pollInterval);updateHandlerInfo =new SolrConfig.UpdateHandlerInfo(- "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false);+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false, null);pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);assertEquals("0:0:30", pollInterval);++ updateHandlerInfo =+ new SolrConfig.UpdateHandlerInfo(+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false, "0:0:56");+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);+ assertEquals("0:0:56", pollInterval);++ final SolrConfig.UpdateHandlerInfo illegalUpdateHandlerInfo =+ new SolrConfig.UpdateHandlerInfo(+ "solr.DirectUpdateHandler2",+ -1,+ 60000,+ -1,+ true,+ -1,+ -1,+ false,+ "garbage-unfortunately");+ assertThrows(+ SolrException.class,+ () -> ReplicateFromLeader.determinePollInterval(illegalUpdateHandlerInfo));}}diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adocindex 812bbcbc3a8..c13790c9da3 100644--- a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc@@ -62,6 +62,10 @@ It is recommended that this be set for as long as is reasonable given the applicA hard commit means that, if a server crashes, Solr will know exactly where your data was stored; a soft commit means that the data is stored, but the location information isn't yet stored.The tradeoff is that a soft commit gives you faster visibility because it's not waiting for background merges to finish.+In a TLOG/PULL replica setup, the commit configuration also influences the interval at which the replica is polling the shard leader.+Users wishing to use a different polling interval in their TLOG/PULL replicas can do so by specifying a `commitPollInterval` value of the form "hh:mm:ss".+"01:00:00" to poll every hour, "00:15:00" to poll every fifteen minutes, etc.+=== Explicit CommitsWhen a client includes a `commit=true` parameter with an update request, this ensures that all index segments affected by the adds and deletes on an update are written to disk as soon as index updates are completed.