Apache Solr open-source search software
SOLR-15286: A brand new follower in the legacy mode should wait to replicate index before reporting healthy (#238)
Pushkar Raste3cd71244d237
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.javaindex e39fd9cd286..2b210adad3c 100644--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java@@ -300,7 +300,7 @@ public class IndexFetcher {/*** Gets the latest commit version and generation from the leader*/- NamedList<Object> getLatestVersion() throws IOException {+ public NamedList<Object> getLatestVersion() throws IOException {ModifiableSolrParams params = new ModifiableSolrParams();params.set(COMMAND, CMD_INDEX_VERSION);params.set(CommonParams.WT, JAVABIN);diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.javaindex f36af311293..71fe0f4e849 100644--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java@@ -233,6 +233,10 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAwthis.pollListener = pollListener;}+ public boolean isFollower() {+ return this.isFollower;+ }+@Overridepublic void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {rsp.setHttpCaching(false);@@ -327,7 +331,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw}@SuppressWarnings("unchecked")- static <T> T getObjectWithBackwardCompatibility(NamedList<?> params, String preferredKey, String alternativeKey) {+ public static <T> T getObjectWithBackwardCompatibility(NamedList<?> params, String preferredKey, String alternativeKey) {Object value = params.get(preferredKey);if (value != null) {return (T) value;diff --git a/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.javaindex 8fcbef310fc..147ad8a2604 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java@@ -18,11 +18,15 @@package org.apache.solr.handler.admin;import java.lang.invoke.MethodHandles;+import java.util.ArrayList;import java.util.Arrays;import java.util.Collection;import java.util.List;+import java.util.Locale;import java.util.stream.Collectors;+import org.apache.lucene.index.IndexCommit;+import org.apache.solr.client.solrj.request.HealthCheckRequest;import org.apache.solr.cloud.CloudDescriptor;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.ClusterState;@@ -30,6 +34,9 @@ import org.apache.solr.common.cloud.Replica.State;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.SolrCore;+import org.apache.solr.handler.IndexFetcher;+import org.apache.solr.handler.ReplicationHandler;import org.apache.solr.handler.RequestHandlerBase;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;@@ -39,12 +46,13 @@ import org.slf4j.LoggerFactory;import static org.apache.solr.common.params.CommonParams.FAILURE;import static org.apache.solr.common.params.CommonParams.OK;import static org.apache.solr.common.params.CommonParams.STATUS;+import static org.apache.solr.handler.ReplicationHandler.GENERATION;/*** Health Check Handler for reporting the health of a specific node.** <p>- * By default the handler returns status <code>200 OK</code> if all checks succeed, else it returns+ * For the Solr Cloud mode by default the handler returns status <code>200 OK</code> if all checks succeed, else it returns* status <code>503 UNAVAILABLE</code>:* </p>* <ol>@@ -52,7 +60,6 @@ import static org.apache.solr.common.params.CommonParams.STATUS;* <li>Node connected to zookeeper.</li>* <li>Node listed in <code>live_nodes</code> in zookeeper.</li>* </ol>- ** <p>* The handler takes an optional request parameter <code>requireHealthyCores=true</code>* which will also require that all local cores that are part of an <b>active shard</b>@@ -61,6 +68,17 @@ import static org.apache.solr.common.params.CommonParams.STATUS;* is fully initialized and stable before proceeding with restarting the next node, and thus* reduce the risk of restarting the last live replica of a shard.* </p>+ *+ * <p>+ * For the legacy mode the handler returns status <code>200 OK</code> if all the cores configured as follower have+ * successfully replicated index from their respective leader after startup. Note that this is a weak check+ * i.e. once a follower has caught up with the leader the health check will keep reporting <code>200 OK</code>+ * even if the follower starts lagging behind. You should specify the acceptable generation lag follower should+ * be with respect to its leader using the <code>maxGenerationLag=<max_generation_lag></code> request+ * parameter. If <code>maxGenerationLag</code> is not provided then health check would simply return OK.+ * </p>+ *+ **/public class HealthCheckHandler extends RequestHandlerBase {@@ -83,34 +101,38 @@ public class HealthCheckHandler extends RequestHandlerBase {@Overridepublic void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {-- CoreContainer cores = getCoreContainer();rsp.setHttpCaching(false);// Core container should not be null and active (redundant check)- if(cores == null || cores.isShutDown()) {+ if (coreContainer == null || coreContainer.isShutDown()) {rsp.setException(new SolrException(SolrException.ErrorCode.SERVER_ERROR, "CoreContainer is either not initialized or shutting down"));return;}- if(!cores.isZooKeeperAware()) {- //TODO: Support standalone instances- rsp.setException(new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Health check is only available when running in SolrCloud mode"));- return;- }- if (log.isDebugEnabled()) {- log.debug("Invoked HealthCheckHandler on [{}]", coreContainer.getZkController().getNodeName());+ if (!coreContainer.isZooKeeperAware()) {+ if (log.isDebugEnabled()) {+ log.debug("Invoked HealthCheckHandler in legacy mode.");+ }+ healthCheckLegacyMode(req, rsp);+ } else {+ if (log.isDebugEnabled()) {+ log.debug("Invoked HealthCheckHandler in cloud mode on [{}]", this.coreContainer.getZkController().getNodeName());+ }+ healthCheckCloudMode(req, rsp);}- ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();+ }++ private void healthCheckCloudMode(SolrQueryRequest req, SolrQueryResponse rsp) {+ ZkStateReader zkStateReader = coreContainer.getZkController().getZkStateReader();ClusterState clusterState = zkStateReader.getClusterState();// Check for isConnected and isClosed- if(zkStateReader.getZkClient().isClosed() || !zkStateReader.getZkClient().isConnected()) {+ if (zkStateReader.getZkClient().isClosed() || !zkStateReader.getZkClient().isConnected()) {rsp.add(STATUS, FAILURE);rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Host Unavailable: Not connected to zk"));return;}// Fail if not in live_nodes- if (!clusterState.getLiveNodes().contains(cores.getZkController().getNodeName())) {+ if (!clusterState.getLiveNodes().contains(coreContainer.getZkController().getNodeName())) {rsp.add(STATUS, FAILURE);rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Host Unavailable: Not in live nodes as per zk"));return;@@ -118,15 +140,15 @@ public class HealthCheckHandler extends RequestHandlerBase {// Optionally require that all cores on this node are active if param 'requireHealthyCores=true'if (req.getParams().getBool(PARAM_REQUIRE_HEALTHY_CORES, false)) {- Collection<CloudDescriptor> coreDescriptors = cores.getCores().stream()- .map(c -> c.getCoreDescriptor().getCloudDescriptor()).collect(Collectors.toList());+ Collection<CloudDescriptor> coreDescriptors = coreContainer.getCores().stream()+ .map(c -> c.getCoreDescriptor().getCloudDescriptor()).collect(Collectors.toList());long unhealthyCores = findUnhealthyCores(coreDescriptors, clusterState);if (unhealthyCores > 0) {- rsp.add(STATUS, FAILURE);- rsp.add("num_cores_unhealthy", unhealthyCores);- rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, unhealthyCores + " out of "- + cores.getNumAllCores() + " replicas are currently initializing or recovering"));- return;+ rsp.add(STATUS, FAILURE);+ rsp.add("num_cores_unhealthy", unhealthyCores);+ rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, unhealthyCores + " out of "+ + coreContainer.getNumAllCores() + " replicas are currently initializing or recovering"));+ return;}rsp.add("message", "All cores are healthy");}@@ -135,6 +157,94 @@ public class HealthCheckHandler extends RequestHandlerBase {rsp.add(STATUS, OK);}+ private void healthCheckLegacyMode(SolrQueryRequest req, SolrQueryResponse rsp) {+ Integer maxGenerationLag = req.getParams().getInt(HealthCheckRequest.PARAM_MAX_GENERATION_LAG);+ List<String> laggingCoresInfo = new ArrayList<>();+ boolean allCoresAreInSync = true;++ // check only if max generation lag is specified+ if (maxGenerationLag != null) {+ // if is not negative+ if (maxGenerationLag < 0) {+ log.error("Invalid value for maxGenerationLag:[{}]", maxGenerationLag);+ rsp.add("message",+ String.format(Locale.ROOT, "Invalid value of maxGenerationLag:%s",+ maxGenerationLag));+ rsp.add(STATUS, FAILURE);+ } else {+ for (SolrCore core : coreContainer.getCores()) {+ ReplicationHandler replicationHandler =+ (ReplicationHandler) core.getRequestHandler(ReplicationHandler.PATH);+ if (replicationHandler.isFollower()) {+ boolean isCoreInSync =+ isWithinGenerationLag(core, replicationHandler, maxGenerationLag, laggingCoresInfo);++ allCoresAreInSync &= isCoreInSync;+ }+ }+ }+ if (allCoresAreInSync) {+ rsp.add("message",+ String.format(Locale.ROOT, "All the followers are in sync with leader (within maxGenerationLag: %d) " ++ "or the cores are acting as leader", maxGenerationLag));+ rsp.add(STATUS, OK);+ } else {+ rsp.add("message",+ String.format(Locale.ROOT, "Cores violating maxGenerationLag:%d.%n%s", maxGenerationLag,+ String.join(",\n", laggingCoresInfo)));+ rsp.add(STATUS, FAILURE);+ }+ } else { // if maxGeneration lag is not specified (is null) we aren't checking for lag+ rsp.add("message", "maxGenerationLag isn't specified. Followers aren't " ++ "checking for the generation lag from the leaders");+ rsp.add(STATUS, OK);+ }+ }++ private boolean isWithinGenerationLag(final SolrCore core, ReplicationHandler replicationHandler,+ int maxGenerationLag, List<String> laggingCoresInfo) {+ IndexFetcher indexFetcher = null;+ try {+ // may not be the best way to get leader's replicableCommit+ NamedList<?> follower =+ ReplicationHandler.getObjectWithBackwardCompatibility(replicationHandler.getInitArgs(), "follower",+ "slave");+ indexFetcher = new IndexFetcher(follower, replicationHandler, core);++ NamedList<?> replicableCommitOnLeader = indexFetcher.getLatestVersion();+ long leaderGeneration = (Long) replicableCommitOnLeader.get(GENERATION);++ // Get our own commit and generation from the commit+ IndexCommit commit = core.getDeletionPolicy().getLatestCommit();+ if (commit != null) {+ long followerGeneration = commit.getGeneration();+ long generationDiff = leaderGeneration - followerGeneration;++ // generationDiff shouldn't be negative except for some edge cases, log it. Some scenarios are+ // 1) commit generation rolls over Long.MAX_VALUE (really unlikely)+ // 2) Leader's index is wiped clean and the follower is still showing commit generation+ // from the the old index+ if (generationDiff < 0) {+ log.warn("core:[{}], generation lag:[{}] is negative.");+ } else if (generationDiff < maxGenerationLag) {+ log.info("core:[{}] generation lag is above acceptable threshold:[{}], " ++ "generation lag:[{}], leader generation:[{}], follower generation:[{}]",+ core, maxGenerationLag, generationDiff, leaderGeneration, followerGeneration);++ laggingCoresInfo.add(String.format(Locale.ROOT, "Core %s is lagging by %d generations", core.getName(), generationDiff));+ return true;+ }+ }+ } catch (Exception e) {+ log.error("Failed to check if the follower is in sync with the leader", e);+ } finally {+ if (indexFetcher != null) {+ indexFetcher.destroy();+ }+ }+ return false;+ }+/*** Find replicas DOWN or RECOVERING, or replicas in clusterstate that do not exist on local node.* We first find local cores which are either not registered or unhealthy, and check each of these againstdiff --git a/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java b/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.javanew file mode 100644index 00000000000..2ba611bbdc0--- /dev/null+++ b/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java@@ -0,0 +1,333 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.handler;++import org.apache.lucene.util.LuceneTestCase;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.embedded.JettyConfig;+import org.apache.solr.client.solrj.embedded.JettySolrRunner;+import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.common.SolrDocumentList;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.util.FileUtils;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import java.io.*;+import java.lang.invoke.MethodHandles;+import java.net.URL;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Properties;++import static org.junit.Assert.assertEquals;+import static org.junit.Assert.assertNotNull;++public final class ReplicationTestHelper {++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ public static final String CONF_DIR = "solr"+ + File.separator + "collection1" + File.separator + "conf"+ + File.separator;+++ public static JettySolrRunner createAndStartJetty(SolrInstance instance) throws Exception {+ FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(instance.getHomeDir(), "solr.xml"));+ Properties nodeProperties = new Properties();+ nodeProperties.setProperty("solr.data.dir", instance.getDataDir());+ JettyConfig jettyConfig = JettyConfig.builder().setContext("/solr").setPort(0).build();+ JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, jettyConfig);+ jetty.start();+ return jetty;+ }++ public static HttpSolrClient createNewSolrClient(String baseUrl) {+ try {+ // setup the client...+ HttpSolrClient client = SolrTestCaseJ4.getHttpSolrClient(baseUrl, 15000, 90000);+ return client;+ }+ catch (Exception ex) {+ throw new RuntimeException(ex);+ }+ }++ public static int index(SolrClient s, Object... fields) throws Exception {+ SolrInputDocument doc = new SolrInputDocument();+ for (int i = 0; i < fields.length; i += 2) {+ doc.addField((String) (fields[i]), fields[i + 1]);+ }+ return s.add(doc).getStatus();+ }++ /**+ * character copy of file using UTF-8. If port is non-null, will be substituted any time "TEST_PORT" is found.+ */+ private static void copyFile(File src, File dst, Integer port, boolean internalCompression) throws IOException {+ try (BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(src), StandardCharsets.UTF_8));+ Writer out = new OutputStreamWriter(new FileOutputStream(dst), StandardCharsets.UTF_8)) {++ for (String line = in.readLine(); null != line; line = in.readLine()) {+ if (null != port) {+ line = line.replace("TEST_PORT", port.toString());+ }+ line = line.replace("COMPRESSION", internalCompression ? "internal" : "false");+ out.write(line);+ }+ }+ }++ public static void assertVersions(SolrClient client1, SolrClient client2) throws Exception {+ NamedList<Object> details = getDetails(client1);+ @SuppressWarnings({"unchecked"})+ ArrayList<NamedList<Object>> commits = (ArrayList<NamedList<Object>>) details.get("commits");+ Long maxVersionClient1 = getVersion(client1);+ Long maxVersionClient2 = getVersion(client2);++ if (maxVersionClient1 > 0 && maxVersionClient2 > 0) {+ assertEquals(maxVersionClient1, maxVersionClient2);+ }++ // check vs /replication?command=indexversion call+ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("qt", ReplicationHandler.PATH);+ params.set("_trace", "assertVersions");+ params.set("command", "indexversion");+ QueryRequest req = new QueryRequest(params);+ NamedList<Object> resp = client1.request(req);+ assertReplicationResponseSucceeded(resp);+ Long version = (Long) resp.get("indexversion");+ assertEquals(maxVersionClient1, version);++ // check vs /replication?command=indexversion call+ resp = client2.request(req);+ assertReplicationResponseSucceeded(resp);+ version = (Long) resp.get("indexversion");+ assertEquals(maxVersionClient2, version);+ }++ @SuppressWarnings({"unchecked"})+ public static Long getVersion(SolrClient client) throws Exception {+ NamedList<Object> details;+ ArrayList<NamedList<Object>> commits;+ details = getDetails(client);+ commits = (ArrayList<NamedList<Object>>) details.get("commits");+ Long maxVersionFollower= 0L;+ for(NamedList<Object> commit : commits) {+ Long version = (Long) commit.get("indexVersion");+ maxVersionFollower = Math.max(version, maxVersionFollower);+ }+ return maxVersionFollower;+ }++ //Simple function to wrap the invocation of replication commands on the various+ //jetty servers.+ public static void invokeReplicationCommand(String baseUrl, String pCommand) throws IOException+ {+ //String leaderUrl = buildUrl(pJettyPort) + "/" + DEFAULT_TEST_CORENAME + ReplicationHandler.PATH+"?command=" + pCommand;+ String url = baseUrl + ReplicationHandler.PATH+"?command=" + pCommand;+ URL u = new URL(url);+ InputStream stream = u.openStream();+ stream.close();+ }++ public static NamedList<Object> query(String query, SolrClient s) throws SolrServerException, IOException {+ ModifiableSolrParams params = new ModifiableSolrParams();++ params.add("q", query);+ params.add("sort","id desc");++ QueryResponse qres = s.query(params);+ return qres.getResponse();+ }++ /** will sleep up to 30 seconds, looking for expectedDocCount */+ public static NamedList<Object> rQuery(int expectedDocCount, String query, SolrClient client) throws Exception {+ int timeSlept = 0;+ NamedList<Object> res = query(query, client);+ while (expectedDocCount != numFound(res)+ && timeSlept < 30000) {+ log.info("Waiting for {} docs", expectedDocCount);+ timeSlept += 100;+ Thread.sleep(100);+ res = query(query, client);+ }+ if (log.isInfoEnabled()) {+ log.info("Waited for {}ms and found {} docs", timeSlept, numFound(res));+ }+ return res;+ }++ public static long numFound(NamedList<Object> res) {+ return ((SolrDocumentList) res.get("response")).getNumFound();+ }++ public static NamedList<Object> getDetails(SolrClient s) throws Exception {+ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("command","details");+ params.set("_trace","getDetails");+ params.set("qt",ReplicationHandler.PATH);+ QueryRequest req = new QueryRequest(params);++ NamedList<Object> res = s.request(req);+ assertReplicationResponseSucceeded(res);++ @SuppressWarnings("unchecked") NamedList<Object> details+ = (NamedList<Object>) res.get("details");++ assertNotNull("null details", details);++ return details;+ }++ public static NamedList<Object> getIndexVersion(SolrClient s) throws Exception {++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("command","indexversion");+ params.set("_trace","getIndexVersion");+ params.set("qt",ReplicationHandler.PATH);+ QueryRequest req = new QueryRequest(params);++ NamedList<Object> res = s.request(req);+ assertReplicationResponseSucceeded(res);++ return res;+ }++ public static void assertReplicationResponseSucceeded(NamedList<?> response) {+ assertNotNull("null response from server", response);+ assertNotNull("Expected replication response to have 'status' field", response.get("status"));+ assertEquals("OK", response.get("status"));+ }++ public static NamedList<Object> reloadCore(SolrClient s, String core) throws Exception {++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("action","reload");+ params.set("core", core);+ params.set("qt","/admin/cores");+ QueryRequest req = new QueryRequest(params);++ try (HttpSolrClient adminClient = adminClient(s)) {+ NamedList<Object> res = adminClient.request(req);+ assertNotNull("null response from server", res);+ return res;+ }+ }++ public static HttpSolrClient adminClient(SolrClient client) {+ String adminUrl = ((HttpSolrClient)client).getBaseURL().replace("/collection1", "");+ return SolrTestCaseJ4.getHttpSolrClient(adminUrl);+ }+++ public static void pullFromTo(String srcUrl, String destUrl) throws IOException {+ URL url;+ InputStream stream;+ String leaderUrl = destUrl+ + ReplicationHandler.PATH+"?wait=true&command=fetchindex&leaderUrl="+ + srcUrl+ + ReplicationHandler.PATH;+ url = new URL(leaderUrl);+ stream = url.openStream();+ stream.close();+ }++ public static class SolrInstance {++ final private String name;+ private Integer testPort;+ final private File homeDir;+ private File confDir;+ private File dataDir;++ /**+ * @param homeDir Base directory to build solr configuration and index in+ * @param name used to pick which+ * "solrconfig-${name}.xml" file gets copied+ * to solrconfig.xml in new conf dir.+ * @param testPort if not null, used as a replacement for+ * TEST_PORT in the cloned config files.+ */+ public SolrInstance(File homeDir, String name, Integer testPort) {+ this.homeDir = homeDir;+ this.name = name;+ this.testPort = testPort;+ }++ public String getHomeDir() {+ return homeDir.toString();+ }++ public String getSchemaFile() {+ return CONF_DIR + "schema-replication1.xml";+ }++ public String getConfDir() {+ return confDir.toString();+ }++ public String getDataDir() {+ return dataDir.getAbsolutePath();+ }++ public String getSolrConfigFile() {+ return CONF_DIR + "solrconfig-"+name+".xml";+ }++ /** If it needs to change */+ public void setTestPort(Integer testPort) {+ this.testPort = testPort;+ }++ public void setUp() throws Exception {+ System.setProperty("solr.test.sys.prop1", "propone");+ System.setProperty("solr.test.sys.prop2", "proptwo");++ Properties props = new Properties();+ props.setProperty("name", "collection1");++ SolrTestCaseJ4.writeCoreProperties(homeDir.toPath().resolve("collection1"), props, "TestReplicationHandler");++ dataDir = new File(homeDir + "/collection1", "data");+ confDir = new File(homeDir + "/collection1", "conf");++ homeDir.mkdirs();+ dataDir.mkdirs();+ confDir.mkdirs();++ copyConfigFile(getSolrConfigFile(), "solrconfig.xml");+ copyConfigFile(getSchemaFile(), "schema.xml");+ copyConfigFile(CONF_DIR + "solrconfig.snippet.randomindexconfig.xml",+ "solrconfig.snippet.randomindexconfig.xml");+ }++ public void copyConfigFile(String srcFile, String destFile)+ throws IOException {+ copyFile(SolrTestCaseJ4.getFile(srcFile),+ new File(confDir, destFile),+ testPort, LuceneTestCase.random().nextBoolean());+ }+ }+}diff --git a/solr/core/src/test/org/apache/solr/handler/TestHealthCheckHandlerLegacyMode.java b/solr/core/src/test/org/apache/solr/handler/TestHealthCheckHandlerLegacyMode.javanew file mode 100644index 00000000000..b19e5cf336e--- /dev/null+++ b/solr/core/src/test/org/apache/solr/handler/TestHealthCheckHandlerLegacyMode.java@@ -0,0 +1,203 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.handler;++import org.apache.lucene.util.LuceneTestCase.Slow;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.SolrTestCaseJ4.SuppressSSL;+import org.apache.solr.client.solrj.embedded.JettySolrRunner;+import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.request.HealthCheckRequest;+import org.apache.solr.common.params.CommonParams;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.util.TestInjection;+import org.junit.After;+import org.junit.Before;+import org.junit.Test;++import java.io.IOException;++import static org.apache.solr.handler.ReplicationTestHelper.*;+++/**+ * Test for HealthCheckHandler in legacy mode+ *+ *+ */+@Slow+@SuppressSSL // Currently unknown why SSL does not work with this test+public class TestHealthCheckHandlerLegacyMode extends SolrTestCaseJ4 {+ HttpSolrClient leaderClientHealthCheck, followerClientHealthCheck;++ JettySolrRunner leaderJetty, followerJetty;+ HttpSolrClient leaderClient, followerClient;+ ReplicationTestHelper.SolrInstance leader = null, follower = null;++ private static final String context = "/solr";++ @Before+ public void setUp() throws Exception {+ super.setUp();++ systemSetPropertySolrDisableUrlAllowList("true");++ leader = new ReplicationTestHelper.SolrInstance(createTempDir("solr-instance").toFile(), "leader", null);+ leader.setUp();+ leaderJetty = ReplicationTestHelper.createAndStartJetty(leader);+ leaderClient = ReplicationTestHelper.createNewSolrClient(buildUrl(leaderJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME);+ leaderClientHealthCheck = ReplicationTestHelper.createNewSolrClient(buildUrl(leaderJetty.getLocalPort(), context));++ follower = new SolrInstance(createTempDir("solr-instance").toFile(), "follower", leaderJetty.getLocalPort());+ follower.setUp();+ followerJetty = createAndStartJetty(follower);+ followerClient = ReplicationTestHelper.createNewSolrClient(buildUrl(followerJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME);+ followerClientHealthCheck = ReplicationTestHelper.createNewSolrClient(buildUrl(followerJetty.getLocalPort(), context));++ System.setProperty("solr.indexfetcher.sotimeout2", "45000");+ }++ public void clearIndexWithReplication() throws Exception {+ if (numFound(ReplicationTestHelper.query("*:*", leaderClient)) != 0) {+ leaderClient.deleteByQuery("*:*");+ leaderClient.commit();+ // wait for replication to sync & verify+ assertEquals(0, numFound(rQuery(0, "*:*", followerClient)));+ }+ }++ @Override+ @After+ public void tearDown() throws Exception {+ super.tearDown();+ if (null != leaderJetty) {+ leaderJetty.stop();+ leaderJetty = null;+ }+ if (null != followerJetty) {+ followerJetty.stop();+ followerJetty = null;+ }+ if (null != leaderClient) {+ leaderClient.close();+ leaderClient = null;+ }+ if (null != followerClient) {+ followerClient.close();+ followerClient = null;+ }+ if (null != leaderClientHealthCheck) {+ leaderClientHealthCheck.close();+ leaderClientHealthCheck = null;+ }++ if (null != followerClientHealthCheck) {+ followerClientHealthCheck.close();+ followerClientHealthCheck = null;+ }+ System.clearProperty("solr.indexfetcher.sotimeout");+ }+++ @Test+ // keep this+ public void doTestHealthCheckWithReplication() throws Exception {+ int nDocs = 500;++ TestInjection.delayBeforeFollowerCommitRefresh = random().nextInt(10);++ // stop replication so that the follower doesn't pull the index+ invokeReplicationCommand(buildUrl(followerJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME, "disablepoll");++ // create multiple commits+ int docsAdded = 0;+ for (int i = 0; docsAdded < nDocs / 2; i++, docsAdded++) {+ index(leaderClient, "id", i, "name", "name = " + i);+ if (i % 10 == 0) {+ leaderClient.commit();+ }+ }++ leaderClient.commit();++ assertNumFoundWithQuery(leaderClient, docsAdded);++ // ensure that the leader is always happy+ // first try without specifying maxGenerationLag lag+ ModifiableSolrParams solrParamsLeaderHealthCheck = new ModifiableSolrParams();+ HealthCheckRequest healthCheckRequestLeader = new HealthCheckRequest();+ assertEquals(CommonParams.OK, healthCheckRequestLeader.process(leaderClientHealthCheck).getResponse().get(CommonParams.STATUS));++ // now try adding maxGenerationLag request param+ solrParamsLeaderHealthCheck.add(HealthCheckRequest.PARAM_MAX_GENERATION_LAG, "2");+ assertEquals(CommonParams.OK, healthCheckRequestLeader.process(leaderClientHealthCheck).getResponse().get(CommonParams.STATUS));++ // follower should report healthy if maxGenerationLag is not specified+ HealthCheckRequest healthCheckRequestFollower = new HealthCheckRequest();+ assertEquals(CommonParams.OK, healthCheckRequestFollower.process(followerClientHealthCheck).getResponse().get(CommonParams.STATUS));++ // ensure follower is unhealthy when maxGenerationLag is specified+ // ModifiableSolrParams params = new ModifiableSolrParams();+ healthCheckRequestFollower = new HealthCheckRequest();+ healthCheckRequestFollower.setMaxGenerationLag(2);+ assertEquals(CommonParams.FAILURE, healthCheckRequestFollower.process(followerClientHealthCheck).getResponse().get(CommonParams.STATUS));++ // enable polling, force replication and ensure that the follower is healthy+ //invokeReplicationCommand(buildUrl(followerJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME, "enablepoll");+ pullFromTo(leaderJetty, followerJetty);++ // check replicated and is healthy+ assertNumFoundWithQuery(followerClient, docsAdded);+ assertEquals(CommonParams.OK, healthCheckRequestFollower.process(followerClientHealthCheck).getResponse().get(CommonParams.STATUS));+ assertVersions(leaderClient, followerClient);++ // index more docs on the leader+ for (int i = docsAdded; docsAdded < nDocs; i++, docsAdded++) {+ index(leaderClient, "id", i, "name", "name = " + i);+ if (i % 10 == 0) {+ leaderClient.commit();+ }+ }+ leaderClient.commit();++ assertNumFoundWithQuery(leaderClient, docsAdded);++ // we have added docs to the leader and polling is disabled, this should fail+ healthCheckRequestFollower = new HealthCheckRequest();+ healthCheckRequestFollower.setMaxGenerationLag(2);+ assertEquals(CommonParams.FAILURE, healthCheckRequestFollower.process(followerClientHealthCheck).getResponse().get(CommonParams.STATUS));++ //force replication and ensure that the follower is healthy+ pullFromTo(leaderJetty, followerJetty);+ assertEquals(CommonParams.OK, healthCheckRequestFollower.process(followerClientHealthCheck).getResponse().get(CommonParams.STATUS));+ assertNumFoundWithQuery(followerClient, docsAdded);+ assertVersions(leaderClient, followerClient);+ }++ public static void pullFromTo(JettySolrRunner srcSolr, JettySolrRunner destSolr) throws IOException {+ String srcUrl = buildUrl(srcSolr.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME;+ String destUrl = buildUrl(destSolr.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME;+ ReplicationTestHelper.pullFromTo(srcUrl, destUrl);+ }++ private void assertNumFoundWithQuery(HttpSolrClient client, int nDocs) throws Exception {+ NamedList<Object> queryRsp = rQuery(nDocs, "*:*", client);+ assertEquals(nDocs, numFound(queryRsp));+ }++}diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.javaindex 498d4d11150..9c2417c3a3b 100644--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java@@ -16,30 +16,6 @@*/package org.apache.solr.handler;-import java.io.BufferedReader;-import java.io.File;-import java.io.FileInputStream;-import java.io.FileOutputStream;-import java.io.FilenameFilter;-import java.io.IOException;-import java.io.InputStream;-import java.io.InputStreamReader;-import java.io.OutputStreamWriter;-import java.io.Writer;-import java.lang.invoke.MethodHandles;-import java.net.MalformedURLException;-import java.net.URL;-import java.nio.charset.StandardCharsets;-import java.nio.file.Paths;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Collection;-import java.util.Date;-import java.util.List;-import java.util.Properties;-import java.util.Set;-import java.util.concurrent.TimeUnit;-import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;import org.apache.lucene.store.Directory;@@ -48,13 +24,12 @@ import org.apache.lucene.store.NIOFSDirectory;import org.apache.lucene.util.Constants;import org.apache.lucene.util.LuceneTestCase.Slow;import org.apache.lucene.util.TestUtil;-import org.apache.solr.BaseDistributedSearchTestCase;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.SolrTestCaseJ4.SuppressSSL;import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.SolrQuery;+import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.embedded.JettyConfig;import org.apache.solr.client.solrj.embedded.JettySolrRunner;@@ -62,8 +37,8 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;import org.apache.solr.client.solrj.request.GenericSolrRequest;import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.client.solrj.request.UpdateRequest;-import org.apache.solr.client.solrj.response.SimpleSolrResponse;import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.client.solrj.response.SimpleSolrResponse;import org.apache.solr.client.solrj.response.UpdateResponse;import org.apache.solr.common.SolrDocument;import org.apache.solr.common.SolrDocumentList;@@ -90,6 +65,17 @@ import org.junit.Test;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+import java.io.File;+import java.io.FilenameFilter;+import java.io.IOException;+import java.io.InputStream;+import java.lang.invoke.MethodHandles;+import java.net.URL;+import java.nio.file.Paths;+import java.util.*;+import java.util.concurrent.TimeUnit;++import static org.apache.solr.handler.ReplicationTestHelper.*;import static org.junit.matchers.JUnitMatchers.containsString;/**@@ -106,10 +92,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- private static final String CONF_DIR = "solr"- + File.separator + "collection1" + File.separator + "conf"- + File.separator;-JettySolrRunner leaderJetty, followerJetty, repeaterJetty;HttpSolrClient leaderClient, followerClient, repeaterClient;SolrInstance leader = null, follower = null, repeater = null;@@ -119,16 +101,16 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {// number of docs to index... decremented for each test case to tell if we accidentally reuse// index from previous test methodstatic int nDocs = 500;-+/* For testing backward compatibility, remove for 10.x */- private static boolean useLegacyParams = false;-+ private static boolean useLegacyParams = false;+@BeforeClasspublic static void beforeClass() {useLegacyParams = rarely();}-+@Beforepublic void setUp() throws Exception {super.setUp();@@ -139,13 +121,13 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {leader = new SolrInstance(createTempDir("solr-instance").toFile(), "leader", null);leader.setUp();leaderJetty = createAndStartJetty(leader);- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());+ leaderClient = ReplicationTestHelper.createNewSolrClient(buildUrl(leaderJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME);follower = new SolrInstance(createTempDir("solr-instance").toFile(), "follower", leaderJetty.getLocalPort());follower.setUp();followerJetty = createAndStartJetty(follower);- followerClient = createNewSolrClient(followerJetty.getLocalPort());-+ followerClient = ReplicationTestHelper.createNewSolrClient(buildUrl(followerJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME);+System.setProperty("solr.indexfetcher.sotimeout2", "45000");}@@ -192,18 +174,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {return jetty;}- static HttpSolrClient createNewSolrClient(int port) {- try {- // setup the client...- final String baseUrl = buildUrl(port) + "/" + DEFAULT_TEST_CORENAME;- HttpSolrClient client = getHttpSolrClient(baseUrl, 15000, 90000);- return client;- }- catch (Exception ex) {- throw new RuntimeException(ex);- }- }-static int index(SolrClient s, Object... fields) throws Exception {SolrInputDocument doc = new SolrInputDocument();for (int i = 0; i < fields.length; i += 2) {@@ -238,13 +208,13 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {}return res;}-+private long numFound(NamedList<Object> res) {return ((SolrDocumentList) res.get("response")).getNumFound();}private NamedList<Object> getDetails(SolrClient s) throws Exception {-+ModifiableSolrParams params = new ModifiableSolrParams();params.set("command","details");@@ -255,16 +225,16 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {NamedList<Object> res = s.request(req);assertReplicationResponseSucceeded(res);- @SuppressWarnings("unchecked") NamedList<Object> details+ @SuppressWarnings("unchecked") NamedList<Object> details= (NamedList<Object>) res.get("details");assertNotNull("null details", details);return details;}-+private NamedList<Object> getIndexVersion(SolrClient s) throws Exception {-+ModifiableSolrParams params = new ModifiableSolrParams();params.set("command","indexversion");params.set("_trace","getIndexVersion");@@ -276,7 +246,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {return res;}-+private NamedList<Object> reloadCore(SolrClient s, String core) throws Exception {ModifiableSolrParams params = new ModifiableSolrParams();@@ -324,20 +294,20 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {@Testpublic void doTestDetails() throws Exception {followerJetty.stop();-+follower.setTestPort(leaderJetty.getLocalPort());follower.copyConfigFile(CONF_DIR + "solrconfig-follower.xml", "solrconfig.xml");followerJetty = createAndStartJetty(follower);-+followerClient.close();leaderClient.close();- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());- followerClient = createNewSolrClient(followerJetty.getLocalPort());-+ leaderClient = ReplicationTestHelper.createNewSolrClient(buildUrl(leaderJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME);+ followerClient = ReplicationTestHelper.createNewSolrClient(buildUrl(followerJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME);+clearIndexWithReplication();- {+ {NamedList<Object> details = getDetails(leaderClient);-+assertEquals("leader isLeader?","true", details.get("isLeader"));assertEquals("leader isFollower?",@@ -361,12 +331,12 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {details = getDetails(followerClient);replicatedAtCount = (List<?>) ((NamedList<?>) details.get("follower")).get("indexReplicatedAtList");}-+assertNotNull("Expected to see that the follower has replicated" + i + ": " + details.toString(), replicatedAtCount);-- // we can have more replications than we added docs because a replication can legally fail and try++ // we can have more replications than we added docs because a replication can legally fail and try// again (sometimes we cannot merge into a live index and have to try again)- assertTrue("i:" + i + " replicationCount:" + replicatedAtCount.size(), replicatedAtCount.size() >= i);+ assertTrue("i:" + i + " replicationCount:" + replicatedAtCount.size(), replicatedAtCount.size() >= i);}assertEquals(i + ": " + "follower isLeader?", "false", details.get("isLeader"));@@ -393,11 +363,11 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {repeater = new SolrInstance(createTempDir("solr-instance").toFile(), "repeater", leaderJetty.getLocalPort());repeater.setUp();repeaterJetty = createAndStartJetty(repeater);- repeaterClient = createNewSolrClient(repeaterJetty.getLocalPort());+ repeaterClient = ReplicationTestHelper.createNewSolrClient(buildUrl(repeaterJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME);+-NamedList<Object> details = getDetails(repeaterClient);-+assertEquals("repeater isLeader?","true", details.get("isLeader"));assertEquals("repeater isFollower?",@@ -408,13 +378,13 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {details.get("follower"));} finally {- try {- if (repeaterJetty != null) repeaterJetty.stop();+ try {+ if (repeaterJetty != null) repeaterJetty.stop();} catch (Exception e) { /* :NOOP: */ }if (repeaterClient != null) repeaterClient.close();}}-+@Testpublic void testLegacyConfiguration() throws Exception {SolrInstance solrInstance = null;@@ -424,11 +394,11 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {solrInstance = new SolrInstance(createTempDir("solr-instance").toFile(), "replication-legacy", leaderJetty.getLocalPort());solrInstance.setUp();instanceJetty = createAndStartJetty(solrInstance);- client = createNewSolrClient(instanceJetty.getLocalPort());+ client = ReplicationTestHelper.createNewSolrClient(buildUrl(instanceJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME);+-NamedList<Object> details = getDetails(client);-+assertEquals("repeater isLeader?","true", details.get("isLeader"));assertEquals("repeater isFollower?",@@ -453,25 +423,25 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {*/public void testEmptyCommits() throws Exception {clearIndexWithReplication();-+// add a doc to leader and commitindex(leaderClient, "id", "1", "name", "empty1");emptyUpdate(leaderClient, "commit", "true");// force replication- pullFromLeaderToFollower();+ pullFromTo(leaderJetty, followerJetty);// verify doc is on followerrQuery(1, "name:empty1", followerClient);assertVersions(leaderClient, followerClient);// do a completely empty commit on leader and force replicationemptyUpdate(leaderClient, "commit", "true");- pullFromLeaderToFollower();+ pullFromTo(leaderJetty, followerJetty);// add another doc and verify follower gets itindex(leaderClient, "id", "2", "name", "empty2");emptyUpdate(leaderClient, "commit", "true");// force replication- pullFromLeaderToFollower();+ pullFromTo(leaderJetty, followerJetty);rQuery(1, "name:empty2", followerClient);assertVersions(leaderClient, followerClient);@@ -479,8 +449,8 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {// add a third doc but don't open a new searcher on leaderindex(leaderClient, "id", "3", "name", "empty3");emptyUpdate(leaderClient, "commit", "true", "openSearcher", "false");- pullFromLeaderToFollower();-+ pullFromTo(leaderJetty, followerJetty);+// verify follower can search the doc, but leader doesn'trQuery(0, "name:empty3", leaderClient);rQuery(1, "name:empty3", followerClient);@@ -488,7 +458,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {// final doc with hard commit, follower and leader both showing all docsindex(leaderClient, "id", "4", "name", "empty4");emptyUpdate(leaderClient, "commit", "true");- pullFromLeaderToFollower();+ pullFromTo(leaderJetty, followerJetty);String q = "name:(empty1 empty2 empty3 empty4)";rQuery(4, q, leaderClient);@@ -505,9 +475,9 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {index(leaderClient, "id", i, "name", "name = " + i);}- invokeReplicationCommand(leaderJetty.getLocalPort(), "disableReplication");- invokeReplicationCommand(followerJetty.getLocalPort(), "disablepoll");-+ invokeReplicationCommand(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME, "disableReplication");+ invokeReplicationCommand(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME, "disablepoll");+leaderClient.commit();assertEquals(nDocs, numFound(rQuery(nDocs, "*:*", leaderClient)));@@ -530,8 +500,8 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {assertEquals(1, numFound(rQuery(1, "id:555", followerClient)));//Let's fetch the index rather than rely on the polling.- invokeReplicationCommand(leaderJetty.getLocalPort(), "enablereplication");- invokeReplicationCommand(followerJetty.getLocalPort(), "fetchindex");+ invokeReplicationCommand(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME, "enablereplication");+ invokeReplicationCommand(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME, "fetchindex");/*//the follower should have done a full copy of the index so the doc with id:555 should not be there in the follower now@@ -543,20 +513,10 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {followerQueryRsp = rQuery(nDocs, "*:*", followerClient);followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response");assertEquals(nDocs, followerQueryResult.getNumFound());-+*/}- //Simple function to wrap the invocation of replication commands on the various- //jetty servers.- static void invokeReplicationCommand(int pJettyPort, String pCommand) throws IOException- {- String leaderUrl = buildUrl(pJettyPort) + "/" + DEFAULT_TEST_CORENAME + ReplicationHandler.PATH+"?command=" + pCommand;- URL u = new URL(leaderUrl);- InputStream stream = u.openStream();- stream.close();- }-@Testpublic void doTestIndexAndConfigReplication() throws Exception {@@ -582,7 +542,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {//compare resultsString cmp = BaseDistributedSearchTestCase.compare(leaderQueryResult, followerQueryResult, 0, null);assertNull(cmp);-+assertVersions(leaderClient, followerClient);//start config files replication test@@ -596,7 +556,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {leaderJetty = createAndStartJetty(leader);leaderClient.close();- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());+ leaderClient = createNewSolrClient(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);follower.setTestPort(leaderJetty.getLocalPort());follower.copyConfigFile(follower.getSolrConfigFile(), "solrconfig.xml");@@ -615,13 +575,13 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {followerJetty = createAndStartJetty(follower);followerClient.close();- followerClient = createNewSolrClient(followerJetty.getLocalPort());+ followerClient = createNewSolrClient(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);//add a doc with new field and commit on leader to trigger index fetch from follower.index(leaderClient, "id", "2000", "name", "name = " + 2000, "newname", "newname = " + 2000);leaderClient.commit();assertEquals(1, numFound( rQuery(1, "*:*", leaderClient)));-+followerQueryRsp = rQuery(1, "*:*", followerClient);assertVersions(leaderClient, followerClient);SolrDocument d = ((SolrDocumentList) followerQueryRsp.get("response")).get(0);@@ -629,7 +589,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {assertTrue(followerFooDir.isDirectory());assertTrue(followerBarFile.exists());-+checkForSingleIndex(leaderJetty);checkForSingleIndex(followerJetty, true);}@@ -661,23 +621,23 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {assertNull(cmp);// start stop polling test- invokeReplicationCommand(followerJetty.getLocalPort(), "disablepoll");-+ invokeReplicationCommand(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME, "disablepoll");+index(leaderClient, "id", 501, "name", "name = " + 501);leaderClient.commit();//get docs from leader and check if number is equal to leaderassertEquals(nDocs+1, numFound(rQuery(nDocs+1, "*:*", leaderClient)));-+// NOTE: this test is wierd, we want to verify it DOESNT replicate...// for now, add a sleep for this.., but the logic is wierd.Thread.sleep(3000);-+//get docs from follower and check if number is not equal to leader; polling is disabledassertEquals(nDocs, numFound(rQuery(nDocs, "*:*", followerClient)));// re-enable replication- invokeReplicationCommand(followerJetty.getLocalPort(), "enablepoll");+ invokeReplicationCommand(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME, "enablepoll");assertEquals(nDocs+1, numFound(rQuery(nDocs+1, "*:*", followerClient)));}@@ -701,7 +661,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {// close and re-create leader client because its connection pool has stale connectionsleaderClient.close();- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());+ leaderClient = createNewSolrClient(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);nDocs--;for (int i = 0; i < nDocs; i++)@@ -746,13 +706,13 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {waitForLeaderToShutdown.waitFor("Gave up after waiting an obscene amount of time for leader to shut down",() -> leaderJetty.isStopped() );-- for(int retries=0; ;retries++) {++ for(int retries=0; ;retries++) {Thread.yield(); // might not be necessary at all// poll interval on follower is 1 second, so we just sleep for a few secondsThread.sleep(2000);-+NamedList<Object> followerDetails=null;try {followerDetails = getFollowerDetails();@@ -769,10 +729,10 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {if (retries>9) {log.error("giving up: ", notYet);throw notYet;- }+ }}}-+leaderJetty.start();// poll interval on follower is 1 second, so we just sleep for a few seconds@@ -825,7 +785,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {followerJetty.stop();followerJetty = createAndStartJetty(follower);followerClient.close();- followerClient = createNewSolrClient(followerJetty.getLocalPort());+ followerClient = ReplicationTestHelper.createNewSolrClient(buildUrl(followerJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME);leaderClient.deleteByQuery("*:*");followerClient.deleteByQuery("*:*");@@ -835,7 +795,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {index(leaderClient, "id", i, "name", "name = " + i);// make sure prepareCommit doesn't mess up commit (SOLR-3938)-+// todo: make SolrJ easier to pass arbitrary params to// TODO: precommit WILL screw with the rest of this test@@ -844,7 +804,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {NamedList<Object> leaderQueryRsp = rQuery(nDocs, "*:*", leaderClient);SolrDocumentList leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response");assertEquals(nDocs, leaderQueryResult.getNumFound());-+String urlKey = "leaderUrl";if (useLegacyParams) {urlKey = "masterUrl";@@ -856,7 +816,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {URL url = new URL(leaderUrl);InputStream stream = url.openStream();stream.close();-+//get docs from follower and check if number is equal to leaderNamedList<Object> followerQueryRsp = rQuery(nDocs, "*:*", followerClient);SolrDocumentList followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response");@@ -866,15 +826,15 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {assertEquals(null, cmp);// index fetch from the follower to the leader-+for (int i = nDocs; i < nDocs + 3; i++)index(followerClient, "id", i, "name", "name = " + i);followerClient.commit();-- pullFromFollowerToLeader();++ pullFromTo(followerJetty, leaderJetty);rQuery(nDocs + 3, "*:*", leaderClient);-+//get docs from follower and check if number is equal to leaderfollowerQueryRsp = rQuery(nDocs + 3, "*:*", followerClient);followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response");@@ -886,9 +846,9 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {assertEquals(null, cmp);assertVersions(leaderClient, followerClient);-- pullFromFollowerToLeader();-++ pullFromTo(followerJetty, leaderJetty);+//get docs from follower and check if number is equal to leaderfollowerQueryRsp = rQuery(nDocs + 3, "*:*", followerClient);followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response");@@ -898,18 +858,18 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response");cmp = BaseDistributedSearchTestCase.compare(leaderQueryResult, followerQueryResult, 0, null);assertEquals(null, cmp);-+assertVersions(leaderClient, followerClient);-+// now force a new index directoryfor (int i = nDocs + 3; i < nDocs + 7; i++)index(leaderClient, "id", i, "name", "name = " + i);-+leaderClient.commit();-- pullFromFollowerToLeader();++ pullFromTo(followerJetty, leaderJetty);rQuery((int) followerQueryResult.getNumFound(), "*:*", leaderClient);-+//get docs from follower and check if number is equal to leaderfollowerQueryRsp = rQuery(nDocs + 3, "*:*", followerClient);followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response");@@ -919,10 +879,10 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response");cmp = BaseDistributedSearchTestCase.compare(leaderQueryResult, followerQueryResult, 0, null);assertEquals(null, cmp);-+assertVersions(leaderClient, followerClient);- pullFromFollowerToLeader();-+ pullFromTo(followerJetty, leaderJetty);+//get docs from follower and check if number is equal to leaderfollowerQueryRsp = rQuery(nDocs + 3, "*:*", followerClient);followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response");@@ -932,27 +892,27 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response");cmp = BaseDistributedSearchTestCase.compare(leaderQueryResult, followerQueryResult, 0, null);assertEquals(null, cmp);-+assertVersions(leaderClient, followerClient);-+NamedList<Object> details = getDetails(leaderClient);-+details = getDetails(followerClient);-+checkForSingleIndex(leaderJetty);checkForSingleIndex(followerJetty);}--++@Test//commented 20-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018public void doTestStressReplication() throws Exception {// change solrconfig on follower// this has no entry for pollinginterval-+// get us a straight standard fs dir rather than mock*dirboolean useStraightStandardDirectory = random().nextBoolean();-+if (useStraightStandardDirectory) {useFactory(null);}@@ -963,30 +923,30 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {try {follower.setTestPort(leaderJetty.getLocalPort());- follower.copyConfigFile(CONF_DIR +"solrconfig-follower1.xml", "solrconfig.xml");- follower.copyConfigFile(CONF_DIR +followerSchema, "schema.xml");+ follower.copyConfigFile(CONF_DIR + "solrconfig-follower1.xml", "solrconfig.xml");+ follower.copyConfigFile(CONF_DIR + followerSchema, "schema.xml");followerJetty.stop();followerJetty = createAndStartJetty(follower);followerClient.close();- followerClient = createNewSolrClient(followerJetty.getLocalPort());+ followerClient = createNewSolrClient(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);leader.copyConfigFile(CONF_DIR + "solrconfig-leader3.xml","solrconfig.xml");leaderJetty.stop();leaderJetty = createAndStartJetty(leader);leaderClient.close();- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());-+ leaderClient = createNewSolrClient(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);+leaderClient.deleteByQuery("*:*");followerClient.deleteByQuery("*:*");followerClient.commit();-+int maxDocs = TEST_NIGHTLY ? 1000 : 75;int rounds = TEST_NIGHTLY ? 45 : 3;int totalDocs = 0;int id = 0;for (int x = 0; x < rounds; x++) {-+final boolean confCoreReload = random().nextBoolean();if (confCoreReload) {// toggle the schema file used@@ -995,25 +955,25 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {FOLLOWER_SCHEMA_2 : FOLLOWER_SCHEMA_1;leader.copyConfigFile(CONF_DIR + followerSchema, "schema.xml");}-+int docs = random().nextInt(maxDocs) + 1;for (int i = 0; i < docs; i++) {index(leaderClient, "id", id++, "name", "name = " + i);}-+totalDocs += docs;leaderClient.commit();-+NamedList<Object> leaderQueryRsp = rQuery(totalDocs, "*:*", leaderClient);SolrDocumentList leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response");assertEquals(totalDocs, leaderQueryResult.getNumFound());-+// index fetch- Date followerCoreStart = watchCoreStartAt(followerClient, 30*1000, null);- pullFromLeaderToFollower();+ Date followerCoreStart = watchCoreStartAt(followerClient, 30 * 1000, null);+ pullFromTo(leaderJetty, followerJetty);if (confCoreReload) {- watchCoreStartAt(followerClient, 30*1000, followerCoreStart);+ watchCoreStartAt(followerClient, 30 * 1000, followerCoreStart);}// get docs from follower and check if number is equal to leader@@ -1025,15 +985,15 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {String cmp = BaseDistributedSearchTestCase.compare(leaderQueryResult,followerQueryResult, 0, null);assertEquals(null, cmp);-+assertVersions(leaderClient, followerClient);-+checkForSingleIndex(leaderJetty);-+if (!Constants.WINDOWS) {checkForSingleIndex(followerJetty);}-+if (random().nextBoolean()) {// move the follower aheadfor (int i = 0; i < 3; i++) {@@ -1041,9 +1001,9 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {}followerClient.commit();}-+}-+} finally {if (useStraightStandardDirectory) {resetFactory();@@ -1058,7 +1018,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {private void checkForSingleIndex(JettySolrRunner jetty) {checkForSingleIndex(jetty, false);}-+private void checkForSingleIndex(JettySolrRunner jetty, boolean afterReload) {CoreContainer cores = jetty.getCoreContainer();Collection<SolrCore> theCores = cores.getCores();@@ -1103,11 +1063,12 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {return list.length;}- private void pullFromLeaderToFollower() throws MalformedURLException,- IOException {- pullFromTo(leaderJetty, followerJetty);+ public static void pullFromTo(JettySolrRunner srcSolr, JettySolrRunner destSolr) throws IOException {+ String srcUrl = buildUrl(srcSolr.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME;+ String destUrl = buildUrl(destSolr.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME;+ ReplicationTestHelper.pullFromTo(srcUrl, destUrl);}-+@Testpublic void doTestRepeater() throws Exception {// no polling@@ -1116,7 +1077,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {followerJetty.stop();followerJetty = createAndStartJetty(follower);followerClient.close();- followerClient = createNewSolrClient(followerJetty.getLocalPort());+ followerClient = createNewSolrClient(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);try {repeater = new SolrInstance(createTempDir("solr-instance").toFile(), "repeater", leaderJetty.getLocalPort());@@ -1127,47 +1088,47 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {if (repeaterClient != null) {repeaterClient.close();}- repeaterClient = createNewSolrClient(repeaterJetty.getLocalPort());-+ repeaterClient = createNewSolrClient(buildUrl(repeaterJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);+for (int i = 0; i < 3; i++)index(leaderClient, "id", i, "name", "name = " + i);leaderClient.commit();-+pullFromTo(leaderJetty, repeaterJetty);-+rQuery(3, "*:*", repeaterClient);-+pullFromTo(repeaterJetty, followerJetty);-+rQuery(3, "*:*", followerClient);-+assertVersions(leaderClient, repeaterClient);assertVersions(repeaterClient, followerClient);-+for (int i = 0; i < 4; i++)index(repeaterClient, "id", i, "name", "name = " + i);repeaterClient.commit();-+pullFromTo(leaderJetty, repeaterJetty);-+rQuery(3, "*:*", repeaterClient);-+pullFromTo(repeaterJetty, followerJetty);-+rQuery(3, "*:*", followerClient);-+for (int i = 3; i < 6; i++)index(leaderClient, "id", i, "name", "name = " + i);-+leaderClient.commit();-+pullFromTo(leaderJetty, repeaterJetty);-+rQuery(6, "*:*", repeaterClient);-+pullFromTo(repeaterJetty, followerJetty);-+rQuery(6, "*:*", followerClient);} finally {@@ -1179,69 +1140,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {repeaterClient.close();}}-- }-- private void assertVersions(SolrClient client1, SolrClient client2) throws Exception {- NamedList<Object> details = getDetails(client1);- @SuppressWarnings({"unchecked"})- ArrayList<NamedList<Object>> commits = (ArrayList<NamedList<Object>>) details.get("commits");- Long maxVersionClient1 = getVersion(client1);- Long maxVersionClient2 = getVersion(client2);- if (maxVersionClient1 > 0 && maxVersionClient2 > 0) {- assertEquals(maxVersionClient1, maxVersionClient2);- }-- // check vs /replication?command=indexversion call- ModifiableSolrParams params = new ModifiableSolrParams();- params.set("qt", ReplicationHandler.PATH);- params.set("_trace", "assertVersions");- params.set("command", "indexversion");- QueryRequest req = new QueryRequest(params);- NamedList<Object> resp = client1.request(req);- assertReplicationResponseSucceeded(resp);- Long version = (Long) resp.get("indexversion");- assertEquals(maxVersionClient1, version);-- // check vs /replication?command=indexversion call- resp = client2.request(req);- assertReplicationResponseSucceeded(resp);- version = (Long) resp.get("indexversion");- assertEquals(maxVersionClient2, version);- }-- @SuppressWarnings({"unchecked"})- private Long getVersion(SolrClient client) throws Exception {- NamedList<Object> details;- ArrayList<NamedList<Object>> commits;- details = getDetails(client);- commits = (ArrayList<NamedList<Object>>) details.get("commits");- Long maxVersionFollower= 0L;- for(NamedList<Object> commit : commits) {- Long version = (Long) commit.get("indexVersion");- maxVersionFollower = Math.max(version, maxVersionFollower);- }- return maxVersionFollower;- }-- private void pullFromFollowerToLeader() throws MalformedURLException,- IOException {- pullFromTo(followerJetty, leaderJetty);- }-- private void pullFromTo(JettySolrRunner from, JettySolrRunner to) throws IOException {- String leaderUrl;- URL url;- InputStream stream;- leaderUrl = buildUrl(to.getLocalPort())- + "/" + DEFAULT_TEST_CORENAME- + ReplicationHandler.PATH+"?wait=true&command=fetchindex&leaderUrl="- + buildUrl(from.getLocalPort())- + "/" + DEFAULT_TEST_CORENAME + ReplicationHandler.PATH;- url = new URL(leaderUrl);- stream = url.openStream();- stream.close();}@Test@@ -1255,26 +1154,25 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {leaderClient.commit();-//change solrconfig having 'replicateAfter startup' option on leaderleader.copyConfigFile(CONF_DIR + "solrconfig-leader2.xml",- "solrconfig.xml");+ "solrconfig.xml");leaderJetty.stop();leaderJetty = createAndStartJetty(leader);leaderClient.close();- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());-+ leaderClient = createNewSolrClient(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);+for (int i = 0; i < nDocs; i++)index(leaderClient, "id", i, "name", "name = " + i);leaderClient.commit();-+NamedList<Object> leaderQueryRsp = rQuery(nDocs, "*:*", leaderClient);SolrDocumentList leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response");assertEquals(nDocs, leaderQueryResult.getNumFound());-+follower.setTestPort(leaderJetty.getLocalPort());follower.copyConfigFile(follower.getSolrConfigFile(), "solrconfig.xml");@@ -1282,7 +1180,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {//start followerfollowerJetty = createAndStartJetty(follower);followerClient.close();- followerClient = createNewSolrClient(followerJetty.getLocalPort());+ followerClient = createNewSolrClient(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);//get docs from follower and check if number is equal to leaderNamedList<Object> followerQueryRsp = rQuery(nDocs, "*:*", followerClient);@@ -1294,67 +1192,67 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {assertEquals(null, cmp);}-+@Testpublic void doTestReplicateAfterStartupWithNoActivity() throws Exception {useFactory(null);try {-+// stop followerfollowerJetty.stop();-+nDocs--;leaderClient.deleteByQuery("*:*");-+leaderClient.commit();-+// change solrconfig having 'replicateAfter startup' option on leaderleader.copyConfigFile(CONF_DIR + "solrconfig-leader2.xml","solrconfig.xml");-+leaderJetty.stop();-+leaderJetty = createAndStartJetty(leader);leaderClient.close();- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());-+ leaderClient = createNewSolrClient(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);+for (int i = 0; i < nDocs; i++)index(leaderClient, "id", i, "name", "name = " + i);-+leaderClient.commit();-+// now we restart to test what happens with no activity before the follower// tries to// replicateleaderJetty.stop();leaderJetty.start();-+// leaderClient = createNewSolrClient(leaderJetty.getLocalPort());-+NamedList<Object> leaderQueryRsp = rQuery(nDocs, "*:*", leaderClient);SolrDocumentList leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response");assertEquals(nDocs, leaderQueryResult.getNumFound());-+follower.setTestPort(leaderJetty.getLocalPort());follower.copyConfigFile(follower.getSolrConfigFile(), "solrconfig.xml");-+// start followerfollowerJetty = createAndStartJetty(follower);followerClient.close();- followerClient = createNewSolrClient(followerJetty.getLocalPort());-+ followerClient = createNewSolrClient(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);+// get docs from follower and check if number is equal to leaderNamedList<Object> followerQueryRsp = rQuery(nDocs, "*:*", followerClient);SolrDocumentList followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response");assertEquals(nDocs, followerQueryResult.getNumFound());-+// compare resultsString cmp = BaseDistributedSearchTestCase.compare(leaderQueryResult,followerQueryResult, 0, null);assertEquals(null, cmp);-+} finally {resetFactory();}@@ -1363,7 +1261,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {@Testpublic void doTestReplicateAfterCoreReload() throws Exception {int docs = TEST_NIGHTLY ? 200000 : 10;-+//stop followerfollowerJetty.stop();@@ -1376,7 +1274,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {leaderJetty = createAndStartJetty(leader);leaderClient.close();- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());+ leaderClient = createNewSolrClient(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);leaderClient.deleteByQuery("*:*");for (int i = 0; i < docs; i++)@@ -1387,44 +1285,44 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {NamedList<Object> leaderQueryRsp = rQuery(docs, "*:*", leaderClient);SolrDocumentList leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response");assertEquals(docs, leaderQueryResult.getNumFound());-+follower.setTestPort(leaderJetty.getLocalPort());follower.copyConfigFile(follower.getSolrConfigFile(), "solrconfig.xml");//start followerfollowerJetty = createAndStartJetty(follower);followerClient.close();- followerClient = createNewSolrClient(followerJetty.getLocalPort());-+ followerClient = createNewSolrClient(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);+//get docs from follower and check if number is equal to leaderNamedList<Object> followerQueryRsp = rQuery(docs, "*:*", followerClient);SolrDocumentList followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response");assertEquals(docs, followerQueryResult.getNumFound());-+//compare resultsString cmp = BaseDistributedSearchTestCase.compare(leaderQueryResult, followerQueryResult, 0, null);assertEquals(null, cmp);-+Object version = getIndexVersion(leaderClient).get("indexversion");-+reloadCore(leaderClient, "collection1");-+assertEquals(version, getIndexVersion(leaderClient).get("indexversion"));-+index(leaderClient, "id", docs + 10, "name", "name = 1");index(leaderClient, "id", docs + 20, "name", "name = 2");leaderClient.commit();-+NamedList<Object> resp = rQuery(docs + 2, "*:*", leaderClient);leaderQueryResult = (SolrDocumentList) resp.get("response");assertEquals(docs + 2, leaderQueryResult.getNumFound());-+//get docs from follower and check if number is equal to leaderfollowerQueryRsp = rQuery(docs + 2, "*:*", followerClient);followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response");assertEquals(docs + 2, followerQueryResult.getNumFound());-+}@Test@@ -1474,7 +1372,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {leaderJetty = createAndStartJetty(leader);leaderClient.close();- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());+ leaderClient = createNewSolrClient(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);follower.setTestPort(leaderJetty.getLocalPort());follower.copyConfigFile(follower.getSolrConfigFile(), "solrconfig.xml");@@ -1482,12 +1380,12 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {followerJetty.stop();followerJetty = createAndStartJetty(follower);followerClient.close();- followerClient = createNewSolrClient(followerJetty.getLocalPort());+ followerClient = createNewSolrClient(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);followerClient.deleteByQuery("*:*");followerClient.commit();rQuery(0, "*:*", followerClient); // sanity check w/retry-+// record collection1's start time on followerfinal Date followerStartTime = watchCoreStartAt(followerClient, 30*1000, null);@@ -1506,7 +1404,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {NamedList<Object> followerQueryRsp2 = rQuery(1, "id:2000", followerClient);SolrDocumentList followerQueryResult2 = (SolrDocumentList) followerQueryRsp2.get("response");assertEquals(1, followerQueryResult2.getNumFound());-+checkForSingleIndex(leaderJetty);checkForSingleIndex(followerJetty, true);}@@ -1528,7 +1426,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {useFactory(null);leaderJetty = createAndStartJetty(leader);leaderClient.close();- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());+ leaderClient = createNewSolrClient(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);//index docsfinal int totalDocs = TestUtil.nextInt(random(), 17, 53);@@ -1554,18 +1452,18 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {//Start again and replicate the datauseFactory(null);leaderJetty = createAndStartJetty(leader);- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());+ leaderClient = createNewSolrClient(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);//start followerfollower.setTestPort(leaderJetty.getLocalPort());follower.copyConfigFile(CONF_DIR + "solrconfig-follower1.xml", "solrconfig.xml");followerJetty = createAndStartJetty(follower);followerClient.close();- followerClient = createNewSolrClient(followerJetty.getLocalPort());+ followerClient = createNewSolrClient(buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);long startTime = System.nanoTime();- pullFromLeaderToFollower();+ pullFromTo(leaderJetty, followerJetty);//Add a few more docs in the leader. Just to make sure that we are replicating the correct index point//These extra docs should not get replicated@@ -1595,7 +1493,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {for (String param : params) {for (String filename : illegalFilenames) {expectThrows(Exception.class, () ->- invokeReplicationCommand(leaderJetty.getLocalPort(), "filecontent&" + param + "=" + filename));+ invokeReplicationCommand(buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME, "filecontent&" + param + "=" + filename));}}}@@ -1678,9 +1576,9 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {assertTrue(dirName + " doesn't exist in expected location for backup " + backupName,new File(backupDir, dirName).exists());}-+index(leaderClient, "id", "1", "name", "foo");-+{ // second backup w/uncommited docfinal String backupName = "empty_backup2";final GenericSolrRequest req = new GenericSolrRequest@@ -1690,7 +1588,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {"name", backupName));final TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);final SimpleSolrResponse rsp = req.process(leaderClient);-+final String dirName = backupStatus.waitForBackupSuccess(backupName, timeout);assertEquals("Did not get expected dir name for backup, did API change?","snapshot.empty_backup2", dirName);@@ -1707,7 +1605,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {}}}-+public void testGetBoolWithBackwardCompatibility() {assertTrue(ReplicationHandler.getBoolWithBackwardCompatibility(params(), "foo", "bar", true));assertFalse(ReplicationHandler.getBoolWithBackwardCompatibility(params(), "foo", "bar", false));@@ -1715,7 +1613,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {assertTrue(ReplicationHandler.getBoolWithBackwardCompatibility(params("bar", "true"), "foo", "bar", false));assertTrue(ReplicationHandler.getBoolWithBackwardCompatibility(params("foo", "true", "bar", "false"), "foo", "bar", false));}-+public void testGetObjectWithBackwardCompatibility() {assertEquals("aaa", ReplicationHandler.getObjectWithBackwardCompatibility(params(), "foo", "bar", "aaa"));assertEquals("bbb", ReplicationHandler.getObjectWithBackwardCompatibility(params("foo", "bbb"), "foo", "bar", "aaa"));@@ -1723,7 +1621,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {assertEquals("bbb", ReplicationHandler.getObjectWithBackwardCompatibility(params("foo", "bbb", "bar", "aaa"), "foo", "bar", "aaa"));assertNull(ReplicationHandler.getObjectWithBackwardCompatibility(params(), "foo", "bar", null));}-+public void testGetObjectWithBackwardCompatibilityFromNL() {NamedList<Object> nl = new NamedList<>();assertNull(ReplicationHandler.getObjectWithBackwardCompatibility(nl, "foo", "bar"));@@ -1732,8 +1630,8 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {nl.add("foo", "aaa");assertEquals("aaa", ReplicationHandler.getObjectWithBackwardCompatibility(nl, "foo", "bar"));}--++private class AddExtraDocs implements Runnable {SolrClient leaderClient;@@ -1748,7 +1646,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {final int totalDocs = TestUtil.nextInt(random(), 1, 10);for (int i = 0; i < totalDocs; i++) {try {- index(leaderClient, "id", i + startId, "name", TestUtil.randomSimpleString(random(), 1000 , 5000));+ index(leaderClient, "id", i + startId, "name", TestUtil.randomSimpleString(random(), 1000, 5000));} catch (Exception e) {//Do nothing. Wasn't able to add doc.}@@ -1760,23 +1658,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {}}}-- /**- * character copy of file using UTF-8. If port is non-null, will be substituted any time "TEST_PORT" is found.- */- private static void copyFile(File src, File dst, Integer port, boolean internalCompression) throws IOException {- try (BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(src), StandardCharsets.UTF_8));- Writer out = new OutputStreamWriter(new FileOutputStream(dst), StandardCharsets.UTF_8)) {-- for (String line = in.readLine(); null != line; line = in.readLine()) {- if (null != port) {- line = line.replace("TEST_PORT", port.toString());- }- line = line.replace("COMPRESSION", internalCompression ? "internal" : "false");- out.write(line);- }- }- }private UpdateResponse emptyUpdate(SolrClient client, String... params)throws SolrServerException, IOException {@@ -1787,10 +1668,10 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {}/**- * Polls the SolrCore stats using the specified client until the "startTime"- * time for collection is after the specified "min". Will loop for+ * Polls the SolrCore stats using the specified client until the "startTime"+ * time for collection is after the specified "min". Will loop for* at most "timeout" milliseconds before throwing an assertion failure.- *+ ** @param client The SolrClient to poll* @param timeout the max milliseconds to continue polling for* @param min the startTime value must exceed this value before the method will return, if null this method will return the first startTime value encountered.@@ -1839,86 +1720,8 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {assertNotNull("Expected replication response to have 'status' field", response.get("status"));assertEquals("OK", response.get("status"));}-- private static String buildUrl(int port) {- return buildUrl(port, context);- }-- static class SolrInstance {-- private String name;- private Integer testPort;- private File homeDir;- private File confDir;- private File dataDir;-- /**- * @param homeDir Base directory to build solr configuration and index in- * @param name used to pick which- * "solrconfig-${name}.xml" file gets copied- * to solrconfig.xml in new conf dir.- * @param testPort if not null, used as a replacement for- * TEST_PORT in the cloned config files.- */- public SolrInstance(File homeDir, String name, Integer testPort) {- this.homeDir = homeDir;- this.name = name;- this.testPort = testPort;- }-- public String getHomeDir() {- return homeDir.toString();- }-- public String getSchemaFile() {- return CONF_DIR + "schema-replication1.xml";- }-- public String getConfDir() {- return confDir.toString();- }-- public String getDataDir() {- return dataDir.getAbsolutePath();- }-- public String getSolrConfigFile() {- return CONF_DIR + "solrconfig-"+name+".xml";- }-- /** If it needs to change */- public void setTestPort(Integer testPort) {- this.testPort = testPort;- }-- public void setUp() throws Exception {- System.setProperty("solr.test.sys.prop1", "propone");- System.setProperty("solr.test.sys.prop2", "proptwo");-- Properties props = new Properties();- props.setProperty("name", "collection1");-- writeCoreProperties(homeDir.toPath().resolve("collection1"), props, "TestReplicationHandler");-- dataDir = new File(homeDir + "/collection1", "data");- confDir = new File(homeDir + "/collection1", "conf");-- homeDir.mkdirs();- dataDir.mkdirs();- confDir.mkdirs();-- copyConfigFile(getSolrConfigFile(), "solrconfig.xml");- copyConfigFile(getSchemaFile(), "schema.xml");- copyConfigFile(CONF_DIR + "solrconfig.snippet.randomindexconfig.xml",- "solrconfig.snippet.randomindexconfig.xml");- }-- public void copyConfigFile(String srcFile, String destFile)- throws IOException {- copyFile(getFile(srcFile),- new File(confDir, destFile),- testPort, random().nextBoolean());- }+ public static String buildUrl(int port) {+ return buildUrl(port, context);}-}+}\ No newline at end of filediff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.javaindex 48d3f6d1e27..fe641b3e1ad 100644--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java@@ -58,7 +58,7 @@ import org.slf4j.LoggerFactory;public class TestReplicationHandlerBackup extends SolrJettyTestBase {JettySolrRunner leaderJetty;- TestReplicationHandler.SolrInstance leader = null;+ ReplicationTestHelper.SolrInstance leader = null;SolrClient leaderClient;private static final String CONF_DIR = "solr" + File.separator + "collection1" + File.separator + "conf"@@ -71,7 +71,7 @@ public class TestReplicationHandlerBackup extends SolrJettyTestBase {private static long docsSeed; // see indexDocs()private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- private static JettySolrRunner createAndStartJetty(TestReplicationHandler.SolrInstance instance) throws Exception {+ private static JettySolrRunner createAndStartJetty(ReplicationTestHelper.SolrInstance instance) throws Exception {FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(instance.getHomeDir(), "solr.xml"));Properties nodeProperties = new Properties();nodeProperties.setProperty("solr.data.dir", instance.getDataDir());@@ -104,7 +104,7 @@ public class TestReplicationHandlerBackup extends SolrJettyTestBase {addNumberToKeepInRequest = false;backupKeepParamName = ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_INIT_PARAM;}- leader = new TestReplicationHandler.SolrInstance(createTempDir("solr-instance").toFile(), "leader", null);+ leader = new ReplicationTestHelper.SolrInstance(createTempDir("solr-instance").toFile(), "leader", null);leader.setUp();leader.copyConfigFile(CONF_DIR + configFile, "solrconfig.xml");diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerDiskOverFlow.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerDiskOverFlow.javaindex 6d2beb41f7a..7ae3c991926 100644--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerDiskOverFlow.java+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerDiskOverFlow.java@@ -47,8 +47,8 @@ import org.slf4j.LoggerFactory;import static org.apache.solr.handler.ReplicationHandler.CMD_FETCH_INDEX;import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST;import static org.apache.solr.handler.TestReplicationHandler.createAndStartJetty;-import static org.apache.solr.handler.TestReplicationHandler.createNewSolrClient;-import static org.apache.solr.handler.TestReplicationHandler.invokeReplicationCommand;+import static org.apache.solr.handler.ReplicationTestHelper.invokeReplicationCommand;+@LogLevel("org.apache.solr.handler.IndexFetcher=DEBUG")@SolrTestCaseJ4.SuppressSSL@@ -61,7 +61,7 @@ public class TestReplicationHandlerDiskOverFlow extends SolrTestCaseJ4 {JettySolrRunner leaderJetty, followerJetty;SolrClient leaderClient, followerClient;- TestReplicationHandler.SolrInstance leader = null, follower = null;+ ReplicationTestHelper.SolrInstance leader = null, follower = null;static String context = "/solr";@@ -74,16 +74,16 @@ public class TestReplicationHandlerDiskOverFlow extends SolrTestCaseJ4 {System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");String factory = random().nextInt(100) < 75 ? "solr.NRTCachingDirectoryFactory" : "solr.StandardDirectoryFactory"; // test the default most of the timeSystem.setProperty("solr.directoryFactory", factory);- leader = new TestReplicationHandler.SolrInstance(createTempDir("solr-instance").toFile(), "leader", null);+ leader = new ReplicationTestHelper.SolrInstance(createTempDir("solr-instance").toFile(), "leader", null);leader.setUp();leaderJetty = createAndStartJetty(leader);- leaderClient = createNewSolrClient(leaderJetty.getLocalPort());+ leaderClient = ReplicationTestHelper.createNewSolrClient( TestReplicationHandler.buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);System.setProperty(TEST_URL_ALLOW_LIST, leaderJetty.getBaseUrl().toString());- follower = new TestReplicationHandler.SolrInstance(createTempDir("solr-instance").toFile(), "follower", leaderJetty.getLocalPort());+ follower = new ReplicationTestHelper.SolrInstance(createTempDir("solr-instance").toFile(), "follower", leaderJetty.getLocalPort());follower.setUp();followerJetty = createAndStartJetty(follower);- followerClient = createNewSolrClient(followerJetty.getLocalPort());+ followerClient = ReplicationTestHelper.createNewSolrClient( TestReplicationHandler.buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME);System.setProperty("solr.indexfetcher.sotimeout2", "45000");}@@ -118,7 +118,7 @@ public class TestReplicationHandlerDiskOverFlow extends SolrTestCaseJ4 {@Testpublic void testDiskOverFlow() throws Exception {- invokeReplicationCommand(followerJetty.getLocalPort(), "disablepoll");+ invokeReplicationCommand(TestReplicationHandler.buildUrl(followerJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME, "disablepoll");//index docslog.info("Indexing to LEADER");int docsInLeader = 1000;@@ -217,7 +217,7 @@ public class TestReplicationHandlerDiskOverFlow extends SolrTestCaseJ4 {@SuppressWarnings("unchecked")private long indexDocs(SolrClient client, int totalDocs, int start) throws Exception {for (int i = 0; i < totalDocs; i++)- TestReplicationHandler.index(client, "id", i + start, "name", TestUtil.randomSimpleString(random(), 1000, 5000));+ ReplicationTestHelper.index(client, "id", i + start, "name", TestUtil.randomSimpleString(random(), 1000, 5000));client.commit(true, true);QueryResponse response = client.query(new SolrQuery().add("qt", "/replication")diff --git a/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.javaindex 75e44750839..4edee45f1bd 100644--- a/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java+++ b/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java@@ -50,7 +50,7 @@ import org.junit.Test;public class TestRestoreCore extends SolrJettyTestBase {JettySolrRunner leaderJetty;- TestReplicationHandler.SolrInstance leader = null;+ ReplicationTestHelper.SolrInstance leader = null;SolrClient leaderClient;private static final String CONF_DIR = "solr" + File.separator + DEFAULT_TEST_CORENAME + File.separator + "conf"@@ -59,7 +59,7 @@ public class TestRestoreCore extends SolrJettyTestBase {private static String context = "/solr";private static long docsSeed; // see indexDocs()- private static JettySolrRunner createAndStartJetty(TestReplicationHandler.SolrInstance instance) throws Exception {+ private static JettySolrRunner createAndStartJetty(ReplicationTestHelper.SolrInstance instance) throws Exception {FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(instance.getHomeDir(), "solr.xml"));Properties nodeProperties = new Properties();nodeProperties.setProperty("solr.data.dir", instance.getDataDir());@@ -87,7 +87,7 @@ public class TestRestoreCore extends SolrJettyTestBase {super.setUp();String configFile = "solrconfig-leader.xml";- leader = new TestReplicationHandler.SolrInstance(createTempDir("solr-instance").toFile(), "leader", null);+ leader = new ReplicationTestHelper.SolrInstance(createTempDir("solr-instance").toFile(), "leader", null);leader.setUp();leader.copyConfigFile(CONF_DIR + configFile, "solrconfig.xml");diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/HealthCheckRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/HealthCheckRequest.javaindex f3aeb402629..5b2dc0e457f 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/HealthCheckRequest.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/HealthCheckRequest.java@@ -21,11 +21,17 @@ import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.impl.HttpSolrClient;import org.apache.solr.client.solrj.response.HealthCheckResponse;+import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;+import java.util.OptionalInt;+import static org.apache.solr.common.params.CommonParams.HEALTH_CHECK_HANDLER_PATH;public class HealthCheckRequest extends SolrRequest<HealthCheckResponse> {+ public static final String PARAM_MAX_GENERATION_LAG = "maxGenerationLag";++ private OptionalInt maxLagAllowed = OptionalInt.empty();public HealthCheckRequest() {this(METHOD.GET, HEALTH_CHECK_HANDLER_PATH);@@ -35,9 +41,18 @@ public class HealthCheckRequest extends SolrRequest<HealthCheckResponse> {super(m, path);}+ public void setMaxGenerationLag(int maxLagAllowed) {+ this.maxLagAllowed = OptionalInt.of(maxLagAllowed);+ }+@Overridepublic SolrParams getParams() {- return null;+ if (maxLagAllowed.isPresent()) {+ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set(PARAM_MAX_GENERATION_LAG, maxLagAllowed.getAsInt());+ return params;+ }+ return null;}@Override