Apache Solr open-source search software
SOLR-17453: Leverage waitForState() instead of busy waiting (#2737)
Pierre Salagnacb657ec0c2176
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 3a03a54985a..907827a6fd0 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -143,7 +143,7 @@ Improvementswhen PKI is used between nodes. (Jason Gerlowski)* SOLR-17383: Resolved overlapping arguments in the Solr CLI. Removed duplicative but differing arguments,- consolidated use of short form arguments -v to not have differing meanings based on tool. Provide deprecation warning+ consolidated use of short form arguments -v to not have differing meanings based on tool. Provide deprecation warningin command line when deprecated arguments are used. (Eric Pugh, Christos Malliaridis)* SOLR-17256: Deprecate SolrRequest `setBasePath` and `getBasePath` methods. SolrJ users wishing to temporarily@@ -181,6 +181,8 @@ Optimizations* SOLR-16503: Switched from HTTP1 to HTTP2 in SolrClientCloudManager by replacing CloudLegacySolrClient with CloudHttp2SolrClient. (Sanjay Dutt, David Smiley)+* SOLR-17453: Leverage waitForState() instead of busy waiting in CREATE, MIGRATE, REINDEXCOLLECTION, MOVEREPLICA commands, and in some tests. (Pierre Salagnac)+Bug Fixes---------------------* SOLR-12429: Uploading a configset with a symbolic link produces a IOException. Now a error message to user generated instead. (Eric Pugh)diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.javaindex ccfd4a1e5e2..5592303580a 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java@@ -36,9 +36,11 @@ import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.NoSuchElementException;+import java.util.Objects;import java.util.Properties;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import org.apache.solr.client.solrj.cloud.AlreadyExistsException;import org.apache.solr.client.solrj.cloud.BadVersionException;import org.apache.solr.client.solrj.cloud.DelegatingCloudManager;@@ -221,24 +223,19 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {}// wait for a while until we see the collection- TimeOut waitUntil =- new TimeOut(30, TimeUnit.SECONDS, ccc.getSolrCloudManager().getTimeSource());- boolean created = false;- while (!waitUntil.hasTimedOut()) {- waitUntil.sleep(100);- created = ccc.getSolrCloudManager().getClusterState().hasCollection(collectionName);- if (created) break;- }- if (!created) {+ try {+ newColl =+ zkStateReader.waitForState(collectionName, 30, TimeUnit.SECONDS, Objects::nonNull);+ } catch (TimeoutException e) {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,- "Could not fully create collection: " + collectionName);+ "Could not fully create collection: " + collectionName,+ e);}// refresh cluster state (value read below comes from Zookeeper watch firing following the// update done previously, be it by Overseer or by this thread when updates are distributed)clusterState = ccc.getSolrCloudManager().getClusterState();- newColl = clusterState.getCollection(collectionName);}final List<ReplicaPosition> replicaPositions;diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.javaindex cdcb3c27449..95d1eb2ac11 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java@@ -31,6 +31,7 @@ import java.util.Collection;import java.util.HashMap;import java.util.Map;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import org.apache.solr.client.solrj.request.CoreAdminRequest;import org.apache.solr.cloud.DistributedClusterStateUpdater;import org.apache.solr.cloud.Overseer;@@ -52,7 +53,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;import org.apache.solr.handler.component.ShardHandler;-import org.apache.solr.util.TimeOut;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -272,27 +272,26 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {// wait for a while until we see the new rulelog.info("Waiting to see routing rule updated in clusterstate");- TimeOut waitUntil =- new TimeOut(60, TimeUnit.SECONDS, ccc.getSolrCloudManager().getTimeSource());- boolean added = false;- while (!waitUntil.hasTimedOut()) {- waitUntil.sleep(100);- sourceCollection = zkStateReader.getClusterState().getCollection(sourceCollection.getName());- sourceSlice = sourceCollection.getSlice(sourceSlice.getName());- Map<String, RoutingRule> rules = sourceSlice.getRoutingRules();- if (rules != null) {- RoutingRule rule = rules.get(sourceRouter.getRouteKeyNoSuffix(splitKey) + "!");- if (rule != null && rule.getRouteRanges().contains(splitRange)) {- added = true;- break;- }- }- }- if (!added) {++ try {+ sourceCollection =+ zkStateReader.waitForState(+ sourceCollection.getName(),+ 60,+ TimeUnit.SECONDS,+ c -> {+ Slice s = c.getSlice(sourceSlice.getName());+ Map<String, RoutingRule> rules = s.getRoutingRules();+ if (rules != null) {+ RoutingRule rule = rules.get(sourceRouter.getRouteKeyNoSuffix(splitKey) + "!");+ return rule != null && rule.getRouteRanges().contains(splitRange);+ }+ return false;+ });+ } catch (TimeoutException e) {throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "Could not add routing rule: " + m);+ SolrException.ErrorCode.SERVER_ERROR, "Could not add routing rule: " + m, e);}-log.info("Routing rule added successfully");// Create temp core on source sharddiff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.javaindex cd5097993b4..48f065f537e 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java@@ -33,6 +33,7 @@ import java.util.Collections;import java.util.List;import java.util.Locale;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import org.apache.solr.cloud.ActiveReplicaWatcher;import org.apache.solr.common.SolrCloseableLatch;import org.apache.solr.common.SolrException;@@ -46,7 +47,6 @@ import org.apache.solr.common.params.CollectionParams;import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;-import org.apache.solr.util.TimeOut;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -161,9 +161,7 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {dataDir.toString(),targetNode,async,- coll,replica,- slice,timeout,waitForFinalState);} else {@@ -187,9 +185,7 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {String dataDir,String targetNode,String async,- DocCollection coll,Replica replica,- Slice slice,int timeout,boolean waitForFinalState)throws Exception {@@ -198,8 +194,8 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {skipCreateReplicaInClusterState = "false";ZkNodeProps removeReplicasProps =new ZkNodeProps(- COLLECTION_PROP, coll.getName(),- SHARD_ID_PROP, slice.getName(),+ COLLECTION_PROP, replica.getCollection(),+ SHARD_ID_PROP, replica.getShard(),REPLICA_PROP, replica.getName());removeReplicasProps.getProperties().put(CoreAdminParams.DELETE_DATA_DIR, false);removeReplicasProps.getProperties().put(CoreAdminParams.DELETE_INDEX, false);@@ -217,8 +213,8 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {String.format(Locale.ROOT,"Failed to cleanup replica collection=%s shard=%s name=%s, failure=%s",- coll.getName(),- slice.getName(),+ replica.getCollection(),+ replica.getShard(),replica.getName(),deleteResult.get("failure"));log.warn(errorString);@@ -226,17 +222,14 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {return;}- TimeOut timeOut =- new TimeOut(20L, TimeUnit.SECONDS, ccc.getSolrCloudManager().getTimeSource());- while (!timeOut.hasTimedOut()) {- coll = ccc.getZkStateReader().getClusterState().getCollection(coll.getName());- if (coll.getReplica(replica.getName()) != null) {- timeOut.sleep(100);- } else {- break;- }- }- if (timeOut.hasTimedOut()) {+ try {+ ccc.getZkStateReader()+ .waitForState(+ replica.getCollection(),+ 20L,+ TimeUnit.SECONDS,+ c -> c.getReplica(replica.getName()) != null);+ } catch (TimeoutException e) {results.add("failure", "Still see deleted replica in clusterstate!");return;}@@ -246,9 +239,9 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {ZkNodeProps addReplicasProps =new ZkNodeProps(COLLECTION_PROP,- coll.getName(),+ replica.getCollection(),SHARD_ID_PROP,- slice.getName(),+ replica.getShard(),CoreAdminParams.NODE,targetNode,CoreAdminParams.CORE_NODE_NAME,@@ -277,8 +270,8 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {String.format(Locale.ROOT,"Failed to create replica for collection=%s shard=%s" + " on node=%s, failure=%s",- coll.getName(),- slice.getName(),+ replica.getCollection(),+ replica.getShard(),targetNode,addResult.get("failure"));results.add("failure", errorString);@@ -302,8 +295,8 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {String.format(Locale.ROOT,"Failed to create replica for collection=%s shard=%s" + " on node=%s, failure=%s",- coll.getName(),- slice.getName(),+ replica.getCollection(),+ replica.getShard(),targetNode,addResult.get("failure"));log.warn(errorString);diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.javaindex 91299b3c259..15324ec61aa 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java@@ -25,8 +25,10 @@ import java.util.HashMap;import java.util.List;import java.util.Locale;import java.util.Map;+import java.util.Objects;import java.util.TreeMap;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import java.util.concurrent.atomic.AtomicInteger;import java.util.function.Function;import java.util.stream.Collectors;@@ -360,22 +362,17 @@ public class ReindexCollectionCmd implements CollApiCmds.CollectionApiCommand {CollectionHandlingUtils.checkResults("creating checkpoint collection " + chkCollection, cmdResults, true);// wait for a while until we see both collections- TimeOut waitUntil =- new TimeOut(30, TimeUnit.SECONDS, ccc.getSolrCloudManager().getTimeSource());- boolean created = false;- while (!waitUntil.hasTimedOut()) {- waitUntil.sleep(100);- // this also refreshes our local var clusterState- clusterState = ccc.getSolrCloudManager().getClusterState();- created =- clusterState.hasCollection(targetCollection)- && clusterState.hasCollection(chkCollection);- if (created) break;- }- if (!created) {+ try {+ for (String col : List.of(targetCollection, chkCollection)) {+ ccc.getZkStateReader().waitForState(col, 30, TimeUnit.SECONDS, Objects::nonNull);+ }+ } catch (TimeoutException e) {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Could not fully create temporary collection(s)");}++ clusterState = ccc.getSolrCloudManager().getClusterState();+if (maybeAbort(collection)) {aborted = true;return;diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.javaindex cc645476aef..99291fc6ad0 100644--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java@@ -53,6 +53,7 @@ import java.util.Objects;import java.util.Random;import java.util.Set;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import java.util.function.Supplier;import javax.servlet.http.HttpServletRequest;import javax.servlet.http.HttpServletResponse;@@ -96,7 +97,6 @@ import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.common.util.StrUtils;import org.apache.solr.common.util.SuppressForbidden;-import org.apache.solr.common.util.TimeSource;import org.apache.solr.common.util.Utils;import org.apache.solr.common.util.ValidatingJsonMap;import org.apache.solr.core.CoreContainer;@@ -125,7 +125,6 @@ import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;import org.apache.solr.servlet.cache.Method;import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;import org.apache.solr.util.RTimerTree;-import org.apache.solr.util.TimeOut;import org.apache.solr.util.tracing.TraceUtils;import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;@@ -388,18 +387,16 @@ public class HttpSolrCall {+ " collection: "+ Utils.toJSONString(rsp.getValues()));}- TimeOut timeOut = new TimeOut(3, TimeUnit.SECONDS, TimeSource.NANO_TIME);- for (; ; ) {- if (cores.getZkController().getClusterState().getCollectionOrNull(SYSTEM_COLL) != null) {- break;- } else {- if (timeOut.hasTimedOut()) {- throw new SolrException(- ErrorCode.SERVER_ERROR,- "Could not find " + SYSTEM_COLL + " collection even after 3 seconds");- }- timeOut.sleep(50);- }++ try {+ cores+ .getZkController()+ .getZkStateReader()+ .waitForState(SYSTEM_COLL, 3, TimeUnit.SECONDS, Objects::nonNull);+ } catch (TimeoutException e) {+ throw new SolrException(+ ErrorCode.SERVER_ERROR,+ "Could not find " + SYSTEM_COLL + " collection even after 3 seconds");}action = RETRY;diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.javaindex ddb72aad118..6652009774f 100644--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java@@ -343,7 +343,7 @@ public class ChaosMonkeyNothingIsSafeWithPullReplicasTest extends AbstractFullDiList<Integer> numShardsNumReplicas = new ArrayList<>(2);numShardsNumReplicas.add(1);numShardsNumReplicas.add(1 + getPullReplicaCount());- checkForCollection("testcollection", numShardsNumReplicas, null);+ checkForCollection("testcollection", numShardsNumReplicas);testSuccessful = true;} finally {diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.javaindex 764035dc043..b097ebea968 100644--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java@@ -255,7 +255,7 @@ public class ChaosMonkeySafeLeaderWithPullReplicasTest extends AbstractFullDistrList<Integer> numShardsNumReplicas = new ArrayList<>(2);numShardsNumReplicas.add(1);numShardsNumReplicas.add(1 + getPullReplicaCount());- checkForCollection("testcollection", numShardsNumReplicas, null);+ checkForCollection("testcollection", numShardsNumReplicas);}private void tryDelete() throws Exception {diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.javaindex 686a0a2554a..16242bcc5eb 100644--- a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java@@ -488,11 +488,13 @@ public class DeleteReplicaTest extends SolrCloudTestCase {}private void waitForNodeLeave(String lostNodeName) throws InterruptedException {- ZkStateReader reader = cluster.getZkStateReader();- TimeOut timeOut = new TimeOut(20, TimeUnit.SECONDS, TimeSource.NANO_TIME);- while (reader.getClusterState().getLiveNodes().contains(lostNodeName)) {- Thread.sleep(100);- if (timeOut.hasTimedOut()) fail("Wait for " + lostNodeName + " to leave failed!");++ try {+ cluster+ .getZkStateReader()+ .waitForLiveNodes(20, TimeUnit.SECONDS, (o, n) -> !n.contains(lostNodeName));+ } catch (TimeoutException e) {+ fail("Wait for " + lostNodeName + " to leave failed!");}}diff --git a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.javaindex 2f3f62ad763..0d4f1c2b722 100644--- a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java@@ -30,6 +30,7 @@ import java.util.List;import java.util.Map;import java.util.Set;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.solr.JSONTestUtil;import org.apache.solr.SolrTestCaseJ4.SuppressSSL;@@ -48,13 +49,11 @@ import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.ZkCoreNodeProps;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.NamedList;-import org.apache.solr.common.util.TimeSource;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrCore;import org.apache.solr.embedded.JettySolrRunner;import org.apache.solr.util.RTimer;import org.apache.solr.util.TestInjection;-import org.apache.solr.util.TimeOut;import org.apache.zookeeper.KeeperException;import org.junit.BeforeClass;import org.junit.Test;@@ -291,27 +290,23 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {protected void waitForState(String collection, String replicaName, Replica.State state, long ms)throws KeeperException, InterruptedException {- TimeOut timeOut = new TimeOut(ms, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);- Replica.State replicaState = Replica.State.ACTIVE;- while (!timeOut.hasTimedOut()) {- ZkStateReader zkr = ZkStateReader.from(cloudClient);- zkr.forceUpdateCollection(collection); // force the state to be fresh- ClusterState cs = zkr.getClusterState();- Collection<Slice> slices = cs.getCollection(collection).getActiveSlices();- Slice slice = slices.iterator().next();- Replica partitionedReplica = slice.getReplica(replicaName);- replicaState = partitionedReplica.getState();- if (replicaState == state) return;+ ZkStateReader zkr = ZkStateReader.from(cloudClient);++ try {+ zkr.waitForState(+ collection,+ ms,+ TimeUnit.MILLISECONDS,+ c -> {+ Collection<Slice> slices = c.getActiveSlices();+ Slice slice = slices.iterator().next();+ Replica partitionedReplica = slice.getReplica(replicaName);+ Replica.State replicaState = partitionedReplica.getState();+ return replicaState == state;+ });+ } catch (TimeoutException e) {+ fail("Timeout waiting for state " + state + " of replica " + replicaName);}- assertEquals(- "Timeout waiting for state "- + state- + " of replica "- + replicaName- + ", current state "- + replicaState,- state,- replicaState);}protected void testRf3() throws Exception {diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderFailureAfterFreshStartTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderFailureAfterFreshStartTest.javaindex 5b7ff3f0428..1bc87e4f4e1 100644--- a/solr/core/src/test/org/apache/solr/cloud/LeaderFailureAfterFreshStartTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/LeaderFailureAfterFreshStartTest.java@@ -39,8 +39,6 @@ import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.util.TimeSource;-import org.apache.solr.util.TimeOut;import org.junit.Test;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -154,11 +152,7 @@ public class LeaderFailureAfterFreshStartTest extends AbstractFullDistribZkTestB// shutdown the original leaderlog.info("Now shutting down initial leader");forceNodeFailures(singletonList(initialLeaderJetty));- waitForNewLeader(- cloudClient,- "shard1",- (Replica) initialLeaderJetty.client.info,- new TimeOut(15, TimeUnit.SECONDS, TimeSource.NANO_TIME));+ waitForNewLeader(cloudClient, "shard1", (Replica) initialLeaderJetty.client.info);waitTillNodesActive();log.info("Updating mappings from zk");updateMappingsFromZk(jettys, clients, true);diff --git a/solr/core/src/test/org/apache/solr/cloud/PeerSyncReplicationTest.java b/solr/core/src/test/org/apache/solr/cloud/PeerSyncReplicationTest.javaindex 7042c3018e2..d9e501e69b6 100644--- a/solr/core/src/test/org/apache/solr/cloud/PeerSyncReplicationTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/PeerSyncReplicationTest.java@@ -44,10 +44,8 @@ import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.util.TimeSource;import org.apache.solr.core.CoreContainer;import org.apache.solr.metrics.SolrMetricManager;-import org.apache.solr.util.TimeOut;import org.junit.Test;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -154,11 +152,7 @@ public class PeerSyncReplicationTest extends AbstractFullDistribZkTestBase {log.info("Now shutting down initial leader");forceNodeFailures(singletonList(initialLeaderJetty));log.info("Updating mappings from zk");- waitForNewLeader(- cloudClient,- "shard1",- (Replica) initialLeaderJetty.client.info,- new TimeOut(15, TimeUnit.SECONDS, TimeSource.NANO_TIME));+ waitForNewLeader(cloudClient, "shard1", (Replica) initialLeaderJetty.client.info);updateMappingsFromZk(jettys, clients, true);assertEquals("PeerSynced node did not become leader",diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.javaindex f5cf82c701e..ec0564c249d 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java@@ -749,20 +749,13 @@ public class TestPullReplica extends SolrCloudTestCase {}}- static void waitForDeletion(String collection) throws InterruptedException, KeeperException {- TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);- while (cluster.getSolrClient().getClusterState().hasCollection(collection)) {- log.info("Collection not yet deleted");- try {- Thread.sleep(100);- if (t.hasTimedOut()) {- fail("Timed out waiting for collection " + collection + " to be deleted.");- }- cluster.getZkStateReader().forceUpdateCollection(collection);- } catch (SolrException e) {- return;- }- }+ static void waitForDeletion(String collection) {+ waitForState(+ "Waiting for collection " + collection + " to be deleted",+ collection,+ (n, c) -> c == null,+ 10,+ TimeUnit.SECONDS);}private DocCollection assertNumberOfReplicas(diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.javaindex f4af183434c..0d68f2f1993 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java@@ -120,7 +120,7 @@ public class TestPullReplicaErrorHandling extends SolrCloudTestCase {log.info("tearDown deleting collection");CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());log.info("Collection deleted");- waitForDeletion(collectionName);+ TestPullReplica.waitForDeletion(collectionName);}collectionName = null;super.tearDown();@@ -347,22 +347,6 @@ public class TestPullReplicaErrorHandling extends SolrCloudTestCase {return proxy;}- private void waitForDeletion(String collection) throws InterruptedException, KeeperException {- TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);- while (cluster.getSolrClient().getClusterState().hasCollection(collection)) {- log.info("Collection not yet deleted");- try {- Thread.sleep(100);- if (t.hasTimedOut()) {- fail("Timed out waiting for collection " + collection + " to be deleted.");- }- cluster.getZkStateReader().forceUpdateCollection(collection);- } catch (SolrException e) {- return;- }- }- }-private CollectionStatePredicate activeReplicaCount(int numWriter, int numActive, int numPassive) {return (liveNodes, collectionState) -> {diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java b/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.javaindex 0b59d5d74ab..20343487523 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java@@ -111,8 +111,7 @@ public class TestRebalanceLeaders extends SolrCloudTestCase {// shardUnique=true for the special property preferredLeader. That was removed at one point, so// we're explicitly testing that as well.@Test- public void testSetArbitraryPropertySliceUnique()- throws IOException, SolrServerException, InterruptedException {+ public void testSetArbitraryPropertySliceUnique() throws IOException, SolrServerException {// Check both special (preferredLeader) and something arbitrary.doTestSetArbitraryPropertySliceUnique("foo" + random().nextInt(1_000_000));removeAllProperties();@@ -124,8 +123,7 @@ public class TestRebalanceLeaders extends SolrCloudTestCase {// individual properties on individual nodes. This one relies on Solr to pick which replicas to// set the property on@Test- public void testBalancePropertySliceUnique()- throws InterruptedException, IOException, SolrServerException {+ public void testBalancePropertySliceUnique() throws IOException, SolrServerException {// Check both cases of "special" property preferred(Ll)eaderdoTestBalancePropertySliceUnique("foo" + random().nextInt(1_000_000));removeAllProperties();@@ -160,7 +158,7 @@ public class TestRebalanceLeaders extends SolrCloudTestCase {// on an individual// replica.private void doTestSetArbitraryPropertySliceUnique(String propIn)- throws InterruptedException, IOException, SolrServerException {+ throws IOException, SolrServerException {final String prop = (random().nextBoolean()) ? propIn : propIn.toUpperCase(Locale.ROOT);// First set the property in some replica in some sliceforceUpdateCollectionStatus();@@ -176,43 +174,29 @@ public class TestRebalanceLeaders extends SolrCloudTestCase {Replica rep = reps[random().nextInt(reps.length)];// Set the property on a particular replicasetProp(slice, rep, prop);- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);-- long count = 0;- boolean rightRep = false;- Slice modSlice;- DocCollection modColl = null; // keeps IDE happy// ensure that no other replica in that slice has the property when we return.- while (timeout.hasTimedOut() == false) {- forceUpdateCollectionStatus();- modColl = cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);- modSlice = modColl.getSlice(slice.getName());- rightRep =- modSlice- .getReplica(rep.getName())- .getBool("property." + prop.toLowerCase(Locale.ROOT), false);- count =- modSlice.getReplicas().stream()- .filter(- thisRep -> thisRep.getBool("property." + prop.toLowerCase(Locale.ROOT), false))- .count();-- if (count == 1 && rightRep) {- break;- }-- TimeUnit.MILLISECONDS.sleep(50);- }- if (count != 1 || rightRep == false) {- fail(- "The property "- + prop- + " was not uniquely distributed in slice "- + slice.getName()- + " "- + modColl.toString());- }+ waitForState(+ "Check property is uniquely distributed in slice: " + prop,+ COLLECTION_NAME,+ (n, c) -> {+ forceUpdateCollectionStatus();+ Slice modSlice = c.getSlice(slice.getName());+ boolean rightRep =+ modSlice+ .getReplica(rep.getName())+ .getBool("property." + prop.toLowerCase(Locale.ROOT), false);+ long count =+ modSlice.getReplicas().stream()+ .filter(+ thisRep ->+ thisRep.getBool("property." + prop.toLowerCase(Locale.ROOT), false))+ .count();++ return count == 1 && rightRep;+ },+ timeoutMs,+ TimeUnit.MILLISECONDS);}}@@ -332,7 +316,7 @@ public class TestRebalanceLeaders extends SolrCloudTestCase {// re-assigned with the// BALANCESHARDUNIQUE command.private void doTestBalancePropertySliceUnique(String propIn)- throws InterruptedException, IOException, SolrServerException {+ throws IOException, SolrServerException {final String prop = (random().nextBoolean()) ? propIn : propIn.toUpperCase(Locale.ROOT);// Concentrate the properties on as few replicas a possible@@ -350,63 +334,55 @@ public class TestRebalanceLeaders extends SolrCloudTestCase {private void verifyPropCorrectlyDistributed(String prop) {- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);-String propLC = prop.toLowerCase(Locale.ROOT);- DocCollection docCollection = null;- while (timeout.hasTimedOut() == false) {- forceUpdateCollectionStatus();- docCollection = cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);- int maxPropCount = Integer.MAX_VALUE;- int minPropCount = Integer.MIN_VALUE;- for (Slice slice : docCollection.getSlices()) {- int repCount = 0;- for (Replica rep : slice.getReplicas()) {- if (rep.getBool("property." + propLC, false)) {- repCount++;+ waitForState(+ "Check property is distributed evenly: " + prop,+ COLLECTION_NAME,+ (liveNodes, docCollection) -> {+ int maxPropCount = 0;+ int minPropCount = Integer.MAX_VALUE;+ for (Slice slice : docCollection.getSlices()) {+ int repCount = 0;+ for (Replica rep : slice.getReplicas()) {+ if (rep.getBool("property." + propLC, false)) {+ repCount++;+ }+ }+ maxPropCount = Math.max(maxPropCount, repCount);+ minPropCount = Math.min(minPropCount, repCount);}- }- maxPropCount = Math.max(maxPropCount, repCount);- minPropCount = Math.min(minPropCount, repCount);- }- if (Math.abs(maxPropCount - minPropCount) < 2) return;- }- log.error("Property {} is not distributed evenly. {}", prop, docCollection);- fail("Property is not distributed evenly " + prop);+ return Math.abs(maxPropCount - minPropCount) < 2;+ },+ timeoutMs,+ TimeUnit.MILLISECONDS);}// Used when we concentrate the leader on a few nodes.private void verifyPropDistributedAsExpected(- Map<String, String> expectedShardReplicaMap, String prop) throws InterruptedException {+ Map<String, String> expectedShardReplicaMap, String prop) {// Make sure that the shard unique are where you expect.- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);-String propLC = prop.toLowerCase(Locale.ROOT);- boolean failure = false;- DocCollection docCollection = null;- while (timeout.hasTimedOut() == false) {- forceUpdateCollectionStatus();- docCollection = cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);- failure = false;- for (Map.Entry<String, String> ent : expectedShardReplicaMap.entrySet()) {- Replica rep = docCollection.getSlice(ent.getKey()).getReplica(ent.getValue());- if (rep.getBool("property." + propLC, false) == false) {- failure = true;- }- }- if (failure == false) {- return;- }- TimeUnit.MILLISECONDS.sleep(100);- }+ String message =+ String.format(+ Locale.ROOT,+ "Checking properties are on the expected replicas. Props:%s Expected:%s",+ prop,+ expectedShardReplicaMap.toString());- fail(- prop- + " properties are not on the expected replicas: "- + docCollection.toString()- + System.lineSeparator()- + "Expected "- + expectedShardReplicaMap.toString());+ waitForState(+ message,+ COLLECTION_NAME,+ (liveNodes, docCollection) -> {+ for (Map.Entry<String, String> ent : expectedShardReplicaMap.entrySet()) {+ Replica rep = docCollection.getSlice(ent.getKey()).getReplica(ent.getValue());+ if (rep.getBool("property." + propLC, false) == false) {+ return false;+ }+ }+ return true;+ },+ timeoutMs,+ TimeUnit.MILLISECONDS);}// Just check that the property is distributed as expectecd. This does _not_ rebalance the leaders@@ -558,14 +534,6 @@ public class TestRebalanceLeaders extends SolrCloudTestCase {// then start them again to concentrate the leaders. It's not necessary that all shards have a// leader.- ExecutorService executorService = ExecutorUtil.newMDCAwareCachedThreadPool("Start Jetty");-- for (JettySolrRunner jetty : jettys) {- cluster.stopJettySolrRunner(jetty);- }-- ExecutorUtil.shutdownAndAwaitTermination(executorService);-for (JettySolrRunner jetty : jettys) {cluster.stopJettySolrRunner(jetty);}@@ -575,7 +543,7 @@ public class TestRebalanceLeaders extends SolrCloudTestCase {}checkReplicasInactive(jettys);- executorService = ExecutorUtil.newMDCAwareCachedThreadPool("Start Jetty");+ ExecutorService executorService = ExecutorUtil.newMDCAwareCachedThreadPool("Start Jetty");for (int idx = 0; idx < jettys.size(); ++idx) {int finalIdx = idx;@@ -604,74 +572,61 @@ public class TestRebalanceLeaders extends SolrCloudTestCase {// Since we have to restart jettys, we don't want to try re-balancing etc. until we're sure all// jettys that should be up are and all replicas are active.- private void checkReplicasInactive(List<JettySolrRunner> downJettys) throws InterruptedException {- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);- DocCollection docCollection = null;- Set<String> liveNodes = null;+ private void checkReplicasInactive(List<JettySolrRunner> downJettys) {Set<String> downJettyNodes = new TreeSet<>();for (JettySolrRunner jetty : downJettys) {downJettyNodes.add(jetty.getBaseUrl().getHost() + ":" + jetty.getBaseUrl().getPort() + "_solr");}- while (timeout.hasTimedOut() == false) {- forceUpdateCollectionStatus();- docCollection = cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);- liveNodes = cluster.getSolrClient().getClusterState().getLiveNodes();- boolean expectedInactive = true;-- for (Slice slice : docCollection.getSlices()) {- for (Replica rep : slice.getReplicas()) {- if (downJettyNodes.contains(rep.getNodeName()) == false) {- continue; // We are on a live node- }- // A replica on an allegedly down node is reported as active.- if (rep.isActive(liveNodes)) {- expectedInactive = false;++ waitForState(+ "Waiting for all replicas to become inactive",+ COLLECTION_NAME,+ (liveNodes, docCollection) -> {+ boolean expectedInactive = true;++ for (Slice slice : docCollection.getSlices()) {+ for (Replica rep : slice.getReplicas()) {+ if (downJettyNodes.contains(rep.getNodeName()) == false) {+ continue; // We are on a live node+ }+ // A replica on an allegedly down node is reported as active.+ if (rep.isActive(liveNodes)) {+ expectedInactive = false;+ }+ }}- }- }- if (expectedInactive) {- return;- }- TimeUnit.MILLISECONDS.sleep(100);- }- fail(- "timed out waiting for all replicas to become inactive: livenodes: "- + liveNodes- + " Collection state: "- + docCollection.toString());+ return expectedInactive;+ },+ timeoutMs,+ TimeUnit.MILLISECONDS);}// We need to wait around until all replicas are active before expecting rebalancing or// distributing shard-unique properties to work.- private void checkAllReplicasActive() throws InterruptedException {- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);- while (timeout.hasTimedOut() == false) {- forceUpdateCollectionStatus();- DocCollection docCollection =- cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);- Set<String> liveNodes = cluster.getSolrClient().getClusterState().getLiveNodes();- boolean allActive = true;- for (Slice slice : docCollection.getSlices()) {- for (Replica rep : slice.getReplicas()) {- if (rep.isActive(liveNodes) == false) {- allActive = false;+ private void checkAllReplicasActive() {+ waitForState(+ "Waiting for all replicas to become active",+ COLLECTION_NAME,+ (liveNodes, docCollection) -> {+ boolean allActive = true;+ for (Slice slice : docCollection.getSlices()) {+ for (Replica rep : slice.getReplicas()) {+ if (rep.isActive(liveNodes) == false) {+ allActive = false;+ }+ }}- }- }- if (allActive) {- return;- }- TimeUnit.MILLISECONDS.sleep(100);- }- fail("timed out waiting for all replicas to become active");+ return allActive;+ },+ timeoutMs,+ TimeUnit.MILLISECONDS);}// use a simple heuristic to put as many replicas with the property on as few nodes as possible.// The point is that then we can execute BALANCESHARDUNIQUE and be sure it worked correctly- private void concentrateProp(String prop)- throws InterruptedException, IOException, SolrServerException {+ private void concentrateProp(String prop) throws IOException, SolrServerException {// find all the live nodes for each slice, assign the leader to the first replica that is in the// lowest position on live_nodesList<String> liveNodes =@@ -704,43 +659,26 @@ public class TestRebalanceLeaders extends SolrCloudTestCase {}// make sure that the property in question is unique per shard.- private Map<String, String> verifyPropUniquePerShard(String prop) throws InterruptedException {- Map<String, String> uniquePropMaps = new TreeMap<>();-- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);- while (timeout.hasTimedOut() == false) {- uniquePropMaps.clear();- if (checkUniquePropPerShard(uniquePropMaps, prop)) {- return uniquePropMaps;- }- TimeUnit.MILLISECONDS.sleep(10);- }- fail(- "There should be exactly one replica with value "- + prop- + " set to true per shard: "- + cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME).toString());- return null; // keeps IDE happy.- }-- // return true if every shard has exactly one replica with the unique property set to "true"- private boolean checkUniquePropPerShard(Map<String, String> uniques, String prop) {- forceUpdateCollectionStatus();- DocCollection docCollection =- cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);+ private void verifyPropUniquePerShard(String prop) {- for (Slice slice : docCollection.getSlices()) {- int propCount = 0;- for (Replica rep : slice.getReplicas()) {- if (rep.getBool("property." + prop.toLowerCase(Locale.ROOT), false)) {- propCount++;- uniques.put(slice.getName(), rep.getName());- }- }- if (1 != propCount) {- return false;- }- }- return true;+ waitForState(+ "Waiting to have exactly one replica with " + prop + "set per shard",+ COLLECTION_NAME,+ (liveNodes, docCollection) -> {+ for (Slice slice : docCollection.getSlices()) {+ int propCount = 0;+ for (Replica rep : slice.getReplicas()) {+ if (rep.getBool("property." + prop.toLowerCase(Locale.ROOT), false)) {+ propCount++;+ }+ }+ if (1 != propCount) {+ return false;+ }+ }+ return true;+ },+ timeoutMs,+ TimeUnit.MILLISECONDS);}}diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.javaindex 563ed686298..d559de6d333 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java@@ -815,17 +815,17 @@ public class TestTlogReplica extends SolrCloudTestCase {cloudClient.request(request);// Wait until a preferredleader flag is set to the new leader candidate- TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);- while (!timeout.hasTimedOut()) {- Map<String, Slice> slices =- cloudClient.getClusterState().getCollection(collectionName).getSlicesMap();- Replica me = slices.get(slice.getName()).getReplica(newLeader.getName());- if (me.getBool("property.preferredleader", false)) {- break;- }- Thread.sleep(100);- }- assertFalse("Timeout waiting for setting preferredleader flag", timeout.hasTimedOut());+ String newLeaderName = newLeader.getName();+ waitForState(+ "Waiting for setting preferredleader flag",+ collectionName,+ (n, c) -> {+ Map<String, Slice> slices = c.getSlicesMap();+ Replica me = slices.get(slice.getName()).getReplica(newLeaderName);+ return me.getBool("property.preferredleader", false);+ },+ 10,+ TimeUnit.SECONDS);// Rebalance leadersparams = new ModifiableSolrParams();@@ -837,18 +837,17 @@ public class TestTlogReplica extends SolrCloudTestCase {cloudClient.request(request);// Wait until a new leader is elected- timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);- while (!timeout.hasTimedOut()) {- docCollection = getCollectionState(collectionName);- Replica leader = docCollection.getSlice(slice.getName()).getLeader();- if (leader != null- && leader.getName().equals(newLeader.getName())- && leader.isActive(cloudClient.getClusterState().getLiveNodes())) {- break;- }- Thread.sleep(100);- }- assertFalse("Timeout waiting for a new leader to be elected", timeout.hasTimedOut());+ waitForState(+ "Waiting for a new leader to be elected",+ collectionName,+ (n, c) -> {+ Replica leader = c.getSlice(slice.getName()).getLeader();+ return leader != null+ && leader.getName().equals(newLeaderName)+ && leader.isActive(cloudClient.getClusterState().getLiveNodes());+ },+ 30,+ TimeUnit.SECONDS);new UpdateRequest().add(sdoc("id", "1"))@@ -1027,19 +1026,13 @@ public class TestTlogReplica extends SolrCloudTestCase {}}- private void waitForDeletion(String collection) throws InterruptedException, KeeperException {- TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);- while (cluster.getSolrClient().getClusterState().hasCollection(collection)) {- try {- Thread.sleep(100);- if (t.hasTimedOut()) {- fail("Timed out waiting for collection " + collection + " to be deleted.");- }- cluster.getZkStateReader().forceUpdateCollection(collection);- } catch (SolrException e) {- return;- }- }+ private void waitForDeletion(String collection) {+ waitForState(+ "Waiting for collection " + collection + " to be deleted",+ collection,+ (n, c) -> c == null,+ 10,+ TimeUnit.SECONDS);}private DocCollection assertNumberOfReplicas(diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.javaindex 43f3367f489..28335c920eb 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java@@ -960,7 +960,7 @@ public class ShardSplitTest extends BasicDistributedZkTest {}List<Integer> list = collectionInfos.get(collectionName);- checkForCollection(collectionName, list, null);+ checkForCollection(collectionName, list);waitForRecoveriesToFinish(false);@@ -1031,7 +1031,7 @@ public class ShardSplitTest extends BasicDistributedZkTest {}List<Integer> list = collectionInfos.get(collectionName);- checkForCollection(collectionName, list, null);+ checkForCollection(collectionName, list);waitForRecoveriesToFinish(false);diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.javaindex 329cc7fee11..34e3b2a1492 100644--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java+++ b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java@@ -31,6 +31,7 @@ import java.util.concurrent.ExecutorCompletionService;import java.util.concurrent.Future;import java.util.concurrent.SynchronousQueue;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import java.util.stream.Collectors;import org.apache.hadoop.hdfs.MiniDFSCluster;import org.apache.lucene.tests.util.LuceneTestCase;@@ -57,13 +58,11 @@ import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.common.util.SolrNamedThreadFactory;-import org.apache.solr.common.util.TimeSource;import org.apache.solr.core.CoreDescriptor;import org.apache.solr.embedded.JettySolrRunner;import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;import org.apache.solr.util.LogLevel;import org.apache.solr.util.TestInjection;-import org.apache.solr.util.TimeOut;import org.junit.AfterClass;import org.junit.Before;import org.junit.BeforeClass;@@ -425,37 +424,43 @@ public class SharedFileSystemAutoReplicaFailoverTest extends AbstractFullDistribprivate void assertSliceAndReplicaCount(String collection, int numSlices, int numReplicas, int timeOutInMs)- throws InterruptedException, IOException {- TimeOut timeOut = new TimeOut(timeOutInMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);- while (!timeOut.hasTimedOut()) {- ClusterState clusterState = cloudClient.getClusterState();- Collection<Slice> slices = clusterState.getCollection(collection).getActiveSlices();- if (slices.size() == numSlices) {- boolean isMatch = true;- for (Slice slice : slices) {- int count = 0;- for (Replica replica : slice.getReplicas()) {- if (replica.getState() == Replica.State.ACTIVE- && clusterState.liveNodesContain(replica.getNodeName())) {- count++;- }- }- if (count < numReplicas) {- isMatch = false;- }- }- if (isMatch) return;- }- Thread.sleep(200);+ throws InterruptedException {++ try {+ ZkStateReader.from(cloudClient)+ .waitForState(+ collection,+ timeOutInMs,+ TimeUnit.MILLISECONDS,+ (liveNodes, c) -> {+ Collection<Slice> slices = c.getActiveSlices();+ if (slices.size() == numSlices) {+ for (Slice slice : slices) {+ int count = 0;+ for (Replica replica : slice.getReplicas()) {+ if (replica.getState() == Replica.State.ACTIVE+ && liveNodes.contains(replica.getNodeName())) {+ count++;+ }+ }+ if (count < numReplicas) {+ return false;+ }+ }+ return true;+ }+ return false;+ });+ } catch (TimeoutException e) {+ fail(+ "Expected numSlices="+ + numSlices+ + " numReplicas="+ + numReplicas+ + " but found "+ + cloudClient.getClusterState().getCollection(collection)+ + " with /live_nodes: "+ + cloudClient.getClusterState().getLiveNodes());}- fail(- "Expected numSlices="- + numSlices- + " numReplicas="- + numReplicas- + " but found "- + cloudClient.getClusterState().getCollection(collection)- + " with /live_nodes: "- + cloudClient.getClusterState().getLiveNodes());}}diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.javaindex 7bb98b8d97b..326daaddc03 100644--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.java+++ b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.java@@ -48,9 +48,7 @@ import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.CollectionParams.CollectionAction;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;-import org.apache.solr.common.util.TimeSource;import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;-import org.apache.solr.util.TimeOut;import org.junit.AfterClass;import org.junit.BeforeClass;import org.junit.Test;@@ -232,14 +230,8 @@ public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {request.setPath("/admin/collections");solrClient.request(request);- final TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);- while (cloudClient.getClusterState().hasCollection(DELETE_DATA_DIR_COLLECTION)) {- if (timeout.hasTimedOut()) {- throw new AssertionError("Timeout waiting to see removed collection leave clusterstate");- }-- Thread.sleep(200);- }+ waitForCollectionToDisappear(+ DELETE_DATA_DIR_COLLECTION, ZkStateReader.from(cloudClient), true, 10);// check that all dirs are gonefor (String dataDir : dataDirs) {diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.javaindex fb138876b30..8c8a115636e 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java@@ -303,7 +303,7 @@ public abstract class AbstractChaosMonkeyNothingIsSafeTestBaseList<Integer> numShardsNumReplicas = new ArrayList<>(2);numShardsNumReplicas.add(1);numShardsNumReplicas.add(1);- checkForCollection("testcollection", numShardsNumReplicas, null);+ checkForCollection("testcollection", numShardsNumReplicas);testSuccessful = true;} finally {diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.javaindex ab765d32699..ef371efcad4 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java@@ -203,7 +203,7 @@ public abstract class AbstractChaosMonkeySafeLeaderTestBase extends AbstractFullList<Integer> numShardsNumReplicas = new ArrayList<>(2);numShardsNumReplicas.add(1);numShardsNumReplicas.add(1);- checkForCollection("testcollection", numShardsNumReplicas, null);+ checkForCollection("testcollection", numShardsNumReplicas);}private void tryDelete() throws Exception {diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.javaindex 37c90d98db9..f256a5fb937 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java@@ -16,7 +16,6 @@*/package org.apache.solr.cloud;-import static java.util.concurrent.TimeUnit.MILLISECONDS;import static java.util.concurrent.TimeUnit.SECONDS;import java.io.File;@@ -41,7 +40,6 @@ import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.core.Diagnostics;import org.apache.solr.core.MockDirectoryFactory;import org.apache.solr.embedded.JettySolrRunner;-import org.apache.solr.util.TimeOut;import org.apache.zookeeper.KeeperException;import org.junit.BeforeClass;import org.slf4j.Logger;@@ -242,65 +240,42 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTeslog.info("Collection has disappeared - collection:{}", collection);}- static void waitForNewLeader(- CloudSolrClient cloudClient, String shardName, Replica oldLeader, TimeOut timeOut)+ static void waitForNewLeader(CloudSolrClient cloudClient, String shardName, Replica oldLeader)throws Exception {- log.info("Will wait for a node to become leader for {} secs", timeOut.timeLeft(SECONDS));+ log.info("Will wait for a node to become leader for 15 secs");ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);- zkStateReader.forceUpdateCollection(DEFAULT_COLLECTION);-- for (; ; ) {- ClusterState clusterState = zkStateReader.getClusterState();- DocCollection coll = clusterState.getCollection("collection1");- Slice slice = coll.getSlice(shardName);- if (slice.getLeader() != null- && !slice.getLeader().equals(oldLeader)- && slice.getLeader().getState() == Replica.State.ACTIVE) {- if (log.isInfoEnabled()) {- log.info(- "Old leader {}, new leader {}. New leader got elected in {} ms",- oldLeader,- slice.getLeader(),- timeOut.timeElapsed(MILLISECONDS));- }- break;- }-- if (timeOut.hasTimedOut()) {- Diagnostics.logThreadDumps("Could not find new leader in specified timeout");- zkStateReader.getZkClient().printLayoutToStream(System.out);- fail(- "Could not find new leader even after waiting for "- + timeOut.timeElapsed(MILLISECONDS)- + "ms");- }- Thread.sleep(100);- }+ long startNs = System.nanoTime();+ try {+ zkStateReader.waitForState(+ "collection1",+ 15,+ TimeUnit.SECONDS,+ (docCollection) -> {+ if (docCollection == null) return false;- zkStateReader.waitForState(- "collection1",- timeOut.timeLeft(SECONDS),- TimeUnit.SECONDS,- (docCollection) -> {- if (docCollection == null) return false;-- Slice slice = docCollection.getSlice(shardName);- if (slice != null- && slice.getLeader() != null- && !slice.getLeader().equals(oldLeader)- && slice.getLeader().getState() == Replica.State.ACTIVE) {- if (log.isInfoEnabled()) {- log.info(- "Old leader {}, new leader {}. New leader got elected in {} ms",- oldLeader,- slice.getLeader(),- timeOut.timeElapsed(MILLISECONDS));+ Slice slice = docCollection.getSlice(shardName);+ if (slice != null+ && slice.getLeader() != null+ && !slice.getLeader().equals(oldLeader)+ && slice.getLeader().getState() == Replica.State.ACTIVE) {+ if (log.isInfoEnabled()) {+ log.info(+ "Old leader {}, new leader {}. New leader got elected in {} ms",+ oldLeader,+ slice.getLeader(),+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs));+ }+ return true;}- return true;- }- return false;- });+ return false;+ });+ } catch (TimeoutException e) {+ // If we failed to get a new leader, print some diagnotics before the test fails+ Diagnostics.logThreadDumps("Could not find new leader in specified timeout");+ zkStateReader.getZkClient().printLayoutToStream(System.out);+ fail("Could not find new leader even after waiting for 15s");+ }}public static void verifyReplicaStatus(diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.javaindex dcdab4e86d1..003ef951b0f 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java@@ -44,6 +44,7 @@ import java.util.concurrent.ExecutorService;import java.util.concurrent.TimeUnit;import java.util.concurrent.TimeoutException;import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicReference;import java.util.function.Consumer;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrQuery;@@ -2217,83 +2218,60 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTesreturn sdoc(fields);}- private String checkCollectionExpectations(- String collectionName,- List<Integer> numShardsNumReplicaList,- List<String> nodesAllowedToRunShards)- throws IOException {- getCommonCloudSolrClient();- ClusterState clusterState = cloudClient.getClusterState();- int expectedSlices = numShardsNumReplicaList.get(0);- // The Math.min thing is here, because we expect replication-factor to be reduced to if there- // are not enough live nodes to spread all shards of a collection over different nodes- int expectedShardsPerSlice = numShardsNumReplicaList.get(1);- int expectedTotalShards = expectedSlices * expectedShardsPerSlice;-- // Map<String,DocCollection> collections = clusterState- // .getCollectionStates();- if (clusterState.hasCollection(collectionName)) {- Map<String, Slice> slices = clusterState.getCollection(collectionName).getSlicesMap();- // did we find expectedSlices slices/shards?- if (slices.size() != expectedSlices) {- return "Found new collection "- + collectionName- + ", but mismatch on number of slices. Expected: "- + expectedSlices- + ", actual: "- + slices.size();- }- int totalShards = 0;- for (String sliceName : slices.keySet()) {- for (Replica replica : slices.get(sliceName).getReplicas()) {- if (nodesAllowedToRunShards != null- && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {- return "Shard "- + replica.getName()- + " created on node "- + replica.getNodeName()- + " not allowed to run shards for the created collection "- + collectionName;- }- }- totalShards += slices.get(sliceName).getReplicas().size();- }- if (totalShards != expectedTotalShards) {- return "Found new collection "- + collectionName- + " with correct number of slices, but mismatch on number of shards. Expected: "- + expectedTotalShards- + ", actual: "- + totalShards;- }- return null;- } else {- return "Could not find new collection " + collectionName;- }- }-- protected void checkForCollection(- String collectionName,- List<Integer> numShardsNumReplicaList,- List<String> nodesAllowedToRunShards)+ protected void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList)throws Exception {// check for an expectedSlices new collection - we poll the state- final TimeOut timeout = new TimeOut(120, TimeUnit.SECONDS, TimeSource.NANO_TIME);- boolean success = false;- String checkResult = "Didnt get to perform a single check";- while (!timeout.hasTimedOut()) {- checkResult =- checkCollectionExpectations(- collectionName, numShardsNumReplicaList, nodesAllowedToRunShards);- if (checkResult == null) {- success = true;- break;- }- Thread.sleep(500);- }- if (!success) {- super.printLayout();- fail(checkResult);+ ZkStateReader reader = ZkStateReader.from(cloudClient);++ AtomicReference<String> message = new AtomicReference<>();+ try {+ reader.waitForState(+ collectionName,+ 120,+ TimeUnit.SECONDS,+ c -> {+ int expectedSlices = numShardsNumReplicaList.get(0);+ // The Math.min thing is here, because we expect replication-factor to be reduced to if+ // there are not enough live nodes to spread all shards of a collection over different+ // nodes.+ int expectedShardsPerSlice = numShardsNumReplicaList.get(1);+ int expectedTotalShards = expectedSlices * expectedShardsPerSlice;++ if (c != null) {+ Collection<Slice> slices = c.getSlices();+ // did we find expectedSlices slices/shards?+ if (slices.size() != expectedSlices) {+ message.set(+ "Found new collection "+ + collectionName+ + ", but mismatch on number of slices. Expected: "+ + expectedSlices+ + ", actual: "+ + slices.size());+ return false;+ }+ int totalShards = 0;+ for (Slice slice : slices) {+ totalShards += slice.getReplicas().size();+ }+ if (totalShards != expectedTotalShards) {+ message.set(+ "Found new collection "+ + collectionName+ + " with correct number of slices, but mismatch on number of shards. Expected: "+ + expectedTotalShards+ + ", actual: "+ + totalShards);+ return false;+ }+ return true;+ } else {+ message.set("Could not find new collection " + collectionName);+ return false;+ }+ });+ } catch (TimeoutException e) {+ fail(message.get());}}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.javaindex 2f77b69b27b..ee7fac68736 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java@@ -26,6 +26,7 @@ import java.util.Set;import java.util.concurrent.SynchronousQueue;import java.util.concurrent.ThreadPoolExecutor;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrQuery;import org.apache.solr.client.solrj.SolrServerException;@@ -33,7 +34,6 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.ZkCoreNodeProps;@@ -41,12 +41,10 @@ import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.common.util.SolrNamedThreadFactory;-import org.apache.solr.common.util.TimeSource;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrPaths;import org.apache.solr.embedded.JettySolrRunner;import org.apache.solr.util.TestInjection;-import org.apache.solr.util.TimeOut;import org.junit.Test;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -97,42 +95,37 @@ public abstract class AbstractUnloadDistributedZkTestBase extends AbstractFullDiprivate void checkCoreNamePresenceAndSliceCount(String collectionName, String coreName, boolean shouldBePresent, int expectedSliceCount)throws Exception {- final TimeOut timeout = new TimeOut(45, TimeUnit.SECONDS, TimeSource.NANO_TIME);- Boolean isPresent = null; // null meaning "don't know"- while (null == isPresent || shouldBePresent != isPresent) {- getCommonCloudSolrClient();- final DocCollection docCollection =- cloudClient.getClusterState().getCollectionOrNull(collectionName);- final Collection<Slice> slices =- (docCollection != null) ? docCollection.getSlices() : Collections.emptyList();- if (timeout.hasTimedOut()) {- printLayout();- fail(- "checkCoreNamePresenceAndSliceCount failed:"- + " collection="- + collectionName- + " CoreName="- + coreName- + " shouldBePresent="- + shouldBePresent- + " isPresent="- + isPresent- + " expectedSliceCount="- + expectedSliceCount- + " actualSliceCount="- + slices.size());- }- if (expectedSliceCount == slices.size()) {- isPresent = false;- for (Slice slice : slices) {- for (Replica replica : slice.getReplicas()) {- if (coreName.equals(replica.get("core"))) {- isPresent = true;+ ZkStateReader reader = ZkStateReader.from(cloudClient);+ try {+ reader.waitForState(+ collectionName,+ 45,+ TimeUnit.SECONDS,+ c -> {+ final Collection<Slice> slices = (c != null) ? c.getSlices() : Collections.emptyList();+ if (expectedSliceCount == slices.size()) {+ for (Slice slice : slices) {+ for (Replica replica : slice.getReplicas()) {+ if (coreName.equals(replica.get("core"))) {+ return shouldBePresent;+ }+ }+ }+ return !shouldBePresent;+ } else {+ return false;}- }- }- }- Thread.sleep(1000);+ });+ } catch (TimeoutException e) {+ printLayout();+ fail(+ "checkCoreNamePresenceAndSliceCount failed:"+ + " collection="+ + collectionName+ + " CoreName="+ + coreName+ + " shouldBePresent="+ + shouldBePresent);}}