Apache Solr open-source search software
SOLR-16114 Remove SolrZookeeper (#759)
Mike Drob98aa52b9b2a2
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 6a1a1e5b07d..e5bca4a7c44 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -63,6 +63,8 @@ Other Changes* SOLR-15886: Remove deprecated showItems configuration value from solrconfig.xml files (Andy Lester via Eric Pugh)+* SOLR-16114: SolrZooKeeper has been removed in favor of using ZooKeeper directly. (Mike Drob)+Build---------------------* SOLR-16053: Upgrade scriptDepVersions (Kevin Risden)diff --git a/solr/core/build.gradle b/solr/core/build.gradleindex 2a04dbb719f..2e4a565403d 100644--- a/solr/core/build.gradle+++ b/solr/core/build.gradle@@ -130,6 +130,7 @@ dependencies {implementation ('org.apache.zookeeper:zookeeper-jute') {exclude group: 'org.apache.yetus', module: 'audience-annotations'}+ testImplementation 'org.apache.zookeeper:zookeeper::tests'// required for instantiating a Zookeeper server (for embedding ZK or running tests)runtimeOnly ('org.xerial.snappy:snappy-java')diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.javaindex f5a3eba247e..650d21f8ef5 100644--- a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java+++ b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java@@ -112,7 +112,7 @@ public class LeaderElector {// If any double-registrations exist for me, remove all but this latest one!// TODO: can we even get into this state?- String prefix = zkClient.getSolrZooKeeper().getSessionId() + "-" + context.id + "-";+ String prefix = zkClient.getZooKeeper().getSessionId() + "-" + context.id + "-";Iterator<String> it = seqs.iterator();while (it.hasNext()) {String elec = it.next();@@ -232,7 +232,7 @@ public class LeaderElector {final String shardsElectZkPath = context.electionPath + LeaderElector.ELECTION_NODE;- long sessionId = zkClient.getSolrZooKeeper().getSessionId();+ long sessionId = zkClient.getZooKeeper().getSessionId();String id = sessionId + "-" + context.id;String leaderSeqPath = null;boolean cont = true;diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.javaindex 1f76e487007..838ec5d24a7 100644--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java@@ -53,7 +53,6 @@ import org.apache.solr.common.AlreadyClosedException;import org.apache.solr.common.SolrCloseable;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.ClusterState;-import org.apache.solr.common.cloud.ConnectionManager;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.SolrZkClient;@@ -1022,11 +1021,7 @@ public class Overseer implements SolrCloseable {"/overseer/queue",zkStats,STATE_UPDATE_MAX_QUEUE,- new ConnectionManager.IsClosed() {- public boolean isClosed() {- return Overseer.this.isClosed() || zkController.getCoreContainer().isShutDown();- }- });+ () -> Overseer.this.isClosed() || zkController.getCoreContainer().isShutDown());}/**diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.javaindex 9e151f2b9cb..0778f7e9b66 100644--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java@@ -317,7 +317,11 @@ public class ZkController implements Closeable {this.leaderConflictResolveWait = cloudConfig.getLeaderConflictResolveWait();this.clientTimeout = cloudConfig.getZkClientTimeout();- DefaultConnectionStrategy strat = new DefaultConnectionStrategy();++ String connectionStrategy = System.getProperty("solr.zookeeper.connectionStrategy");+ ZkClientConnectionStrategy strat =+ ZkClientConnectionStrategy.forName(connectionStrategy, new DefaultConnectionStrategy());+String zkACLProviderClass = cloudConfig.getZkACLProviderClass();ZkACLProvider zkACLProvider = null;if (zkACLProviderClass != null && zkACLProviderClass.trim().length() > 0) {@@ -471,13 +475,7 @@ public class ZkController implements Closeable {}},zkACLProvider,- new ConnectionManager.IsClosed() {-- @Override- public boolean isClosed() {- return cc.isShutDown();- }- });+ cc::isShutDown);// Refuse to start if ZK has a non empty /clusterstate.jsoncheckNoOldClusterstate(zkClient);@@ -1057,8 +1055,7 @@ public class ZkController implements Closeable {}boolean deleted =- deletedLatch.await(- zkClient.getSolrZooKeeper().getSessionTimeout() * 2, TimeUnit.MILLISECONDS);+ deletedLatch.await(zkClient.getZooKeeper().getSessionTimeout() * 2, TimeUnit.MILLISECONDS);if (!deleted) {throw new SolrException(ErrorCode.SERVER_ERROR,@@ -1159,7 +1156,7 @@ public class ZkController implements Closeable {String chrootPath = zkHost.substring(zkHost.indexOf("/"), zkHost.length());SolrZkClient tmpClient =- new SolrZkClient(zkHost.substring(0, zkHost.indexOf("/")), 60000, 30000, null, null, null);+ new SolrZkClient(zkHost.substring(0, zkHost.indexOf("/")), 60000, 30000);boolean exists = tmpClient.exists(chrootPath, true);if (!exists && create) {log.info("creating chroot {}", chrootPath);diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex d4e46fb5ca7..e30ca217bbf 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -1886,7 +1886,7 @@ public class CoreContainer {if (docCollection != null) {Replica replica = docCollection.getReplica(cd.getCloudDescriptor().getCoreNodeName());- assert replica != null;+ assert replica != null : cd.getCloudDescriptor().getCoreNodeName() + " had no replica";if (replica.getType() == Replica.Type.TLOG) { // TODO: needed here?getZkController().stopReplicationFromLeader(core.getName());if (!cd.getCloudDescriptor().isLeader()) {diff --git a/solr/core/src/test/org/apache/solr/cloud/ConnectionManagerTest.java b/solr/core/src/test/org/apache/solr/cloud/ConnectionManagerTest.javaindex daf0e9585c5..83fce07ab05 100644--- a/solr/core/src/test/org/apache/solr/cloud/ConnectionManagerTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ConnectionManagerTest.java@@ -24,13 +24,14 @@ import java.util.concurrent.TimeoutException;import org.apache.lucene.util.LuceneTestCase.Slow;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.common.cloud.ConnectionManager;-import org.apache.solr.common.cloud.DefaultConnectionStrategy;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.util.SolrNamedThreadFactory;+import org.apache.zookeeper.TestableZooKeeper;import org.apache.zookeeper.WatchedEvent;import org.apache.zookeeper.Watcher;import org.apache.zookeeper.Watcher.Event.EventType;import org.apache.zookeeper.Watcher.Event.KeeperState;+import org.apache.zookeeper.ZooKeeper;import org.junit.Ignore;import org.junit.Test;@@ -53,10 +54,11 @@ public class ConnectionManagerTest extends SolrTestCaseJ4 {try {assertFalse(cm.isLikelyExpired());- zkClient.getSolrZooKeeper().closeCnxn();+ ZooKeeper zk = zkClient.getZooKeeper();+ assertTrue(zk instanceof TestableZooKeeper);+ ((TestableZooKeeper) zk).testableConnloss();+ server.expire(zkClient.getZooKeeper().getSessionId());- long sessionId = zkClient.getSolrZooKeeper().getSessionId();- server.expire(sessionId);Thread.sleep(TIMEOUT);assertTrue(cm.isLikelyExpired());@@ -145,7 +147,7 @@ public class ConnectionManagerTest extends SolrTestCaseJ4 {}}- private static class MockZkClientConnectionStrategy extends DefaultConnectionStrategy {+ private static class MockZkClientConnectionStrategy extends TestConnectionStrategy {int called = 0;boolean exceptionThrown = false;diff --git a/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java b/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.javaindex ac0afecb1e9..8e37f4c60fe 100644--- a/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java@@ -288,7 +288,7 @@ public class DistributedQueueTest extends SolrTestCaseJ4 {}private void forceSessionExpire() throws InterruptedException, TimeoutException {- long sessionId = zkClient.getSolrZooKeeper().getSessionId();+ long sessionId = zkClient.getZooKeeper().getSessionId();zkServer.expire(sessionId);zkClient.getConnectionManager().waitForDisconnected(10000);zkClient.getConnectionManager().waitForConnected(10000);@@ -299,7 +299,7 @@ public class DistributedQueueTest extends SolrTestCaseJ4 {Thread.sleep(50);}assertTrue(zkClient.isConnected());- assertFalse(sessionId == zkClient.getSolrZooKeeper().getSessionId());+ assertNotEquals(sessionId, zkClient.getZooKeeper().getSessionId());}protected ZkDistributedQueue makeDistributedQueue(String dqZNode) throws Exception {diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.javaindex d2df6971d8c..0c901b06782 100644--- a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java@@ -101,10 +101,7 @@ public class LeaderElectionIntegrationTest extends SolrCloudTestCase {// timeout the leaderString leader = getLeader(collection);JettySolrRunner jetty = getRunner(leader);- ZkController zkController = jetty.getCoreContainer().getZkController();-- zkController.getZkClient().getSolrZooKeeper().closeCnxn();- cluster.getZkServer().expire(zkController.getZkClient().getSolrZooKeeper().getSessionId());+ cluster.expireZkSession(jetty);for (int i = 0; i < 60; i++) { // wait till leader is changedif (jetty != getRunner(getLeader(collection))) {diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.javaindex 1ef126083ba..38b4dc56607 100644--- a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java@@ -41,6 +41,8 @@ import org.apache.solr.common.util.Utils;import org.apache.zookeeper.KeeperException;import org.apache.zookeeper.KeeperException.NoNodeException;import org.apache.zookeeper.KeeperException.SessionExpiredException;+import org.apache.zookeeper.TestableZooKeeper;+import org.apache.zookeeper.ZooKeeper;import org.junit.AfterClass;import org.junit.BeforeClass;import org.junit.Test;@@ -531,32 +533,30 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {};Thread connLossThread =- new Thread() {- @Override- public void run() {-- while (!stopStress) {- try {- Thread.sleep(50);- int j;- j = random().nextInt(threads.size());+ new Thread(+ () -> {+ while (!stopStress) {try {- threads.get(j).es.zkClient.getSolrZooKeeper().closeCnxn();- if (random().nextBoolean()) {- long sessionId = zkClient.getSolrZooKeeper().getSessionId();- server.expire(sessionId);+ Thread.sleep(50);+ int j;+ j = random().nextInt(threads.size());+ try {+ ZooKeeper zk = threads.get(j).es.zkClient.getZooKeeper();+ assertTrue(zk instanceof TestableZooKeeper);+ ((TestableZooKeeper) zk).testableConnloss();+ if (random().nextBoolean()) {+ server.expire(zk.getSessionId());+ }+ } catch (Exception e) {+ e.printStackTrace();}- } catch (Exception e) {- e.printStackTrace();- }- Thread.sleep(500);+ Thread.sleep(500);- } catch (Exception e) {+ } catch (Exception e) {+ }}- }- }- };+ });scheduleThread.start();connLossThread.start();@@ -582,7 +582,7 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {// cleanup any threads still runningfor (ClientThread thread : threads) {- thread.es.zkClient.getSolrZooKeeper().close();+ thread.es.zkClient.getZooKeeper().close();thread.close();}diff --git a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.javaindex d17f6ec7c7d..0a4ebf0c2a3 100644--- a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java@@ -134,7 +134,7 @@ public class OutOfBoxZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {protected void assertOpenACLUnsafeAllover(SolrZkClient zkClient, String path, List<String> verifiedList) throws Exception {- List<ACL> acls = zkClient.getSolrZooKeeper().getACL(path, new Stat());+ List<ACL> acls = zkClient.getZooKeeper().getACL(path, new Stat());if (log.isInfoEnabled()) {log.info("Verifying {}", path);}diff --git a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.javaindex 0a741c12afa..610c2948f4c 100644--- a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java@@ -107,7 +107,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {new SolrZkClientFactoryUsingCompletelyNewProviders(null, null, null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);zkClient- .getSolrZooKeeper()+ .getZooKeeper().addAuthInfo("digest",("connectAndAllACLUsername:connectAndAllACLPassword").getBytes(DATA_ENCODING));diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java b/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.javaindex 3328da1b3c1..7e043bd23fb 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java@@ -66,8 +66,7 @@ public class TestLeaderElectionZkExpiry extends SolrTestCaseJ4 {long timeout =System.nanoTime() + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS);while (System.nanoTime() < timeout) {- long sessionId = zkController.getZkClient().getSolrZooKeeper().getSessionId();- server.expire(sessionId);+ server.expire(zkController.getZkClient().getZooKeeper().getSessionId());try {Thread.sleep(10);} catch (InterruptedException e) {diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.javaindex 3992c19ca81..95758a639ac 100644--- a/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java@@ -182,8 +182,7 @@ public class ZkSolrClientTest extends SolrTestCaseJ4 {// simulate session expiration// one option- long sessionId = zkClient.getSolrZooKeeper().getSessionId();- server.expire(sessionId);+ server.expire(zkClient.getZooKeeper().getSessionId());// another option// zkClient.getSolrZooKeeper().getConnection().disconnect();diff --git a/solr/licenses/zookeeper-3.7.0-tests.jar.sha1 b/solr/licenses/zookeeper-3.7.0-tests.jar.sha1new file mode 100644index 00000000000..ec7eff71132--- /dev/null+++ b/solr/licenses/zookeeper-3.7.0-tests.jar.sha1@@ -0,0 +1 @@+093f2c34c33ee16f9ff3f9352c79eafd3cd9040adiff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/SaslZkACLProviderTest.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/SaslZkACLProviderTest.javaindex 530aab2914e..4d200d50a91 100644--- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/SaslZkACLProviderTest.java+++ b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/SaslZkACLProviderTest.java@@ -91,8 +91,7 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 {try (SolrZkClient zkClient =new SolrZkClientWithACLs(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT)) {- ZooKeeperSaslClient saslClient =- zkClient.getSolrZooKeeper().getConnection().zooKeeperSaslClient;+ ZooKeeperSaslClient saslClient = zkClient.getZooKeeper().getSaslClient();assumeFalse("Could not set up ZK with SASL", saslClient.isFailed());zkClient.makePath("/solr", false, true);} catch (KeeperException e) {diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.javaindex c2c35941f05..a437a6bdf00 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java@@ -27,6 +27,7 @@ import org.apache.solr.common.SolrException;import org.apache.zookeeper.WatchedEvent;import org.apache.zookeeper.Watcher;import org.apache.zookeeper.Watcher.Event.KeeperState;+import org.apache.zookeeper.ZooKeeper;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -76,8 +77,8 @@ public class ConnectionManager implements Watcher {}}- public abstract static class IsClosed {- public abstract boolean isClosed();+ public interface IsClosed {+ boolean isClosed();}private volatile LikelyExpiredState likelyExpiredState = LikelyExpiredState.EXPIRED;@@ -181,7 +182,7 @@ public class ConnectionManager implements Watcher {this,new ZkClientConnectionStrategy.ZkUpdate() {@Override- public void update(SolrZooKeeper keeper) {+ public void update(ZooKeeper keeper) {try {waitForConnected(Long.MAX_VALUE);@@ -206,6 +207,17 @@ public class ConnectionManager implements Watcher {throw new RuntimeException(e1);}}++ private void closeKeeper(ZooKeeper keeper) {+ try {+ keeper.close();+ } catch (InterruptedException e) {+ // Restore the interrupted status+ Thread.currentThread().interrupt();+ log.error("", e);+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);+ }+ }});break;@@ -304,15 +316,4 @@ public class ConnectionManager implements Watcher {throw new TimeoutException("Did not disconnect");}}-- private void closeKeeper(SolrZooKeeper keeper) {- try {- keeper.close();- } catch (InterruptedException e) {- // Restore the interrupted status- Thread.currentThread().interrupt();- log.error("", e);- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);- }- }}diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DefaultConnectionStrategy.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DefaultConnectionStrategy.javaindex d0797a67d82..42d8cbe31e9 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/DefaultConnectionStrategy.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DefaultConnectionStrategy.java@@ -21,6 +21,7 @@ import java.lang.invoke.MethodHandles;import java.util.concurrent.TimeoutException;import org.apache.solr.common.AlreadyClosedException;import org.apache.zookeeper.Watcher;+import org.apache.zookeeper.ZooKeeper;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -32,7 +33,7 @@ public class DefaultConnectionStrategy extends ZkClientConnectionStrategy {@Overridepublic void connect(String serverAddress, int timeout, Watcher watcher, ZkUpdate updater)throws IOException, InterruptedException, TimeoutException {- SolrZooKeeper zk = createSolrZooKeeper(serverAddress, timeout, watcher);+ ZooKeeper zk = createZooKeeper(serverAddress, timeout, watcher);boolean success = false;try {updater.update(zk);@@ -52,7 +53,7 @@ public class DefaultConnectionStrategy extends ZkClientConnectionStrategy {final ZkUpdate updater)throws IOException, InterruptedException, TimeoutException {log.warn("Connection expired - starting a new one...");- SolrZooKeeper zk = createSolrZooKeeper(serverAddress, zkClientTimeout, watcher);+ ZooKeeper zk = createZooKeeper(serverAddress, zkClientTimeout, watcher);boolean success = false;try {updater.update(zk);diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.javaindex e3b2bc99763..0222e2f115d 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java@@ -57,6 +57,7 @@ import org.slf4j.LoggerFactory;* All Solr ZooKeeper interactions should go through this class rather than ZooKeeper. This class* handles synchronous connects and reconnections.*/+// The constructor overloads are a little awkward, it would be nice to move this to a builderpublic class SolrZkClient implements Closeable {static final String NEWL = System.getProperty("line.separator");@@ -67,7 +68,7 @@ public class SolrZkClient implements Closeable {private ConnectionManager connManager;- private volatile SolrZooKeeper keeper;+ private volatile ZooKeeper keeper;private ZkCmdExecutor zkCmdExecutor;@@ -93,29 +94,19 @@ public class SolrZkClient implements Closeable {public SolrZkClient() {}public SolrZkClient(String zkServerAddress, int zkClientTimeout) {- this(zkServerAddress, zkClientTimeout, new DefaultConnectionStrategy(), null);+ this(zkServerAddress, zkClientTimeout, DEFAULT_CLIENT_CONNECT_TIMEOUT);}public SolrZkClient(String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout) {- this(- zkServerAddress,- zkClientTimeout,- zkClientConnectTimeout,- new DefaultConnectionStrategy(),- null);+ this(zkServerAddress, zkClientTimeout, zkClientConnectTimeout, null);}public SolrZkClient(String zkServerAddress,int zkClientTimeout,int zkClientConnectTimeout,- OnReconnect onReonnect) {- this(- zkServerAddress,- zkClientTimeout,- zkClientConnectTimeout,- new DefaultConnectionStrategy(),- onReonnect);+ OnReconnect onReconnect) {+ this(zkServerAddress, zkClientTimeout, zkClientConnectTimeout, null, onReconnect);}public SolrZkClient(@@ -143,24 +134,6 @@ public class SolrZkClient implements Closeable {null);}- public SolrZkClient(- String zkServerAddress,- int zkClientTimeout,- int clientConnectTimeout,- ZkClientConnectionStrategy strat,- final OnReconnect onReconnect,- BeforeReconnect beforeReconnect) {- this(- zkServerAddress,- zkClientTimeout,- clientConnectTimeout,- strat,- onReconnect,- beforeReconnect,- null,- null);- }-public SolrZkClient(String zkServerAddress,int zkClientTimeout,@@ -173,7 +146,9 @@ public class SolrZkClient implements Closeable {this.zkServerAddress = zkServerAddress;this.higherLevelIsClosed = higherLevelIsClosed;if (strat == null) {- strat = new DefaultConnectionStrategy();+ String connectionStrategy = System.getProperty("solr.zookeeper.connectionStrategy");+ strat =+ ZkClientConnectionStrategy.forName(connectionStrategy, new DefaultConnectionStrategy());}this.zkClientConnectionStrategy = strat;@@ -185,16 +160,7 @@ public class SolrZkClient implements Closeable {this.zkClientTimeout = zkClientTimeout;// we must retry at least as long as the session timeout- zkCmdExecutor =- new ZkCmdExecutor(- zkClientTimeout,- new IsClosed() {-- @Override- public boolean isClosed() {- return SolrZkClient.this.isClosed();- }- });+ zkCmdExecutor = new ZkCmdExecutor(zkClientTimeout, SolrZkClient.this::isClosed);connManager =new ConnectionManager("ZooKeeperConnection Watcher:" + zkServerAddress,@@ -203,13 +169,7 @@ public class SolrZkClient implements Closeable {strat,onReconnect,beforeReconnect,- new IsClosed() {-- @Override- public boolean isClosed() {- return SolrZkClient.this.isClosed();- }- });+ SolrZkClient.this::isClosed);try {strat.connect(@@ -217,7 +177,7 @@ public class SolrZkClient implements Closeable {zkClientTimeout,wrapWatcher(connManager),zooKeeper -> {- SolrZooKeeper oldKeeper = keeper;+ ZooKeeper oldKeeper = keeper;keeper = zooKeeper;try {closeKeeper(oldKeeper);@@ -738,8 +698,8 @@ public class SolrZkClient implements Closeable {}/** Allows package private classes to update volatile ZooKeeper. */- void updateKeeper(SolrZooKeeper keeper) throws InterruptedException {- SolrZooKeeper oldKeeper = this.keeper;+ void updateKeeper(ZooKeeper keeper) throws InterruptedException {+ ZooKeeper oldKeeper = this.keeper;this.keeper = keeper;if (oldKeeper != null) {oldKeeper.close();@@ -748,11 +708,11 @@ public class SolrZkClient implements Closeable {if (isClosed) this.keeper.close();}- public SolrZooKeeper getSolrZooKeeper() {+ public ZooKeeper getZooKeeper() {return keeper;}- private void closeKeeper(SolrZooKeeper keeper) {+ private void closeKeeper(ZooKeeper keeper) {if (keeper != null) {try {keeper.close();diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.javadeleted file mode 100644index 4d126b4971c..00000000000--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java+++ /dev/null@@ -1,113 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.common.cloud;--import java.io.IOException;-import java.lang.reflect.Field;-import java.lang.reflect.InvocationTargetException;-import java.lang.reflect.Method;-import java.net.SocketAddress;-import java.security.AccessController;-import java.security.PrivilegedAction;-import java.util.Set;-import java.util.concurrent.CopyOnWriteArraySet;-import org.apache.solr.common.util.SuppressForbidden;-import org.apache.zookeeper.ClientCnxn;-import org.apache.zookeeper.Watcher;-import org.apache.zookeeper.ZooKeeper;--// we use this class to expose nasty stuff for tests-@SuppressWarnings({"try"})-public class SolrZooKeeper extends ZooKeeper {- final Set<Thread> spawnedThreads = new CopyOnWriteArraySet<>();-- // for test debug- // static Map<SolrZooKeeper,Exception> clients = new ConcurrentHashMap<SolrZooKeeper,Exception>();-- public SolrZooKeeper(String connectString, int sessionTimeout, Watcher watcher)- throws IOException {- super(connectString, sessionTimeout, watcher);- // clients.put(this, new RuntimeException());- }-- public ClientCnxn getConnection() {- return cnxn;- }-- public SocketAddress getSocketAddress() {- return testableLocalSocketAddress();- }-- public void closeCnxn() {- final Thread t =- new Thread(- new Runnable() {- @Override- public void run() {- try {- AccessController.doPrivileged(- (PrivilegedAction<Void>) this::closeZookeeperChannel);- } finally {- spawnedThreads.remove(Thread.currentThread());- }- }-- @SuppressForbidden(reason = "Hack for Zookeeper needs access to private methods.")- private Void closeZookeeperChannel() {- final ClientCnxn cnxn = getConnection();- synchronized (cnxn) {- try {- final Field sendThreadFld = cnxn.getClass().getDeclaredField("sendThread");- sendThreadFld.setAccessible(true);- Object sendThread = sendThreadFld.get(cnxn);- if (sendThread != null) {- Method method =- sendThread.getClass().getDeclaredMethod("testableCloseSocket");- method.setAccessible(true);- try {- method.invoke(sendThread);- } catch (InvocationTargetException e) {- // is fine- }- }- } catch (Exception e) {- throw new RuntimeException("Closing Zookeeper send channel failed.", e);- }- }- return null; // Void- }- },- "closeCnxn");- spawnedThreads.add(t);- t.start();- }-- @Override- public synchronized void close() throws InterruptedException {- super.close();- }-- // public static void assertCloses() {- // if (clients.size() > 0) {- // Iterator<Exception> stacktraces = clients.values().iterator();- // Exception cause = null;- // cause = stacktraces.next();- // throw new RuntimeException("Found a bad one!", cause);- // }- // }--}diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.javaindex 1ca3b55da04..47776b4eaaf 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java@@ -24,6 +24,7 @@ import java.util.concurrent.TimeoutException;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.ZkCredentialsProvider.ZkCredentials;import org.apache.zookeeper.Watcher;+import org.apache.zookeeper.ZooKeeper;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -90,7 +91,7 @@ public abstract class ZkClientConnectionStrategy {}public interface ZkUpdate {- void update(SolrZooKeeper zooKeeper) throws InterruptedException, TimeoutException, IOException;+ void update(ZooKeeper zooKeeper) throws InterruptedException, TimeoutException, IOException;}public void setZkCredentialsToAddAutomatically(@@ -109,10 +110,10 @@ public abstract class ZkClientConnectionStrategy {return zkCredentialsToAddAutomatically;}- protected SolrZooKeeper createSolrZooKeeper(+ protected ZooKeeper createZooKeeper(final String serverAddress, final int zkClientTimeout, final Watcher watcher)throws IOException {- SolrZooKeeper result = new SolrZooKeeper(serverAddress, zkClientTimeout, watcher);+ ZooKeeper result = newZooKeeperInstance(serverAddress, zkClientTimeout, watcher);zkCredentialsToAddAutomaticallyUsed = true;for (ZkCredentials zkCredentials : zkCredentialsToAddAutomatically.getCredentials()) {@@ -121,4 +122,35 @@ public abstract class ZkClientConnectionStrategy {return result;}++ // Override for testing+ protected ZooKeeper newZooKeeperInstance(+ final String serverAddress, final int zkClientTimeout, final Watcher watcher)+ throws IOException {+ return new ZooKeeper(serverAddress, zkClientTimeout, watcher);+ }++ /**+ * Instantiate a new connection strategy for the given class name+ *+ * @param name the name of the strategy to use+ * @return the strategy instance, or null if it could not be loaded+ */+ public static ZkClientConnectionStrategy forName(String name, ZkClientConnectionStrategy def) {+ log.debug("Attempting to load zk connection strategy '{}'", name);+ if (name == null) {+ return def;+ }++ try {+ // TODO should this use SolrResourceLoader?+ return Class.forName(name)+ .asSubclass(ZkClientConnectionStrategy.class)+ .getConstructor()+ .newInstance();+ } catch (Exception e) {+ log.warn("Exception when loading '{}' ZK connection strategy.", name, e);+ return def;+ }+ }}diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.javaindex 649091c427b..c4d3b6c5788 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java@@ -2221,7 +2221,7 @@ public class ZkStateReader implements SolrCloseable {public boolean update() throws KeeperException, InterruptedException {log.debug("Checking ZK for most up to date Aliases {}", ALIASES);// Call sync() first to ensure the subsequent read (getData) is up to date.- zkClient.getSolrZooKeeper().sync(ALIASES, null, null);+ zkClient.getZooKeeper().sync(ALIASES, null, null);Stat stat = new Stat();final byte[] data = zkClient.getData(ALIASES, null, stat, true);return setIfNewer(Aliases.fromJSON(data, stat.getVersion()));diff --git a/solr/test-framework/build.gradle b/solr/test-framework/build.gradleindex 569f0e275c5..bdb078520df 100644--- a/solr/test-framework/build.gradle+++ b/solr/test-framework/build.gradle@@ -38,15 +38,17 @@ dependencies {implementation 'org.apache.lucene:lucene-queries'implementation 'org.apache.lucene:lucene-suggest'- implementation('org.apache.zookeeper:zookeeper', {++ var zkExcludes = {exclude group: "org.apache.yetus", module: "audience-annotations"exclude group: "log4j", module: "log4j"exclude group: "org.slf4j", module: "slf4j-log4j12"- })- implementation ('org.apache.zookeeper:zookeeper-jute') {- exclude group: 'org.apache.yetus', module: 'audience-annotations'}+ implementation('org.apache.zookeeper:zookeeper', zkExcludes)+ implementation('org.apache.zookeeper:zookeeper-jute', zkExcludes)+ implementation('org.apache.zookeeper:zookeeper::tests', zkExcludes)+implementation 'commons-io:commons-io'implementation 'org.slf4j:slf4j-api'implementation 'org.apache.logging.log4j:log4j-api'diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.javaindex 99a6960338a..bdd127356a7 100644--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java@@ -108,6 +108,7 @@ import org.apache.solr.client.solrj.response.SolrResponseBase;import org.apache.solr.client.solrj.util.ClientUtils;import org.apache.solr.cloud.IpTables;import org.apache.solr.cloud.MiniSolrCloudCluster;+import org.apache.solr.cloud.TestConnectionStrategy;import org.apache.solr.common.SolrDocument;import org.apache.solr.common.SolrDocumentList;import org.apache.solr.common.SolrException;@@ -301,6 +302,7 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {System.setProperty("solr.v2RealPath", "true");System.setProperty("zookeeper.forceSync", "no");System.setProperty("jetty.testMode", "true");+ System.setProperty("solr.zookeeper.connectionStrategy", TestConnectionStrategy.class.getName());System.setProperty("enable.update.log", Boolean.toString(usually()));System.setProperty("tests.shardhandler.randomSeed", Long.toString(random().nextLong()));System.setProperty("solr.clustering.enabled", "false");diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractVMParamsZkACLAndCredentialsProvidersTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractVMParamsZkACLAndCredentialsProvidersTestBase.javaindex 35247b5b3a5..21e9b134e12 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractVMParamsZkACLAndCredentialsProvidersTestBase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractVMParamsZkACLAndCredentialsProvidersTestBase.java@@ -77,12 +77,7 @@ public class AbstractVMParamsZkACLAndCredentialsProvidersTestBase extends SolrTeSolrZkClient zkClient =new SolrZkClient(- zkServer.getZkHost(),- AbstractZkTestCase.TIMEOUT,- AbstractZkTestCase.TIMEOUT,- null,- null,- null);+ zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT);zkClient.makePath("/solr", false, true);zkClient.close();@@ -107,7 +102,7 @@ public class AbstractVMParamsZkACLAndCredentialsProvidersTestBase extends SolrTe// no (or completely open) ACLs added. Therefore hack your way into being authorized for// creating anywayzkClient- .getSolrZooKeeper()+ .getZooKeeper().addAuthInfo("digest",("connectAndAllACLUsername:connectAndAllACLPassword").getBytes(StandardCharsets.UTF_8));@@ -204,7 +199,7 @@ public class AbstractVMParamsZkACLAndCredentialsProvidersTestBase extends SolrTe// no (or completely open) ACLs added. Therefore hack your way into being authorized for// creating anywayzkClient- .getSolrZooKeeper()+ .getZooKeeper().addAuthInfo("digest",("connectAndAllACLUsername:connectAndAllACLPassword")diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.javaindex 020445142c9..25babc2ed57 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java@@ -16,6 +16,7 @@*/package org.apache.solr.cloud;+import java.io.IOException;import java.lang.invoke.MethodHandles;import java.util.ArrayList;import java.util.List;@@ -47,6 +48,8 @@ import org.apache.solr.util.RTimer;import org.apache.solr.util.TestInjection;import org.apache.solr.util.TimeOut;import org.apache.zookeeper.KeeperException;+import org.apache.zookeeper.TestableZooKeeper;+import org.apache.zookeeper.ZooKeeper;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -149,8 +152,7 @@ public class ChaosMonkey {if (cores != null) {monkeyLog("expire session for " + jetty.getLocalPort() + " !");causeConnectionLoss(jetty);- long sessionId = cores.getZkController().getZkClient().getSolrZooKeeper().getSessionId();- zkServer.expire(sessionId);+ zkServer.expire(cores.getZkController().getZkClient().getZooKeeper().getSessionId());}}@@ -180,7 +182,21 @@ public class ChaosMonkey {if (cores != null) {monkeyLog("Will cause connection loss on " + jetty.getLocalPort());SolrZkClient zkClient = cores.getZkController().getZkClient();- zkClient.getSolrZooKeeper().closeCnxn();+ causeConnectionLoss(zkClient.getZooKeeper());+ }+ }++ public static void causeConnectionLoss(ZooKeeper zooKeeper) {+ assert zooKeeper instanceof TestableZooKeeper+ : "Can only cause connection loss for TestableZookeeper";+ if (zooKeeper instanceof TestableZooKeeper) {+ try {+ ((TestableZooKeeper) zooKeeper).testableConnloss();+ } catch (IOException ignored) {+ // best effort+ }+ } else {+ // TODO what now?}}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.javaindex cc4c230c367..5cb076e0dcc 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java@@ -790,16 +790,14 @@ public class MiniSolrCloudCluster {"Cannot find Jetty for a replica with core url " + replica.getCoreUrl());}- /** Make the zookeeper session on a particular jetty expire */+ /** Make the zookeeper session on a particular jetty lose connection and expire */public void expireZkSession(JettySolrRunner jetty) {CoreContainer cores = jetty.getCoreContainer();if (cores != null) {- SolrZkClient zkClient = cores.getZkController().getZkClient();- zkClient.getSolrZooKeeper().closeCnxn();- long sessionId = zkClient.getSolrZooKeeper().getSessionId();- zkServer.expire(sessionId);+ ChaosMonkey.causeConnectionLoss(jetty);+ zkServer.expire(cores.getZkController().getZkClient().getZooKeeper().getSessionId());if (log.isInfoEnabled()) {- log.info("Expired zookeeper session {} from node {}", sessionId, jetty.getBaseUrl());+ log.info("Expired zookeeper session from node {}", jetty.getBaseUrl());}}}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/TestConnectionStrategy.java b/solr/test-framework/src/java/org/apache/solr/cloud/TestConnectionStrategy.javanew file mode 100644index 00000000000..88d94faaa85--- /dev/null+++ b/solr/test-framework/src/java/org/apache/solr/cloud/TestConnectionStrategy.java@@ -0,0 +1,35 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cloud;++import java.io.IOException;+import org.apache.solr.common.cloud.DefaultConnectionStrategy;+import org.apache.zookeeper.TestableZooKeeper;+import org.apache.zookeeper.Watcher;+import org.apache.zookeeper.ZooKeeper;++/**+ * Connection strategy that creates instances of {@link TestableZooKeeper} instead of plain {@link+ * ZooKeeper} objects. Useful for adding pause and disconnect events.+ */+public class TestConnectionStrategy extends DefaultConnectionStrategy {+ @Override+ protected ZooKeeper newZooKeeperInstance(+ String serverAddress, int zkClientTimeout, Watcher watcher) throws IOException {+ return new TestableZooKeeper(serverAddress, zkClientTimeout, watcher);+ }+}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.javaindex ed6d8badb63..1271baa9009 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java@@ -36,7 +36,6 @@ import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicReference;import javax.management.JMException;import org.apache.solr.SolrTestCaseJ4;@@ -44,25 +43,25 @@ import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.util.IOUtils;import org.apache.solr.common.util.ObjectReleaseTracker;-import org.apache.solr.common.util.TimeSource;import org.apache.solr.common.util.Utils;-import org.apache.solr.util.TimeOut;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;import org.apache.zookeeper.Op;import org.apache.zookeeper.WatchedEvent;import org.apache.zookeeper.Watcher;+import org.apache.zookeeper.ZooDefs;import org.apache.zookeeper.data.Stat;import org.apache.zookeeper.jmx.ManagedUtil;import org.apache.zookeeper.server.NIOServerCnxnFactory;+import org.apache.zookeeper.server.Request;import org.apache.zookeeper.server.ServerCnxn;import org.apache.zookeeper.server.ServerCnxnFactory;import org.apache.zookeeper.server.ServerConfig;-import org.apache.zookeeper.server.SessionTracker.Session;import org.apache.zookeeper.server.ZKDatabase;import org.apache.zookeeper.server.ZooKeeperServer;import org.apache.zookeeper.server.persistence.FileTxnSnapLog;import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;+import org.apache.zookeeper.test.ClientBase;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -383,7 +382,7 @@ public class ZkTestServer {try {int port = cnxnFactory.getLocalPort();if (port > 0) {- waitForServerDown(getZkHost(), 30000);+ ClientBase.waitForServerDown(getZkHost(), 30000);}} catch (NullPointerException ignored) {// server never successfully started@@ -498,23 +497,9 @@ public class ZkTestServer {}public void expire(final long sessionId) {- zkServer.zooKeeperServer.expire(- new Session() {- @Override- public long getSessionId() {- return sessionId;- }-- @Override- public int getTimeout() {- return 4000;- }-- @Override- public boolean isClosing() {- return false;- }- });+ log.debug("Closing zookeeper connection for session {}", sessionId);+ Request si = new Request(null, sessionId, 0, ZooDefs.OpCode.closeSession, null, null);+ zkServer.zooKeeperServer.submitRequest(si);}public ZKDatabase getZKDatabase() {@@ -605,7 +590,7 @@ public class ZkTestServer {}log.info("start zk server on port: {}", port);- waitForServerUp(getZkHost(), 30000);+ ClientBase.waitForServerUp(getZkHost(), 30000);init(solrFormat);} catch (Exception e) {@@ -658,51 +643,6 @@ public class ZkTestServer {ObjectReleaseTracker.release(this);}- public static boolean waitForServerDown(String hp, long timeoutMs) {- log.info("waitForServerDown: {}", hp);- final TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);- while (true) {- try {- HostPort hpobj = parseHostPortList(hp).get(0);- send4LetterWord(hpobj.host, hpobj.port, "stat");- } catch (IOException e) {- return true;- }-- if (timeout.hasTimedOut()) {- throw new RuntimeException("Time out waiting for ZooKeeper shutdown!");- }- try {- Thread.sleep(250);- } catch (InterruptedException e) {- // ignore- }- }- }-- public static boolean waitForServerUp(String hp, long timeoutMs) {- log.info("waitForServerUp: {}", hp);- final TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);- while (true) {- try {- HostPort hpobj = parseHostPortList(hp).get(0);- send4LetterWord(hpobj.host, hpobj.port, "stat");- return true;- } catch (IOException e) {- e.printStackTrace();- }-- if (timeout.hasTimedOut()) {- throw new RuntimeException("Time out waiting for ZooKeeper to startup!");- }- try {- Thread.sleep(250);- } catch (InterruptedException e) {- // ignore- }- }- }-public static class HostPort {String host;int port;diff --git a/solr/test-framework/src/test/org/apache/solr/cloud/MiniSolrCloudClusterTest.java b/solr/test-framework/src/test/org/apache/solr/cloud/MiniSolrCloudClusterTest.javaindex 0c19564e3e6..0c0bb5317d5 100644--- a/solr/test-framework/src/test/org/apache/solr/cloud/MiniSolrCloudClusterTest.java+++ b/solr/test-framework/src/test/org/apache/solr/cloud/MiniSolrCloudClusterTest.java@@ -65,9 +65,12 @@ public class MiniSolrCloudClusterTest extends SolrTestCaseJ4 {};fail("Expected an exception to be thrown from MiniSolrCloudCluster");} catch (Exception e) {- assertEquals("Error starting up MiniSolrCloudCluster", e.getMessage());+ assertEquals("Incorrect exception", "Error starting up MiniSolrCloudCluster", e.getMessage());assertEquals("Expected one suppressed exception", 1, e.getSuppressed().length);- assertEquals("Fake exception on startup!", e.getSuppressed()[0].getMessage());+ assertEquals(+ "Incorrect suppressed exception",+ "Fake exception on startup!",+ e.getSuppressed()[0].getMessage());} finally {if (cluster != null) cluster.shutdown();}