Apache Solr open-source search software
Continue cleanups post removing HDFS from Solr (#3913)
Eric Pugh98cf180ba24b
diff --git a/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.javaindex fdd48350f7a..c1ef93b790e 100644--- a/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java+++ b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java@@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;public class BlockDirectory extends FilterDirectory implements ShutdownAwareDirectory {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- public static final long BLOCK_SHIFT = Integer.getInteger("solr.hdfs.blockcache.blockshift", 13);+ public static final long BLOCK_SHIFT = Integer.getInteger("solr.blockcache.blockshift", 13);public static final int BLOCK_SIZE = 1 << BLOCK_SHIFT;public static final long BLOCK_MOD = BLOCK_SIZE - 1L;diff --git a/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java b/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.javaindex 79d44d0fc6b..df3f1a058b2 100644--- a/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java+++ b/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java@@ -28,7 +28,7 @@ import org.apache.lucene.store.IndexOutput;public abstract class CustomBufferedIndexInput extends IndexInput {public static final int BUFFER_SIZE =- Integer.getInteger("solr.hdfs.readbuffer.size.default", 32768);+ Integer.getInteger("solr.blockcache.readbuffer.size.default", 32768);private int bufferSize = BUFFER_SIZE;diff --git a/solr/core/src/java/org/apache/solr/blockcache/Metrics.java b/solr/core/src/java/org/apache/solr/blockcache/Metrics.javaindex ed820fbb500..4070cd557c2 100644--- a/solr/core/src/java/org/apache/solr/blockcache/Metrics.java+++ b/solr/core/src/java/org/apache/solr/blockcache/Metrics.java@@ -157,12 +157,12 @@ public class Metrics extends SolrCacheBase implements SolrInfoBean {@Overridepublic String getName() {- return "hdfsBlockCache";+ return "blockCache";}@Overridepublic String getDescription() {- return "Provides metrics for the HdfsDirectoryFactory BlockCache.";+ return "Provides metrics for the BlockCache.";}@Overridediff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.javaindex 48f065f537e..9bcae03e812 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java@@ -154,8 +154,8 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {replica.getBool(ZkStateReader.SHARED_STORAGE_PROP, false) && dataDir != null;if (isSharedFS && inPlaceMove) {- log.debug("-- moveHdfsReplica");- moveHdfsReplica(+ log.debug("-- moveSharedFsReplica");+ moveSharedFsReplica(clusterState,results,dataDir.toString(),@@ -179,7 +179,7 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {}}- private void moveHdfsReplica(+ private void moveSharedFsReplica(ClusterState clusterState,NamedList<Object> results,String dataDir,diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.javaindex 464c47cc53e..8ecd634b757 100644--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java@@ -446,6 +446,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {// `init(UpdateHandler, SolrCore` is never actually called concurrently in application code// (`TestHdfsUpdateLog.testFSThreadSafety()`, introduced by SOLR-7113, seems to be the only// place that requires true thread safety from this method?).+ // HDFS was removed in Solr 10, and therefore the test referenced is gone as well.if (debug) {log.debug("UpdateHandler init: tlogDir={}, next id={} this is a reopen or double init ... nothing else to do.",@@ -509,8 +510,8 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {/*** Resolves any relative path wrt the highest core-scoped level (whatever that means for a- * particular implementation). For most filesystems, this will be the core instanceDir, but there- * are other cases; e.g., HdfsUpdateLog will resolve paths relative to the core dataDir.+ * particular implementation). For most filesystems, this will be the core instanceDir, but that+ * is not a hard and fast rule.** <p>If the input path is already absolute, it will be returned unmodified.*diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.javaindex df0c8f98a2e..34532421bf5 100644--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java@@ -1378,7 +1378,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {// Streaming updates can delay shutdown and cause big update reorderings (new streams can't be// initiated, but existing streams carry on). This is why we check if the CC is shutdown.- // See SOLR-8203 and loop HdfsChaosMonkeyNothingIsSafeTest (and check for inconsistent shards)+ // See SOLR-8203 and loop ChaosMonkeyNothingIsSafeTest (and check for inconsistent shards)// to test.if (req.getCoreContainer().isShutDown()) {throw new SolrException(diff --git a/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java b/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.javaindex dbd69bca4d9..7a1a0c16fa8 100644--- a/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java+++ b/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java@@ -142,10 +142,10 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {String name = "test.eof";createFile(name, fsDir, directory);long fsLength = fsDir.fileLength(name);- long hdfsLength = directory.fileLength(name);- assertEquals(fsLength, hdfsLength);+ long blockLength = directory.fileLength(name);+ assertEquals(fsLength, blockLength);testEof(name, fsDir, fsLength);- testEof(name, directory, hdfsLength);+ testEof(name, directory, blockLength);fsDir.close();}@@ -189,11 +189,12 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {testRandomAccessWrites();}- private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) throws IOException {+ private void assertInputsEquals(String name, Directory fsDir, Directory blockDirectory)+ throws IOException {int reads = random.nextInt(MAX_NUMBER_OF_READS);IndexInput fsInput = fsDir.openInput(name, IOContext.DEFAULT);- IndexInput hdfsInput = hdfs.openInput(name, IOContext.DEFAULT);- assertEquals(fsInput.length(), hdfsInput.length());+ IndexInput blockInput = blockDirectory.openInput(name, IOContext.DEFAULT);+ assertEquals(fsInput.length(), blockInput.length());int fileLength = (int) fsInput.length();for (int i = 0; i < reads; i++) {int rnd;@@ -204,7 +205,7 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {}byte[] fsBuf = new byte[rnd + MIN_BUFFER_SIZE];- byte[] hdfsBuf = new byte[fsBuf.length];+ byte[] blockBuf = new byte[fsBuf.length];int offset = random.nextInt(fsBuf.length);int length = random.nextInt(fsBuf.length - offset);@@ -217,23 +218,24 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {fsInput.seek(pos);fsInput.readBytes(fsBuf, offset, length);- hdfsInput.seek(pos);- hdfsInput.readBytes(hdfsBuf, offset, length);+ blockInput.seek(pos);+ blockInput.readBytes(blockBuf, offset, length);for (int f = offset; f < length; f++) {- if (fsBuf[f] != hdfsBuf[f]) {+ if (fsBuf[f] != blockBuf[f]) {fail("read [" + i + "]");}}}fsInput.close();- hdfsInput.close();+ blockInput.close();}- private void createFile(String name, Directory fsDir, Directory hdfs) throws IOException {+ private void createFile(String name, Directory fsDir, Directory blockDirectory)+ throws IOException {int writes = random.nextInt(MAX_NUMBER_OF_WRITES);int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE;IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT);- IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT);+ IndexOutput blockOutput = blockDirectory.createOutput(name, IOContext.DEFAULT);for (int i = 0; i < writes; i++) {byte[] buf =new byte@@ -243,10 +245,10 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {int offset = random.nextInt(buf.length);int length = random.nextInt(buf.length - offset);fsOutput.writeBytes(buf, offset, length);- hdfsOutput.writeBytes(buf, offset, length);+ blockOutput.writeBytes(buf, offset, length);}fsOutput.close();- hdfsOutput.close();+ blockOutput.close();}private String getName() {diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.javaindex b99330645f1..d4ad2628c32 100644--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java@@ -16,21 +16,474 @@*/package org.apache.solr.cloud;+import static org.apache.solr.cloud.SolrCloudTestCase.configurePrsDefault;++import java.nio.file.Files;+import java.nio.file.Path;+import java.util.concurrent.TimeUnit;+import org.apache.lucene.tests.mockfile.FilterPath;+import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.solr.SolrTestCaseJ4.SuppressSSL;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.cloud.ZkNodeProps;+import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.params.CommonParams;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.embedded.JettySolrRunner;+import org.apache.solr.handler.BackupStatusChecker;+import org.apache.solr.handler.ReplicationHandler;+import org.junit.BeforeClass;import org.junit.Test;/*** This test simply does a bunch of basic things in solrcloud mode and asserts things work as- * expected. Implementation moved to AbstractBasicDistributedZk2TestBase as it is used by HDFS- * contrib module tests.+ * expected.*/+// Backups do checksum validation against a footer value not present in 'SimpleText'[email protected]({"SimpleText"})@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")-public class BasicDistributedZk2Test extends AbstractBasicDistributedZk2TestBase {+public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {+ private static final String SHARD2 = "shard2";+ private static final String SHARD1 = "shard1";+ private static final String ONE_NODE_COLLECTION = "onenodecollection";+ private final boolean onlyLeaderIndexes = random().nextBoolean();++ public BasicDistributedZk2Test() {+ super();+ // we need DVs on point fields to compute stats & facets+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP))+ System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");++ sliceCount = 2;+ }- @Test@Override+ protected boolean useTlogReplicas() {+ return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug+ // and it's TestInjection use+ }++ @BeforeClass+ public static void _setPrsDefault() {+ configurePrsDefault();+ }++ @Test@ShardsFixed(num = 4)public void test() throws Exception {- super.test();+ boolean testFinished = false;+ try {+ handle.clear();+ handle.put("timestamp", SKIPVAL);++ testNodeWithoutCollectionForwarding();++ indexr(+ id,+ 1,+ i1,+ 100,+ tlong,+ 100,+ t1,+ "now is the time for all good men",+ "foo_f",+ 1.414f,+ "foo_b",+ "true",+ "foo_d",+ 1.414d);++ commit();++ // make sure we are in a steady state...+ waitForRecoveriesToFinish(false);++ assertDocCounts(false);++ indexAbunchOfDocs();++ // check again+ waitForRecoveriesToFinish(false);++ commit();++ assertDocCounts(VERBOSE);+ checkQueries();++ assertDocCounts(VERBOSE);++ query("q", "*:*", "sort", "n_tl1 desc");++ bringDownShardIndexSomeDocsAndRecover();++ query("q", "*:*", "sort", "n_tl1 desc");++ // test adding another replica to a shard - it should do a+ // recovery/replication to pick up the index from the leader+ addNewReplica();++ long docId = testUpdateAndDelete();++ // index a bad doc...+ expectThrows(SolrException.class, () -> indexr(t1, "a doc with no id"));++ // TODO: bring this to its own method?+ // try indexing to a leader that has no replicas up+ ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);+ ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, SHARD2);++ String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP);+ chaosMonkey.stopShardExcept(SHARD2, nodeName);++ SolrClient client = getClient(nodeName);++ index_specific(client, "id", docId + 1, t1, "what happens here?");++ // expire a session...+ CloudJettyRunner cloudJetty = shardToJetty.get(SHARD1).get(0);+ chaosMonkey.expireSession(cloudJetty.jetty);+ // Wait until the jetty is reconnected, otherwise the following index command could fail+ cloudJetty+ .jetty+ .getCoreContainer()+ .getZkController()+ .getZkClient()+ .getCuratorFramework()+ .blockUntilConnected(50, TimeUnit.MILLISECONDS);++ indexr("id", docId + 1, t1, "slip this doc in");++ waitForRecoveriesToFinish(false);++ checkShardConsistency(SHARD1);+ checkShardConsistency(SHARD2);++ testFinished = true;+ } finally {+ if (!testFinished) {+ printLayoutOnTearDown = true;+ }+ }+ }++ private void testNodeWithoutCollectionForwarding() throws Exception {+ assertEquals(+ 0,+ CollectionAdminRequest.createCollection(ONE_NODE_COLLECTION, "conf1", 1, 1)+ .setCreateNodeSet("")+ .process(cloudClient)+ .getStatus());+ assertTrue(+ CollectionAdminRequest.addReplicaToShard(ONE_NODE_COLLECTION, "shard1")+ .setCoreName(ONE_NODE_COLLECTION + "core")+ .process(cloudClient)+ .isSuccess());++ waitForCollection(ZkStateReader.from(cloudClient), ONE_NODE_COLLECTION, 1);+ waitForRecoveriesToFinish(ONE_NODE_COLLECTION, ZkStateReader.from(cloudClient), false);++ ZkStateReader.from(cloudClient).getLeaderRetry(ONE_NODE_COLLECTION, SHARD1, 30000);++ int docs = 2;+ for (JettySolrRunner jetty : jettys) {+ final String clientUrl = getBaseUrl(jetty);+ addAndQueryDocs(clientUrl, docs);+ docs += 2;+ }+ }++ // 2 docs added every call+ private void addAndQueryDocs(final String baseUrl, int docs) throws Exception {++ SolrQuery query = new SolrQuery("*:*");++ try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollection")) {+ // add a doc+ client.add(sdoc("id", docs));+ client.commit();++ QueryResponse results = client.query(query);+ assertEquals(docs - 1, results.getResults().getNumFound());++ SolrInputDocument doc = new SolrInputDocument();+ doc.addField("id", docs + 1);+ client.add(doc);+ client.commit();++ query = new SolrQuery("*:*");+ query.set("rows", 0);+ results = client.query(query);+ assertEquals(docs, results.getResults().getNumFound());+ }+ }++ private long testUpdateAndDelete() throws Exception {+ long docId = 99999999L;+ indexr("id", docId, t1, "originalcontent");++ commit();++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.add("q", t1 + ":originalcontent");+ QueryResponse results = clients.get(0).query(params);+ assertEquals(1, results.getResults().getNumFound());++ // update doc+ indexr("id", docId, t1, "updatedcontent");++ commit();++ results = clients.get(0).query(params);+ assertEquals(0, results.getResults().getNumFound());++ params.set("q", t1 + ":updatedcontent");++ results = clients.get(0).query(params);+ assertEquals(1, results.getResults().getNumFound());++ UpdateRequest uReq = new UpdateRequest();+ // uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN);+ uReq.deleteById(Long.toString(docId)).process(clients.get(0));++ commit();++ results = clients.get(0).query(params);+ assertEquals(0, results.getResults().getNumFound());+ return docId;+ }++ private void bringDownShardIndexSomeDocsAndRecover() throws Exception {+ SolrQuery query = new SolrQuery("*:*");+ query.set("distrib", false);++ commit();++ long deadShardCount =+ shardToJetty.get(SHARD2).get(0).client.solrClient.query(query).getResults().getNumFound();++ query("q", "*:*", "sort", "n_tl1 desc");++ int oldLiveNodes =+ ZkStateReader.from(cloudClient)+ .getZkClient()+ .getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true)+ .size();++ assertEquals(5, oldLiveNodes);++ // kill a shard+ CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0);++ // ensure shard is dead+ expectThrows(+ SolrServerException.class,+ "This server should be down and this update should have failed",+ () -> index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1, "specific doc!"));++ commit();++ query("q", "*:*", "sort", "n_tl1 desc");++ // long cloudClientDocs = cloudClient.query(new+ // SolrQuery("*:*")).getResults().getNumFound();+ // System.out.println("clouddocs:" + cloudClientDocs);++ // try to index to a living shard at shard2++ long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();++ ZkStateReader.from(cloudClient).getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 60000);++ try {+ index_specific(+ shardToJetty.get(SHARD1).get(1).client.solrClient,+ id,+ 1000,+ i1,+ 108,+ t1,+ "specific doc!");+ } catch (Exception e) {+ // wait and try again+ Thread.sleep(4000);+ index_specific(+ shardToJetty.get(SHARD1).get(1).client.solrClient,+ id,+ 1000,+ i1,+ 108,+ t1,+ "specific doc!");+ }++ commit();++ checkShardConsistency(true, false);++ query("q", "*:*", "sort", "n_tl1 desc");++ long numFound2 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();++ assertEquals(numFound1 + 1, numFound2);++ SolrInputDocument doc = new SolrInputDocument();+ doc.addField("id", 1001);++ controlClient.add(doc);++ // try adding a doc with CloudSolrServer+ UpdateRequest ureq = new UpdateRequest();+ ureq.add(doc);+ // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);++ try {+ ureq.process(cloudClient);+ } catch (SolrServerException e) {+ // try again+ Thread.sleep(3500);+ ureq.process(cloudClient);+ }++ commit();++ query("q", "*:*", "sort", "n_tl1 desc");++ long numFound3 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();++ // lets just check that the one doc since last commit made it in...+ assertEquals(numFound2 + 1, numFound3);++ // test debugging+ testDebugQueries();++ if (VERBOSE) {+ System.err.println(controlClient.query(new SolrQuery("*:*")).getResults().getNumFound());++ for (SolrClient client : clients) {+ try {+ SolrQuery q = new SolrQuery("*:*");+ q.set("distrib", false);+ System.err.println(client.query(q).getResults().getNumFound());+ } catch (Exception e) {++ }+ }+ }+ // TODO: This test currently fails because debug info is obtained only+ // on shards with matches.+ // query("q","matchesnothing","fl","*,score", "debugQuery", "true");++ // this should trigger a recovery phase on deadShard+ deadShard.jetty.start();++ // make sure we have published we are recovering+ Thread.sleep(1500);++ waitForRecoveriesToFinish(false);++ deadShardCount =+ shardToJetty.get(SHARD1).get(0).client.solrClient.query(query).getResults().getNumFound();+ // if we properly recovered, we should now have the couple missing docs that+ // came in while shard was down+ checkShardConsistency(true, false);++ // recover over 100 docs so we do more than just peer sync (replicate recovery)+ chaosMonkey.stopJetty(deadShard);++ for (int i = 0; i < 226; i++) {+ doc = new SolrInputDocument();+ doc.addField("id", 2000 + i);+ controlClient.add(doc);+ ureq = new UpdateRequest();+ ureq.add(doc);+ // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);+ ureq.process(cloudClient);+ }+ commit();++ Thread.sleep(1500);++ deadShard.jetty.start();++ // make sure we have published we are recovering+ Thread.sleep(1500);++ waitForThingsToLevelOut(1, TimeUnit.MINUTES);++ Thread.sleep(500);++ waitForRecoveriesToFinish(false);++ checkShardConsistency(true, false);++ // try a backup command+ try (final SolrClient client =+ getHttpSolrClient((String) shardToJetty.get(SHARD2).get(0).info.get("base_url"))) {+ final String backupName = "the_backup";+ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("qt", ReplicationHandler.PATH);+ params.set("command", "backup");+ params.set("name", backupName);+ final Path location = FilterPath.unwrap(createTempDir()).toRealPath();+ // Allow non-standard location outside SOLR_HOME+ jettys.forEach(j -> j.getCoreContainer().getAllowPaths().add(location));+ params.set("location", location.toString());++ QueryRequest request = new QueryRequest(params);+ client.request(request, DEFAULT_TEST_COLLECTION_NAME);++ final BackupStatusChecker backupStatus =+ new BackupStatusChecker(client, "/" + DEFAULT_TEST_COLLECTION_NAME + "/replication");+ final String backupDirName = backupStatus.waitForBackupSuccess(backupName, 30);+ assertTrue(+ "Backup dir does not exist: " + backupDirName,+ Files.exists(location.resolve(backupDirName)));+ }+ }++ private void addNewReplica() throws Exception {++ waitForRecoveriesToFinish(false);++ // new server should be part of first shard+ // how many docs are on the new shard?+ for (CloudJettyRunner cjetty : shardToJetty.get(SHARD1)) {+ if (VERBOSE)+ System.err.println(+ "shard1 total:"+ + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());+ }+ for (CloudJettyRunner cjetty : shardToJetty.get(SHARD2)) {+ if (VERBOSE)+ System.err.println(+ "shard2 total:"+ + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());+ }++ checkShardConsistency(SHARD1);+ checkShardConsistency(SHARD2);++ assertDocCounts(VERBOSE);+ }++ private void testDebugQueries() throws Exception {+ handle.put("explain", SKIPVAL);+ handle.put("debug", UNORDERED);+ handle.put("time", SKIPVAL);+ handle.put("track", SKIP);+ query("q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true");+ query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true");+ query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING);+ query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS);+ query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY);}}diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.javaindex 7975cfb68d3..f9779cfb551 100644--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java@@ -16,21 +16,1724 @@*/package org.apache.solr.cloud;+import java.io.IOException;+import java.io.InputStream;+import java.lang.invoke.MethodHandles;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Collections;+import java.util.HashMap;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.concurrent.Callable;+import java.util.concurrent.CompletionService;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorCompletionService;+import java.util.concurrent.Future;+import java.util.concurrent.SynchronousQueue;+import java.util.concurrent.ThreadPoolExecutor;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicLong;+import java.util.concurrent.atomic.AtomicReference;+import org.apache.lucene.util.IOUtils;+import org.apache.solr.JSONTestUtil;import org.apache.solr.SolrTestCaseJ4.SuppressSSL;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.SolrRequest;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.apache.HttpSolrClient;+import org.apache.solr.client.solrj.request.AbstractUpdateRequest;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;+import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;+import org.apache.solr.client.solrj.request.GenericSolrRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.client.solrj.request.StreamingUpdateRequest;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.client.solrj.response.CollectionAdminResponse;+import org.apache.solr.client.solrj.response.FacetField;+import org.apache.solr.client.solrj.response.Group;+import org.apache.solr.client.solrj.response.GroupCommand;+import org.apache.solr.client.solrj.response.GroupResponse;+import org.apache.solr.client.solrj.response.InputStreamResponseParser;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.client.solrj.response.UpdateResponse;+import org.apache.solr.cloud.api.collections.CollectionHandlingUtils;+import org.apache.solr.common.SolrDocument;+import org.apache.solr.common.SolrDocumentList;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.cloud.ClusterState;+import org.apache.solr.common.cloud.DocCollection;+import org.apache.solr.common.cloud.Replica;+import org.apache.solr.common.cloud.Slice;+import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.params.CollectionParams.CollectionAction;+import org.apache.solr.common.params.CommonParams;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.params.SolrParams;+import org.apache.solr.common.params.UpdateParams;+import org.apache.solr.common.util.ExecutorUtil;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.SolrNamedThreadFactory;+import org.apache.solr.embedded.JettySolrRunner;+import org.apache.solr.util.TestInjection;+import org.apache.solr.util.TestInjection.Hook;+import org.junit.BeforeClass;import org.junit.Test;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;/*** This test simply does a bunch of basic things in solrcloud mode and asserts things work as- * expected. Implementation moved to AbstractBasicDistributedZkTestBase as it is used by many HDFS- * contrib tests.+ * expected.*/@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")-public class BasicDistributedZkTest extends AbstractBasicDistributedZkTestBase {+public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ private static final String DEFAULT_COLLECTION = "collection1";++ private final boolean onlyLeaderIndexes = random().nextBoolean();++ String tsort = "t_sortable";++ private Map<String, List<SolrClient>> otherCollectionClients = new HashMap<>();++ private String oneInstanceCollection = "oneInstanceCollection";+ private String oneInstanceCollection2 = "oneInstanceCollection2";++ private AtomicInteger nodeCounter = new AtomicInteger();++ CompletionService<Object> completionService;+ Set<Future<Object>> pending;++ private static Hook newSearcherHook =+ new Hook() {+ volatile CountDownLatch latch;+ AtomicReference<String> collection = new AtomicReference<>();++ @Override+ public void newSearcher(String collectionName) {+ String c = collection.get();+ if (c != null && c.equals(collectionName)) {+ log.info("Hook detected newSearcher");+ try {+ latch.countDown();+ } catch (NullPointerException e) {++ }+ }+ }++ @Override+ public void waitForSearcher(+ String collection, int cnt, int timeoutms, boolean failOnTimeout)+ throws InterruptedException {+ latch = new CountDownLatch(cnt);+ this.collection.set(collection);+ boolean timeout = !latch.await(timeoutms, TimeUnit.MILLISECONDS);+ if (timeout && failOnTimeout) {+ fail("timed out waiting for new searcher event " + latch.getCount());+ }+ }+ };++ public BasicDistributedZkTest() {+ // we need DVs on point fields to compute stats & facets+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP))+ System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");++ sliceCount = 2;+ completionService = new ExecutorCompletionService<>(executor);+ pending = new HashSet<>();+ }++ @BeforeClass+ public static void beforeBDZKTClass() {+ TestInjection.newSearcherHook(newSearcherHook);+ }++ @Override+ protected boolean useTlogReplicas() {+ // TODO: tlog replicas makes commits take way to long due to what is likely a bug and it's+ // TestInjection use+ return false;+ }- @Test@Override+ protected void setDistributedParams(ModifiableSolrParams params) {++ if (r.nextBoolean()) {+ // don't set shards, let that be figured out from the cloud state+ } else {+ // use shard ids rather than physical locations+ StringBuilder sb = new StringBuilder();+ for (int i = 0; i < getShardCount(); i++) {+ if (i > 0) sb.append(',');+ sb.append("shard" + (i + 3));+ }+ params.set("shards", sb.toString());+ }+ }++ @Test@ShardsFixed(num = 4)public void test() throws Exception {- super.test();+ // setLoggingLevel(null);++ ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);+ // make sure we have leaders for each shard+ for (int j = 1; j < sliceCount; j++) {+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);+ } // make sure we again have leaders for each shard++ waitForRecoveriesToFinish(false);++ handle.clear();+ handle.put("timestamp", SKIPVAL);++ del("*:*");+ queryAndCompareShards(params("q", "*:*", "distrib", "false", "sanity_check", "is_empty"));++ // ask every individual replica of every shard to update+commit the same doc id+ // with an incrementing counter on each update+commit+ int foo_i_counter = 0;+ for (SolrClient client : clients) {+ foo_i_counter++;+ indexDoc(+ client,+ params("commit", "true"), // SOLR-4923+ sdoc(id, 1, i1, 100, tlong, 100, "foo_i", foo_i_counter));+ // after every update+commit, check all the shards consistency+ queryAndCompareShards(+ params("q", "id:1", "distrib", "false", "sanity_check", "non_distrib_id_1_lookup"));+ queryAndCompareShards(+ params(+ "q", "id:1",+ "sanity_check", "distrib_id_1_lookup"));+ }++ indexr(+ id,+ 1,+ i1,+ 100,+ tlong,+ 100,+ t1,+ "now is the time for all good men",+ "foo_f",+ 1.414f,+ "foo_b",+ "true",+ "foo_d",+ 1.414d,+ tsort,+ "now is the time for all good men");+ indexr(+ id,+ 2,+ i1,+ 50,+ tlong,+ 50,+ t1,+ "to come to the aid of their country.",+ tsort,+ "to come to the aid of their country.");+ indexr(id, 3, i1, 2, tlong, 2, t1, "how now brown cow", tsort, "how now brown cow");+ indexr(+ id,+ 4,+ i1,+ -100,+ tlong,+ 101,+ t1,+ "the quick fox jumped over the lazy dog",+ tsort,+ "the quick fox jumped over the lazy dog");+ indexr(+ id,+ 5,+ i1,+ 500,+ tlong,+ 500,+ t1,+ "the quick fox jumped way over the lazy dog",+ tsort,+ "the quick fox jumped over the lazy dog");+ indexr(+ id,+ 6,+ i1,+ -600,+ tlong,+ 600,+ t1,+ "humpty dumpy sat on a wall",+ tsort,+ "the quick fox jumped over the lazy dog");+ indexr(+ id,+ 7,+ i1,+ 123,+ tlong,+ 123,+ t1,+ "humpty dumpy had a great fall",+ tsort,+ "the quick fox jumped over the lazy dog");+ indexr(+ id,+ 8,+ i1,+ 876,+ tlong,+ 876,+ t1,+ "all the kings horses and all the kings men",+ tsort,+ "all the kings horses and all the kings men");+ indexr(+ id,+ 9,+ i1,+ 7,+ tlong,+ 7,+ t1,+ "couldn't put humpty together again",+ tsort,+ "the quick fox jumped over the lazy dog");+ indexr(id, 10, i1, 4321, tlong, 4321, t1, "this too shall pass", tsort, "this too shall pass");+ indexr(+ id,+ 11,+ i1,+ -987,+ tlong,+ 987,+ t1,+ "An eye for eye only ends up making the whole world blind.",+ tsort,+ "An eye for eye only ends up making the whole world blind.");+ indexr(+ id,+ 12,+ i1,+ 379,+ tlong,+ 379,+ t1,+ "Great works are performed, not by strength, but by perseverance.",+ tsort,+ "Great works are performed, not by strength, but by perseverance.");+ indexr(+ id,+ 13,+ i1,+ 232,+ tlong,+ 232,+ t1,+ "no eggs on wall, lesson learned",+ oddField,+ "odd man out",+ tsort,+ "no eggs on wall, lesson learned");++ indexr(+ id,+ 14,+ "SubjectTerms_mfacet",+ new String[] {"mathematical models", "mathematical analysis"});+ indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});+ indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});+ String[] vals = new String[100];+ for (int i = 0; i < 100; i++) {+ vals[i] = "test " + i;+ }+ indexr(id, 17, "SubjectTerms_mfacet", vals);++ for (int i = 100; i < 150; i++) {+ indexr(id, i);+ }++ commit();++ testTokenizedGrouping();+ testSortableTextFaceting();+ testSortableTextSorting();+ testSortableTextGrouping();++ queryAndCompareShards(+ params(+ "q", "*:*",+ "sort", "id desc",+ "distrib", "false",+ "sanity_check", "is_empty"));++ // random value sort+ for (String f : fieldNames) {+ query(false, new String[] {"q", "*:*", "sort", f + " desc"});+ query(false, new String[] {"q", "*:*", "sort", f + " asc"});+ }++ // these queries should be exactly ordered and scores should exactly match+ query(false, new String[] {"q", "*:*", "sort", i1 + " desc"});+ query(false, new String[] {"q", "*:*", "sort", i1 + " asc"});+ query(false, new String[] {"q", "*:*", "sort", i1 + " desc", "fl", "*,score"});+ query(false, new String[] {"q", "*:*", "sort", "n_tl1 asc", "fl", "*,score"});+ query(false, new String[] {"q", "*:*", "sort", "n_tl1 desc"});+ handle.put("maxScore", SKIPVAL);+ query(+ false,+ new String[] {"q", "{!func}" + i1}); // does not expect maxScore. So if it comes ,ignore it.+ // JavaBinCodec.writeSolrDocumentList()+ // is agnostic of request params.+ handle.remove("maxScore");+ query(+ false,+ new String[] {+ "q", "{!func}" + i1, "fl", "*,score"+ }); // even scores should match exactly here++ handle.put("highlighting", UNORDERED);+ handle.put("response", UNORDERED);++ handle.put("maxScore", SKIPVAL);+ query(false, new String[] {"q", "quick"});+ query(false, new String[] {"q", "all", "fl", "id", "start", "0"});+ query(+ false,+ new String[] {"q", "all", "fl", "foofoofoo", "start", "0"}); // no fields in returned docs+ query(false, new String[] {"q", "all", "fl", "id", "start", "100"});++ handle.put("score", SKIPVAL);+ query(false, new String[] {"q", "quick", "fl", "*,score"});+ query(false, new String[] {"q", "all", "fl", "*,score", "start", "1"});+ query(false, new String[] {"q", "all", "fl", "*,score", "start", "100"});++ query(+ false,+ new String[] {+ "q", "now their fox sat had put", "fl", "*,score", "hl", "true", "hl.fl", t1+ });++ query(+ false,+ new String[] {+ "q", "now their fox sat had put", "fl", "foofoofoo", "hl", "true", "hl.fl", t1+ });++ query(false, new String[] {"q", "matchesnothing", "fl", "*,score"});++ query(false, new Object[] {"q", "*:*", "rows", 100, "facet", "true", "facet.field", t1});+ query(+ false,+ new Object[] {+ "q",+ "*:*",+ "rows",+ 100,+ "facet",+ "true",+ "facet.field",+ t1,+ "facet.limit",+ -1,+ "facet.sort",+ "count"+ });+ query(+ false,+ new Object[] {+ "q",+ "*:*",+ "rows",+ 100,+ "facet",+ "true",+ "facet.field",+ t1,+ "facet.limit",+ -1,+ "facet.sort",+ "count",+ "facet.mincount",+ 2+ });+ query(+ false,+ new Object[] {+ "q",+ "*:*",+ "rows",+ 100,+ "facet",+ "true",+ "facet.field",+ t1,+ "facet.limit",+ -1,+ "facet.sort",+ "index"+ });+ query(+ false,+ new Object[] {+ "q",+ "*:*",+ "rows",+ 100,+ "facet",+ "true",+ "facet.field",+ t1,+ "facet.limit",+ -1,+ "facet.sort",+ "index",+ "facet.mincount",+ 2+ });+ query(+ false,+ new Object[] {+ "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.limit", 1+ });+ query(+ false,+ new Object[] {+ "q",+ "*:*",+ "rows",+ 100,+ "facet",+ "true",+ "facet.query",+ "quick",+ "facet.query",+ "all",+ "facet.query",+ "*:*"+ });+ query(+ false,+ new Object[] {+ "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.offset", 1+ });+ query(+ false,+ new Object[] {+ "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.mincount", 2+ });++ // test faceting multiple things at once+ query(+ false,+ new Object[] {+ "q",+ "*:*",+ "rows",+ 100,+ "facet",+ "true",+ "facet.query",+ "quick",+ "facet.query",+ "all",+ "facet.query",+ "*:*",+ "facet.field",+ t1+ });++ // test filter tagging, facet exclusion, and naming (multi-select facet support)+ query(+ false,+ new Object[] {+ "q",+ "*:*",+ "rows",+ 100,+ "facet",+ "true",+ "facet.query",+ "{!key=myquick}quick",+ "facet.query",+ "{!key=myall ex=a}all",+ "facet.query",+ "*:*",+ "facet.field",+ "{!key=mykey ex=a}" + t1,+ "facet.field",+ "{!key=other ex=b}" + t1,+ "facet.field",+ "{!key=again ex=a,b}" + t1,+ "facet.field",+ t1,+ "fq",+ "{!tag=a}id_i1:[1 TO 7]",+ "fq",+ "{!tag=b}id_i1:[3 TO 9]"+ });+ query(+ false,+ new Object[] {+ "q",+ "*:*",+ "facet",+ "true",+ "facet.field",+ "{!ex=t1}SubjectTerms_mfacet",+ "fq",+ "{!tag=t1}SubjectTerms_mfacet:(test 1)",+ "facet.limit",+ "10",+ "facet.mincount",+ "1"+ });++ // test field that is valid in schema but missing in all shards+ query(+ false,+ new Object[] {+ "q", "*:*", "rows", 100, "facet", "true", "facet.field", missingField, "facet.mincount", 2+ });+ // test field that is valid in schema and missing in some shards+ query(+ false,+ new Object[] {+ "q", "*:*", "rows", 100, "facet", "true", "facet.field", oddField, "facet.mincount", 2+ });++ query(+ false, new Object[] {"q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", i1});++ /* TODO: the failure may come back in "exception"+ try {+ // test error produced for field that is invalid for schema+ query("q","*:*", "rows",100, "facet","true", "facet.field",invalidField, "facet.mincount",2);+ TestCase.fail("SolrServerException expected for invalid field that is not in schema");+ } catch (SolrServerException ex) {+ // expected+ }+ */++ // Try to get better coverage for refinement queries by turning off over requesting.+ // This makes it much more likely that we may not get the top facet values and hence+ // we turn of that checking.+ handle.put("facet_fields", SKIPVAL);+ query(+ false,+ new Object[] {+ "q",+ "*:*",+ "rows",+ 0,+ "facet",+ "true",+ "facet.field",+ t1,+ "facet.limit",+ 5,+ "facet.shard.limit",+ 5+ });+ // check a complex key name+ query(+ false,+ new Object[] {+ "q",+ "*:*",+ "rows",+ 0,+ "facet",+ "true",+ "facet.field",+ "{!key='a b/c \\' \\} foo'}" + t1,+ "facet.limit",+ 5,+ "facet.shard.limit",+ 5+ });+ handle.remove("facet_fields");++ // index the same document to two servers and make sure things+ // don't blow up.+ if (clients.size() >= 2) {+ index(id, 100, i1, 107, t1, "oh no, a duplicate!");+ for (int i = 0; i < clients.size(); i++) {+ index_specific(i, id, 100, i1, 107, t1, "oh no, a duplicate!");+ }+ commit();+ query(false, new Object[] {"q", "duplicate", "hl", "true", "hl.fl", t1});+ query(false, new Object[] {"q", "fox duplicate horses", "hl", "true", "hl.fl", t1});+ query(false, new Object[] {"q", "*:*", "rows", 100});+ }++ // test debugging+ handle.put("explain", SKIPVAL);+ handle.put("debug", UNORDERED);+ handle.put("time", SKIPVAL);+ handle.put("track", SKIP);+ query(+ false,+ new Object[] {+ "q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true"+ });+ query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"});+ query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING});+ query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS});+ query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY});++ // try add commitWithin+ long before = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();+ for (SolrClient client : clients) {+ assertEquals(+ "unexpected pre-commitWithin document count on node: "+ + ((HttpSolrClient) client).getBaseURL(),+ before,+ client.query(new SolrQuery("*:*")).getResults().getNumFound());+ }++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("commitWithin", 10);+ add(cloudClient, params, List.of(getDoc("id", 300), getDoc("id", 301)));++ newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);++ ClusterState clusterState = getCommonCloudSolrClient().getClusterState();+ DocCollection dColl = clusterState.getCollection(DEFAULT_COLLECTION);++ assertSliceCounts("should have found 2 docs, 300 and 301", before + 2, dColl);++ // try deleteById commitWithin+ UpdateRequest deleteByIdReq = new UpdateRequest();+ deleteByIdReq.deleteById("300");+ deleteByIdReq.setCommitWithin(10);+ deleteByIdReq.process(cloudClient);++ newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);++ assertSliceCounts("deleteById commitWithin did not work", before + 1, dColl);++ // try deleteByQuery commitWithin+ UpdateRequest deleteByQueryReq = new UpdateRequest();+ deleteByQueryReq.deleteByQuery("id:301");+ deleteByQueryReq.setCommitWithin(10);+ deleteByQueryReq.process(cloudClient);++ newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);++ assertSliceCounts("deleteByQuery commitWithin did not work", before, dColl);++ // TODO: This test currently fails because debug info is obtained only+ // on shards with matches.+ // query("q","matchesnothing","fl","*,score", "debugQuery", "true");++ // would be better if these where all separate tests - but much, much+ // slower+ doOptimisticLockingAndUpdating();+ testShardParamVariations();+ testMultipleCollections();+ testANewCollectionInOneInstance();+ testSearchByCollectionName();+ testUpdateByCollectionName();+ testANewCollectionInOneInstanceWithManualShardAssignement();+ testNumberOfCommitsWithCommitAfterAdd();++ testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit");+ testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit");++ testStopAndStartCoresInOneInstance();+ }++ private void testSortableTextFaceting() throws Exception {+ SolrQuery query = new SolrQuery("*:*");+ query.addFacetField(tsort);+ query.setFacetMissing(false);+ QueryResponse resp = queryRandomShard(query);+ List<FacetField> ffs = resp.getFacetFields();+ for (FacetField ff : ffs) {+ if (ff.getName().equals(tsort) == false) continue;+ for (FacetField.Count count : ff.getValues()) {+ long num = count.getCount();+ switch (count.getName()) {+ case "all the kings horses and all the kings men":+ case "An eye for eye only ends up making the whole world blind.":+ case "Great works are performed, not by strength, but by perseverance.":+ case "how now brown cow":+ case "no eggs on wall, lesson learned":+ case "now is the time for all good men":+ case "this too shall pass":+ case "to come to the aid of their country.":+ assertEquals("Should have exactly one facet count for field " + ff.getName(), 1, num);+ break;+ case "the quick fox jumped over the lazy dog":+ assertEquals("Should have 5 docs for the lazy dog", 5, num);+ break;+ default:+ fail("No case for facet '" + ff.getName() + "'");+ }+ }+ }+ }++ private void testSortableTextSorting() throws Exception {+ SolrQuery query = new SolrQuery("*:*");+ query.addSort(tsort, SolrQuery.ORDER.desc);+ query.addField("*");+ query.addField("eoe_sortable");+ query.addField(tsort);+ QueryResponse resp = queryRandomShard(query);++ SolrDocumentList docs = resp.getResults();++ String title = docs.get(0).getFieldValue(tsort).toString();+ for (SolrDocument doc : docs) {+ assertTrue(+ "Docs should be back in sorted order, descending",+ title.compareTo(doc.getFieldValue(tsort).toString()) >= 0);+ title = doc.getFieldValue(tsort).toString();+ }+ }++ private void testSortableTextGrouping() throws Exception {+ SolrQuery query = new SolrQuery("*:*");+ query.add("group", "true");+ query.add("group.field", tsort);+ QueryResponse resp = queryRandomShard(query);+ GroupResponse groupResp = resp.getGroupResponse();+ List<GroupCommand> grpCmds = groupResp.getValues();+ for (GroupCommand grpCmd : grpCmds) {+ if (grpCmd.getName().equals(tsort) == false) continue;+ for (Group grp : grpCmd.getValues()) {+ long count = grp.getResult().getNumFound();+ if (grp.getGroupValue() == null)+ continue; // Don't count the groups without an entry as the numnber is variable+ switch (grp.getGroupValue()) {+ case "all the kings horses and all the kings men":+ case "An eye for eye only ends up making the whole world blind.":+ case "Great works are performed, not by strength, but by perseverance.":+ case "how now brown cow":+ case "no eggs on wall, lesson learned":+ case "now is the time for all good men":+ case "this too shall pass":+ case "to come to the aid of their country.":+ assertEquals(+ "Should have exactly one facet count for field " + grpCmd.getName(), 1, count);+ break;+ case "the quick fox jumped over the lazy dog":+ assertEquals("Should have 5 docs for the lazy dog", 5, count);+ break;+ default:+ fail("No case for facet '" + grpCmd.getName() + "'");+ }+ }+ }+ }++ private void testTokenizedGrouping() throws Exception {+ SolrException ex =+ expectThrows(+ SolrException.class,+ () -> {+ query(false, new String[] {"q", "*:*", "group", "true", "group.field", t1});+ });+ assertTrue(+ "Expected error from server that SortableTextFields are required",+ ex.getMessage()+ .contains(+ "Sorting on a tokenized field that is not a SortableTextField is not supported in cloud mode"));+ }++ private void assertSliceCounts(String msg, long expected, DocCollection dColl) throws Exception {+ long found = checkSlicesSameCounts(dColl);++ if (found != expected) {+ // we get one do over in a bad race+ Thread.sleep(1000);+ found = checkSlicesSameCounts(dColl);+ }++ assertEquals(msg, expected, checkSlicesSameCounts(dColl));+ }++ // Ensure that total docs found is the expected number.+ private void waitForDocCount(long expectedNumFound, long waitMillis, String failureMessage)+ throws Exception {+ AtomicLong total = new AtomicLong(-1);+ try {+ ZkStateReader.from(getCommonCloudSolrClient())+ .waitForState(+ DEFAULT_COLLECTION,+ waitMillis,+ TimeUnit.MILLISECONDS,+ (n, c) -> {+ long docTotal;+ try {+ docTotal = checkSlicesSameCounts(c);+ } catch (SolrServerException | IOException e) {+ throw new RuntimeException(e);+ }+ total.set(docTotal);+ if (docTotal == expectedNumFound) {+ return true;+ }+ return false;+ });+ } catch (TimeoutException | InterruptedException e) {++ }+ // We could fail here if we broke out of the above because we exceeded the time allowed.+ assertEquals(failureMessage, expectedNumFound, total.get());++ // This should be redundant, but it caught a test error after all.+ for (SolrClient client : clients) {+ assertEquals(+ failureMessage,+ expectedNumFound,+ client.query(new SolrQuery("*:*")).getResults().getNumFound());+ }+ }++ // Insure that counts are the same for all replicas in each shard+ // Return the total doc count for the query.+ private long checkSlicesSameCounts(DocCollection dColl) throws SolrServerException, IOException {+ long docTotal = 0; // total number of documents found counting only one replica per slice.+ for (Slice slice : dColl.getActiveSlices()) {+ long sliceDocCount = -1;+ for (Replica rep : slice.getReplicas()) {+ try (SolrClient one = getHttpSolrClient(rep)) {+ SolrQuery query = new SolrQuery("*:*");+ query.setDistrib(false);+ QueryResponse resp = one.query(query);+ long hits = resp.getResults().getNumFound();+ if (sliceDocCount == -1) {+ sliceDocCount = hits;+ docTotal += hits;+ } else {+ if (hits != sliceDocCount) {+ return -1;+ }+ }+ }+ }+ }+ return docTotal;+ }++ private void testShardParamVariations() throws Exception {+ SolrQuery query = new SolrQuery("*:*");+ Map<String, Long> shardCounts = new HashMap<>();++ for (String shard : shardToJetty.keySet()) {+ // every client should give the same numDocs for this shard+ // shffle the clients in a diff order for each shard+ List<SolrClient> solrclients = new ArrayList<>(this.clients);+ Collections.shuffle(solrclients, random());+ for (SolrClient client : solrclients) {+ query.set("shards", shard);+ long numDocs = client.query(query).getResults().getNumFound();+ assertTrue("numDocs < 0 for shard " + shard + " via " + client, 0 <= numDocs);+ if (!shardCounts.containsKey(shard)) {+ shardCounts.put(shard, numDocs);+ }+ assertEquals(+ "inconsitent numDocs for shard " + shard + " via " + client,+ shardCounts.get(shard).longValue(),+ numDocs);++ List<CloudJettyRunner> replicaJetties = new ArrayList<>(shardToJetty.get(shard));+ Collections.shuffle(replicaJetties, random());++ // each replica should also give the same numDocs+ ArrayList<String> replicaAlts = new ArrayList<>(replicaJetties.size() * 2);+ for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) {+ String replica = replicaJetty.url;+ query.set("shards", replica);++ // replicas already shuffled, use this in the alternative check below+ if (0 == random().nextInt(3) || replicaAlts.size() < 2) {+ replicaAlts.add(replica);+ }++ numDocs = client.query(query).getResults().getNumFound();+ assertTrue("numDocs < 0 for replica " + replica + " via " + client, 0 <= numDocs);+ assertEquals(+ "inconsitent numDocs for shard "+ + shard+ + " in replica "+ + replica+ + " via "+ + client,+ shardCounts.get(shard).longValue(),+ numDocs);+ }++ // any combination of replica alternatives should give same numDocs+ String replicas = String.join("|", replicaAlts);+ query.set("shards", replicas);+ numDocs = client.query(query).getResults().getNumFound();+ assertTrue("numDocs < 0 for replicas " + replicas + " via " + client, 0 <= numDocs);+ assertEquals(+ "inconsitent numDocs for replicas " + replicas + " via " + client,+ shardCounts.get(shard).longValue(),+ numDocs);+ }+ }++ // sums of multiple shards should add up regardless of how we+ // query those shards or which client we use+ long randomShardCountsExpected = 0;+ ArrayList<String> randomShards = new ArrayList<>(shardCounts.size());+ for (Map.Entry<String, Long> shardData : shardCounts.entrySet()) {+ if (random().nextBoolean() || randomShards.size() < 2) {+ String shard = shardData.getKey();+ randomShardCountsExpected += shardData.getValue();+ if (random().nextBoolean()) {+ // use shard id+ randomShards.add(shard);+ } else {+ // use some set explicit replicas+ ArrayList<String> replicas = new ArrayList<>(7);+ for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) {+ if (0 == random().nextInt(3) || 0 == replicas.size()) {+ replicas.add(replicaJetty.url);+ }+ }+ Collections.shuffle(replicas, random());+ randomShards.add(String.join("|", replicas));+ }+ }+ }+ String randShards = String.join(",", randomShards);+ query.set("shards", randShards);+ for (SolrClient client : this.clients) {+ assertEquals(+ "numDocs for " + randShards + " via " + client,+ randomShardCountsExpected,+ client.query(query).getResults().getNumFound());+ }++ // total num docs must match sum of every shard's numDocs+ query = new SolrQuery("*:*");+ long totalShardNumDocs = 0;+ for (Long c : shardCounts.values()) {+ totalShardNumDocs += c;+ }+ for (SolrClient client : clients) {+ assertEquals(+ "sum of shard numDocs on client: " + client,+ totalShardNumDocs,+ client.query(query).getResults().getNumFound());+ }+ assertTrue("total numDocs <= 0, WTF? Test is useless", 0 < totalShardNumDocs);+ }++ private void testStopAndStartCoresInOneInstance() throws Exception {+ JettySolrRunner jetty = jettys.get(0);+ try (final SolrClient httpSolrClient = (HttpSolrClient) jetty.newClient(15000, 60000)) {+ ThreadPoolExecutor executor = null;+ try {+ executor =+ new ExecutorUtil.MDCAwareThreadPoolExecutor(+ 0,+ Integer.MAX_VALUE,+ 5,+ TimeUnit.SECONDS,+ new SynchronousQueue<Runnable>(),+ new SolrNamedThreadFactory("testExecutor"));+ int cnt = 3;++ // create the cores+ createCollectionInOneInstance(+ httpSolrClient, jetty.getNodeName(), executor, "multiunload2", 1, cnt);+ } finally {+ if (executor != null) {+ ExecutorUtil.shutdownAndAwaitTermination(executor);+ }+ }+ }++ cloudJettys.get(0).jetty.stop();+ printLayout();++ cloudJettys.get(0).jetty.start();+ ZkStateReader.from(cloudClient).forceUpdateCollection("multiunload2");+ try {+ ZkStateReader.from(cloudClient).getLeaderRetry("multiunload2", "shard1", 30000);+ } catch (SolrException e) {+ printLayout();+ throw e;+ }++ printLayout();+ }++ /** Create a collection in single node */+ public static void createCollectionInOneInstance(+ final SolrClient client,+ String nodeName,+ ThreadPoolExecutor executor,+ final String collection,+ final int numShards,+ int numReplicas) {+ assertNotNull(nodeName);+ try {+ assertEquals(+ 0,+ CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1)+ .setCreateNodeSet("")+ .process(client)+ .getStatus());+ } catch (SolrServerException | IOException e) {+ throw new RuntimeException(e);+ }+ for (int i = 0; i < numReplicas; i++) {+ final int freezeI = i;+ executor.execute(+ () -> {+ try {+ assertTrue(+ CollectionAdminRequest.addReplicaToShard(+ collection, "shard" + ((freezeI % numShards) + 1))+ .setCoreName(collection + freezeI)+ .setNode(nodeName)+ .process(client)+ .isSuccess());+ } catch (SolrServerException | IOException e) {+ throw new RuntimeException(e);+ }+ });+ }+ }++ @Override+ protected CollectionAdminResponse createCollection(+ Map<String, List<Integer>> collectionInfos,+ String collectionName,+ String configSetName,+ int numShards,+ int numReplicas,+ SolrClient client,+ String createNodeSetStr)+ throws SolrServerException, IOException {+ // TODO: Use CollectionAdminRequest for this test+ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("action", CollectionAction.CREATE.toString());++ params.set(CollectionHandlingUtils.NUM_SLICES, numShards);+ params.set(ZkStateReader.REPLICATION_FACTOR, numReplicas);+ if (createNodeSetStr != null)+ params.set(CollectionHandlingUtils.CREATE_NODE_SET, createNodeSetStr);++ int clientIndex = clients.size() > 1 ? random().nextInt(2) : 0;+ List<Integer> list = new ArrayList<>();+ list.add(numShards);+ list.add(numReplicas);+ if (collectionInfos != null) {+ collectionInfos.put(collectionName, list);+ }+ params.set("name", collectionName);+ params.set("collection.configName", configSetName);+ QueryRequest request = new QueryRequest(params);+ request.setPath("/admin/collections");++ CollectionAdminResponse res = new CollectionAdminResponse();+ if (client == null) {+ final String baseUrl = ((HttpSolrClient) clients.get(clientIndex)).getBaseURL();++ try (SolrClient aClient = createNewSolrClient("", baseUrl)) {+ res.setResponse(aClient.request(request));+ }+ } else {+ res.setResponse(client.request(request));+ }+ return res;+ }++ @Override+ protected Replica getLeaderFromZk(String collection, String slice) {+ ClusterState clusterState = getCommonCloudSolrClient().getClusterState();+ Replica leader = clusterState.getCollection(collection).getLeader(slice);+ if (leader == null) {+ throw new RuntimeException("Could not find leader:" + collection + " " + slice);+ }+ return leader;+ }++ /**+ * Expects a RegexReplaceProcessorFactories in the chain which will "double up" the values in two+ * (stored) string fields.+ *+ * <p>If the values are "double-doubled" or "not-doubled" then we know the processor was not run+ * the appropriate number of times+ */+ private void testUpdateProcessorsRunOnlyOnce(final String chain) throws Exception {++ final String fieldA = "regex_dup_A_s";+ final String fieldB = "regex_dup_B_s";+ final String val = "x";+ final String expected = "x_x";+ final ModifiableSolrParams updateParams = new ModifiableSolrParams();+ updateParams.add(UpdateParams.UPDATE_CHAIN, chain);++ final int numLoops = atLeast(50);++ for (int i = 1; i < numLoops; i++) {+ // add doc to random client+ SolrClient updateClient = clients.get(random().nextInt(clients.size()));+ SolrInputDocument doc = new SolrInputDocument();+ addFields(doc, id, i, fieldA, val, fieldB, val);+ UpdateResponse ures = add(updateClient, updateParams, doc);+ assertEquals(chain + ": update failed", 0, ures.getStatus());+ ures = updateClient.commit();+ assertEquals(chain + ": commit failed", 0, ures.getStatus());+ }++ // query for each doc, and check both fields to ensure the value is correct+ for (int i = 1; i < numLoops; i++) {+ final String query = id + ":" + i;+ QueryResponse qres = queryRandomShard(new SolrQuery(query));+ assertEquals(chain + ": query failed: " + query, 0, qres.getStatus());+ assertEquals(+ chain + ": didn't find correct # docs with query: " + query,+ 1,+ qres.getResults().getNumFound());+ SolrDocument doc = qres.getResults().get(0);++ for (String field : new String[] {fieldA, fieldB}) {+ assertEquals(+ chain + ": doc#" + i + " has wrong value for " + field,+ expected,+ doc.getFirstValue(field));+ }+ }+ }++ // cloud level test mainly needed just to make sure that versions and errors are propagated+ // correctly+ private void doOptimisticLockingAndUpdating() throws Exception {+ log.info("### STARTING doOptimisticLockingAndUpdating");+ printLayout();++ final SolrInputDocument sd = sdoc("id", 1000, "_version_", -1);+ indexDoc(sd);++ ignoreException("version conflict");+ for (SolrClient client : clients) {+ SolrException e = expectThrows(SolrException.class, () -> client.add(sd));+ assertEquals(409, e.code());+ }+ unIgnoreException("version conflict");++ // TODO: test deletes. SolrJ needs a good way to pass version for delete...++ final SolrInputDocument sd2 = sdoc("id", 1000, "foo_i", 5);+ clients.get(0).add(sd2);++ List<Integer> expected = new ArrayList<>();+ int val = 0;+ for (SolrClient client : clients) {+ val += 10;+ client.add(sdoc("id", 1000, "val_i", map("add", val), "foo_i", val));+ expected.add(val);+ }++ QueryRequest qr = new QueryRequest(params("qt", "/get", "id", "1000"));+ for (SolrClient client : clients) {+ val += 10;+ NamedList<?> rsp = client.request(qr);+ String match = JSONTestUtil.matchObj("/val_i", rsp.get("doc"), expected);+ if (match != null) throw new RuntimeException(match);+ }+ }++ private void testNumberOfCommitsWithCommitAfterAdd() throws SolrServerException, IOException {+ log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd");+ long startCommits = getNumCommits((HttpSolrClient) clients.get(0));++ NamedList<Object> result =+ clients+ .get(0)+ .request(+ new StreamingUpdateRequest(+ "/update", getFile("books_numeric_ids.csv"), "application/csv")+ .setCommitWithin(900000)+ .setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true));++ long endCommits = getNumCommits((HttpSolrClient) clients.get(0));+ assertEquals(startCommits + 1L, endCommits);+ }++ private Long getNumCommits(HttpSolrClient sourceClient) throws SolrServerException, IOException {+ String collection = sourceClient.getDefaultCollection();+ try (SolrClient client =+ new HttpSolrClient.Builder(sourceClient.getBaseURL())+ .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)+ .withSocketTimeout(60000, TimeUnit.MILLISECONDS)+ .build()) {+ var req =+ new GenericSolrRequest(+ SolrRequest.METHOD.GET,+ "/admin/metrics",+ SolrRequest.SolrRequestType.ADMIN,+ SolrParams.of("wt", "prometheus"));+ req.setResponseParser(new InputStreamResponseParser("prometheus"));++ NamedList<Object> resp = client.request(req);+ try (InputStream in = (InputStream) resp.get("stream")) {+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);+ String metricName = "solr_core_update_commit_ops";++ return (long)+ output+ .lines()+ .filter(+ l ->+ l.startsWith(metricName)+ && l.contains("collection=\"" + collection + "\"")+ && l.contains("ops=\"commits\""))+ .mapToDouble(s -> Double.parseDouble(s.substring(s.lastIndexOf(" "))))+ .sum();+ }+ }+ }++ private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception {+ log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement");+ assertEquals(+ 0,+ CollectionAdminRequest.createCollection(oneInstanceCollection2, "conf1", 2, 2)+ .setCreateNodeSet("")+ .process(cloudClient)+ .getStatus());++ List<SolrClient> collectionClients = new ArrayList<>();+ for (int i = 0; i < 4; i++) {+ CollectionAdminResponse resp =+ CollectionAdminRequest.addReplicaToShard(oneInstanceCollection2, "shard" + ((i % 2) + 1))+ .setNode(jettys.get(0).getNodeName())+ .process(cloudClient);+ for (String coreName : resp.getCollectionCoresStatus().keySet()) {+ collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString()));+ }+ }++ SolrClient client1 = collectionClients.get(0);+ SolrClient client2 = collectionClients.get(1);+ SolrClient client3 = collectionClients.get(2);+ SolrClient client4 = collectionClients.get(3);++ // no one should be recovering+ waitForRecoveriesToFinish(+ oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()), false, true);++ assertAllActive(oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()));++ // printLayout();++ // TODO: enable when we don't falsely get slice1...+ // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000);+ // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000);+ client2.add(getDoc(id, "1"));+ client3.add(getDoc(id, "2"));+ client4.add(getDoc(id, "3"));++ client1.commit();+ SolrQuery query = new SolrQuery("*:*");+ query.set("distrib", false);+ long oneDocs = client1.query(query).getResults().getNumFound();+ long twoDocs = client2.query(query).getResults().getNumFound();+ long threeDocs = client3.query(query).getResults().getNumFound();+ long fourDocs = client4.query(query).getResults().getNumFound();++ query.set("collection", oneInstanceCollection2);+ query.set("distrib", true);+ long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound();++ // System.out.println("1:" + oneDocs);+ // System.out.println("2:" + twoDocs);+ // System.out.println("3:" + threeDocs);+ // System.out.println("4:" + fourDocs);+ // System.out.println("All Docs:" + allDocs);++ // assertEquals(oneDocs, threeDocs);+ // assertEquals(twoDocs, fourDocs);+ // assertNotSame(oneDocs, twoDocs);+ assertEquals(3, allDocs);++ // we added a role of none on these creates - check for it+ ZkStateReader zkStateReader = ZkStateReader.from(getCommonCloudSolrClient());+ zkStateReader.forceUpdateCollection(oneInstanceCollection2);+ Map<String, Slice> slices =+ zkStateReader.getClusterState().getCollection(oneInstanceCollection2).getSlicesMap();+ assertNotNull(slices);++ Replica leader =+ getCommonCloudSolrClient()+ .getClusterState()+ .getCollection(oneInstanceCollection2)+ .getLeader("shard1");++ // now test that unloading a core gets us a new leader+ try (SolrClient unloadClient =+ new HttpSolrClient.Builder(jettys.get(0).getBaseUrl().toString())+ .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)+ .withSocketTimeout(60000, TimeUnit.MILLISECONDS)+ .build()) {+ Unload unloadCmd = new Unload(true);+ unloadCmd.setCoreName(leader.getCoreName());++ String leaderUrl = leader.getCoreUrl();++ testExecutor.execute(+ new Runnable() {++ @Override+ public void run() {+ try {+ unloadClient.request(unloadCmd);+ } catch (SolrServerException e) {+ throw new RuntimeException(e);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ });++ try {+ ZkStateReader.from(getCommonCloudSolrClient())+ .waitForState(+ oneInstanceCollection2,+ 20000,+ TimeUnit.MILLISECONDS,+ (n, c) -> {+ try {+ if (leaderUrl.equals(+ zkStateReader.getLeaderUrl(oneInstanceCollection2, "shard1", 10000))) {+ return false;+ }+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ return true;+ });+ } catch (TimeoutException | InterruptedException e) {+ fail("Leader never changed");+ }+ }++ IOUtils.close(collectionClients);+ }++ private void testSearchByCollectionName() throws SolrServerException, IOException {+ log.info("### STARTING testSearchByCollectionName");+ SolrClient client = clients.get(0);+ final String baseUrl = ((HttpSolrClient) client).getBaseURL();++ // the cores each have different names, but if we add the collection name to the url+ // we should get mapped to the right core+ try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) {+ SolrQuery query = new SolrQuery("*:*");+ long oneDocs = client1.query(query).getResults().getNumFound();+ assertEquals(3, oneDocs);+ }+ }++ private void testUpdateByCollectionName() throws SolrServerException, IOException {+ log.info("### STARTING testUpdateByCollectionName");+ SolrClient client = clients.get(0);+ final String baseUrl = ((HttpSolrClient) client).getBaseURL();++ // the cores each have different names, but if we add the collection name to the url+ // we should get mapped to the right core+ // test hitting an update url+ try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) {+ client1.commit();+ }+ }++ private void testANewCollectionInOneInstance() throws Exception {+ log.info("### STARTING testANewCollectionInOneInstance");+ CollectionAdminResponse response =+ CollectionAdminRequest.createCollection(oneInstanceCollection, "conf1", 2, 2)+ .setCreateNodeSet(jettys.get(0).getNodeName())+ .process(cloudClient);+ assertEquals(0, response.getStatus());+ List<SolrClient> collectionClients = new ArrayList<>();+ for (String coreName : response.getCollectionCoresStatus().keySet()) {+ collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString()));+ }++ SolrClient client1 = collectionClients.get(0);+ SolrClient client2 = collectionClients.get(1);+ SolrClient client3 = collectionClients.get(2);+ SolrClient client4 = collectionClients.get(3);++ waitForRecoveriesToFinish(+ oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()), false);+ assertAllActive(oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()));++ client2.add(getDoc(id, "1"));+ client3.add(getDoc(id, "2"));+ client4.add(getDoc(id, "3"));++ client1.commit();+ SolrQuery query = new SolrQuery("*:*");+ query.set("distrib", false);+ long oneDocs = client1.query(query).getResults().getNumFound();+ long twoDocs = client2.query(query).getResults().getNumFound();+ long threeDocs = client3.query(query).getResults().getNumFound();+ long fourDocs = client4.query(query).getResults().getNumFound();++ query.set("collection", oneInstanceCollection);+ query.set("distrib", true);+ long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound();++ // System.out.println("1:" + oneDocs);+ // System.out.println("2:" + twoDocs);+ // System.out.println("3:" + threeDocs);+ // System.out.println("4:" + fourDocs);+ // System.out.println("All Docs:" + allDocs);++ assertEquals(3, allDocs);+ IOUtils.close(collectionClients);+ }++ private void createSolrCore(+ final String collection,+ List<SolrClient> collectionClients,+ final String baseUrl,+ final int num,+ final String shardId) {+ Callable<Object> call =+ () -> {+ try (SolrClient client = getHttpSolrClient(baseUrl)) {+ // client.setConnectionTimeout(15000);+ Create createCmd = new Create();+ createCmd.setCoreName(collection + num);+ createCmd.setCollection(collection);++ if (random().nextBoolean()) {+ // sometimes we use an explicit core node name+ createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet());+ }++ if (shardId == null) {+ createCmd.setNumShards(2);+ }+ createCmd.setDataDir(getDataDir(createTempDir(collection).toString()));+ if (shardId != null) {+ createCmd.setShardId(shardId);+ }+ client.request(createCmd);+ } catch (Exception e) {+ log.error("error creating core", e);+ // fail+ }+ return null;+ };++ pending.add(completionService.submit(call));++ collectionClients.add(createNewSolrClient(collection + num, baseUrl));+ }++ private void testMultipleCollections() throws Exception {+ log.info("### STARTING testMultipleCollections");+ // create another 2 collections and search across them+ createNewCollection("collection2");+ createNewCollection("collection3");++ while (pending != null && pending.size() > 0) {++ Future<Object> future = completionService.take();+ if (future == null) return;+ pending.remove(future);+ }++ indexDoc("collection2", getDoc(id, "10000000"));+ indexDoc("collection2", getDoc(id, "10000001"));+ indexDoc("collection2", getDoc(id, "10000003"));++ getSolrClient("collection2").add(getDoc(id, "10000004"));++ indexDoc("collection3", getDoc(id, "20000000"));+ indexDoc("collection3", getDoc(id, "20000001"));++ getSolrClient("collection3").add(getDoc(id, "10000005"));++ otherCollectionClients.get("collection2").get(0).commit();+ otherCollectionClients.get("collection3").get(0).commit();++ SolrClient clientForCollection1 = getSolrClient("collection1");+ long collection1Docs =+ clientForCollection1.query(new SolrQuery("*:*")).getResults().getNumFound();++ long collection2Docs =+ otherCollectionClients+ .get("collection2")+ .get(0)+ .query(new SolrQuery("*:*"))+ .getResults()+ .getNumFound();++ long collection3Docs =+ otherCollectionClients+ .get("collection3")+ .get(0)+ .query(new SolrQuery("*:*"))+ .getResults()+ .getNumFound();++ SolrQuery query = new SolrQuery("*:*");+ query.set("collection", "collection2,collection3");+ long found = clients.get(0).query(query).getResults().getNumFound();+ assertEquals(collection2Docs + collection3Docs, found);++ query = new SolrQuery("*:*");+ query.set("collection", "collection1,collection2,collection3");+ found = clients.get(0).query(query).getResults().getNumFound();+ assertEquals(collection1Docs + collection2Docs + collection3Docs, found);++ // try to search multiple with cloud client+ found = getCommonCloudSolrClient().query(query).getResults().getNumFound();+ assertEquals(collection1Docs + collection2Docs + collection3Docs, found);++ query.set("collection", "collection2,collection3");+ found = getCommonCloudSolrClient().query(query).getResults().getNumFound();+ assertEquals(collection2Docs + collection3Docs, found);++ query.set("collection", "collection3");+ found = getCommonCloudSolrClient().query(query).getResults().getNumFound();+ assertEquals(collection3Docs, found);++ query.remove("collection");+ found = getCommonCloudSolrClient().query(query).getResults().getNumFound();+ assertEquals(collection1Docs, found);++ assertEquals(collection3Docs, collection2Docs - 1);+ }++ protected void indexDoc(String collection, SolrInputDocument doc)+ throws IOException, SolrServerException {+ List<SolrClient> clients = otherCollectionClients.get(collection);+ int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size();+ SolrClient client = clients.get(which);+ client.add(doc);+ }++ @SuppressWarnings({"unchecked"})+ private void createNewCollection(final String collection) throws InterruptedException {+ try {+ assertEquals(+ 0,+ CollectionAdminRequest.createCollection(collection, "conf1", 2, 1)+ .setCreateNodeSet("")+ .process(cloudClient)+ .getStatus());+ } catch (Exception e) {+ log.error("error creating collection", e);+ // fails+ }+ final List<SolrClient> collectionClients = new ArrayList<>();+ otherCollectionClients.put(collection, collectionClients);+ int unique = 0;+ for (final JettySolrRunner runner : jettys) {+ unique++;+ final int frozeUnique = unique;+ Callable<Object> call =+ () -> {+ try {+ assertTrue(+ CollectionAdminRequest.addReplicaToShard(+ collection, "shard" + ((frozeUnique % 2) + 1))+ .setNode(runner.getNodeName())+ .process(cloudClient)+ .isSuccess());+ } catch (Exception e) {+ log.error("error adding replica", e);+ // fails+ }+ return null;+ };++ collectionClients.add(createNewSolrClient(collection, runner.getBaseUrl().toString()));+ pending.add(completionService.submit(call));+ while (pending != null && pending.size() > 0) {++ Future<Object> future = completionService.take();+ if (future == null) return;+ pending.remove(future);+ }+ }+ }++ @Override+ protected SolrClient createNewSolrClient(String collection, String baseUrl) {++ SolrClient client = getHttpSolrClient(baseUrl, collection);++ return client;+ }++ /**+ * @param collection the name of a collection or core to set as the "default" on the created+ * client.+ * @param baseUrl the "base" URL of a Solr node. Should <em>not</em> contain a collection or core+ * name.+ * @param connectionTimeoutMillis the HTTP connection timeout in milliseconds+ * @param socketTimeoutMillis the HTTP socket-read timeout in milliseconds+ */+ protected SolrClient createNewSolrClient(+ String collection, String baseUrl, int connectionTimeoutMillis, int socketTimeoutMillis) {++ SolrClient client =+ new HttpSolrClient.Builder(baseUrl)+ .withDefaultCollection(collection)+ .withConnectionTimeout(connectionTimeoutMillis, TimeUnit.MILLISECONDS)+ .withSocketTimeout(socketTimeoutMillis, TimeUnit.MILLISECONDS)+ .build();++ return client;+ }++ @Override+ protected QueryResponse queryRandomShard(ModifiableSolrParams params)+ throws SolrServerException, IOException {++ if (r.nextBoolean()) return super.queryRandomShard(params);++ if (r.nextBoolean()) params.set("collection", DEFAULT_COLLECTION);++ QueryResponse rsp = getCommonCloudSolrClient().query(params);+ return rsp;+ }++ @Override+ public void distribTearDown() throws Exception {+ super.distribTearDown();+ if (otherCollectionClients != null) {+ for (List<SolrClient> clientList : otherCollectionClients.values()) {+ IOUtils.close(clientList);+ }+ }+ otherCollectionClients = null;+ List<Runnable> tasks = executor.shutdownNow();+ assertTrue(tasks.isEmpty());}}diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.javaindex 3fb1bbabc6e..3f4ab451c85 100644--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java@@ -16,11 +16,324 @@*/package org.apache.solr.cloud;+import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;+import java.util.Set;+import java.util.concurrent.TimeUnit;+import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.solr.SolrTestCaseJ4.SuppressSSL;+import org.apache.solr.client.solrj.apache.CloudLegacySolrClient;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.cloud.ZkStateReader;+import org.junit.AfterClass;+import org.junit.BeforeClass;+import org.junit.Test;-/*- * Implementation moved to AbstractChaosMonkeyNothingIsSafeTestBase.java as it is also- * used by the HDFS contrib tests.- */@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")-public class ChaosMonkeyNothingIsSafeTest extends AbstractChaosMonkeyNothingIsSafeTestBase {}+public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase {+ private static final int FAIL_TOLERANCE = 100;++ private static final Integer RUN_LENGTH =+ Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));++ private final boolean onlyLeaderIndexes = random().nextBoolean();++ @BeforeClass+ public static void beforeSuperClass() {+ schemaString = "schema15.xml"; // we need a string id+ System.setProperty("solr.autoCommit.maxTime", "15000");+ System.clearProperty("solr.httpclient.retries");+ System.clearProperty("solr.retries.on.forward");+ System.clearProperty("solr.retries.to.followers");+ setErrorHook();+ }++ @AfterClass+ public static void afterSuperClass() {+ System.clearProperty("solr.autoCommit.maxTime");+ clearErrorHook();+ }++ @Override+ protected void destroyServers() throws Exception {++ super.destroyServers();+ }++ protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};+ protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};++ private int clientSoTimeout = 60000;++ private volatile FullThrottleStoppableIndexingThread ftIndexThread;++ private final boolean runFullThrottle;++ @Override+ public String[] getFieldNames() {+ return fieldNames;+ }++ @Override+ public RandVal[] getRandValues() {+ return randVals;+ }++ @Override+ public void distribSetUp() throws Exception {+ super.distribSetUp();+ // can help to hide this when testing and looking at logs+ // ignoreException("shard update error");+ useFactory("solr.StandardDirectoryFactory");+ }++ @Override+ public void distribTearDown() throws Exception {+ try {+ ftIndexThread.safeStop();+ } catch (NullPointerException e) {+ // okay+ }+ super.distribTearDown();+ }++ public ChaosMonkeyNothingIsSafeTest() {+ super();+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1"));+ if (sliceCount == -1) {+ sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1;+ }++ int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1"));+ if (numShards == -1) {+ // we make sure that there's at least one shard with more than one replica+ // so that the ChaosMonkey has something to kill+ numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1;+ }+ fixShardCount(numShards);++ // TODO: we only do this sometimes so that we can sometimes compare against control,+ // it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer+ runFullThrottle = random().nextBoolean();+ }++ @Override+ protected boolean useTlogReplicas() {+ return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug+ // and it's TestInjection use+ }++ @Override+ protected CloudSolrClient createCloudClient(String defaultCollection) {+ return this.createCloudClient(defaultCollection, this.clientSoTimeout);+ }++ protected CloudSolrClient createCloudClient(String defaultCollection, int socketTimeout) {++ return getCloudSolrClient(+ zkServer.getZkAddress(), defaultCollection, random().nextBoolean(), 30000, socketTimeout);+ }++ @Test+ @SuppressWarnings({"try"})+ public void test() throws Exception {+ // None of the operations used here are particularly costly, so this should work.+ // Using this low timeout will also help us catch index stalling.+ clientSoTimeout = 5000;++ boolean testSuccessful = false;+ try (CloudSolrClient ourCloudClient = createCloudClient(DEFAULT_COLLECTION)) {+ handle.clear();+ handle.put("timestamp", SKIPVAL);+ ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);+ // make sure we have leaders for each shard+ for (int j = 1; j < sliceCount; j++) {+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);+ } // make sure we again have leaders for each shard++ waitForRecoveriesToFinish(false);++ // we cannot do delete by query+ // as it's not supported for recovery+ del("*:*");++ List<StoppableThread> threads = new ArrayList<>();+ List<StoppableIndexingThread> indexTreads = new ArrayList<>();+ int threadCount = TEST_NIGHTLY ? 3 : 1;+ int i = 0;+ for (i = 0; i < threadCount; i++) {+ StoppableIndexingThread indexThread =+ new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true);+ threads.add(indexThread);+ indexTreads.add(indexThread);+ indexThread.start();+ }++ threadCount = 1;+ i = 0;+ for (i = 0; i < threadCount; i++) {+ StoppableSearchThread searchThread = new StoppableSearchThread(cloudClient);+ threads.add(searchThread);+ searchThread.start();+ }++ if (runFullThrottle) {+ ftIndexThread =+ new FullThrottleStoppableIndexingThread(+ ((CloudLegacySolrClient) cloudClient).getHttpClient(),+ controlClient,+ cloudClient,+ clients,+ "ft1",+ true,+ this.clientSoTimeout);+ ftIndexThread.start();+ }++ chaosMonkey.startTheMonkey(true, 10000);+ try {+ long runLength;+ if (RUN_LENGTH != -1) {+ runLength = RUN_LENGTH;+ } else {+ int[] runTimes;+ if (TEST_NIGHTLY) {+ runTimes =+ new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000};+ } else {+ runTimes = new int[] {5000, 7000, 15000};+ }+ runLength = runTimes[random().nextInt(runTimes.length - 1)];+ }++ Thread.sleep(runLength);+ } finally {+ chaosMonkey.stopTheMonkey();+ }++ // ideally this should go into chaosMonkey+ restartZk(1000 * (5 + random().nextInt(4)));++ if (runFullThrottle) {+ ftIndexThread.safeStop();+ }++ for (StoppableThread indexThread : threads) {+ indexThread.safeStop();+ }++ // start any downed jetties to be sure we still will end up with a leader per shard...++ // wait for stop...+ for (StoppableThread indexThread : threads) {+ indexThread.join();+ }++ // try and wait for any replications and what not to finish...++ Thread.sleep(2000);++ // wait until there are no recoveries...+ waitForThingsToLevelOut();++ // make sure we again have leaders for each shard+ for (int j = 1; j < sliceCount; j++) {+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000);+ }++ commit();++ // TODO: assert we didn't kill everyone++ zkStateReader.updateLiveNodes();+ assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0);++ // we expect full throttle fails, but cloud client should not easily fail+ for (StoppableThread indexThread : threads) {+ if (indexThread instanceof StoppableIndexingThread+ && !(indexThread instanceof FullThrottleStoppableIndexingThread)) {+ int failCount = ((StoppableIndexingThread) indexThread).getFailCount();+ assertFalse(+ "There were too many update fails ("+ + failCount+ + " > "+ + FAIL_TOLERANCE+ + ") - we expect it can happen, but shouldn't easily",+ failCount > FAIL_TOLERANCE);+ }+ }++ waitForThingsToLevelOut(20, TimeUnit.SECONDS);++ commit();++ Set<String> addFails = getAddFails(indexTreads);+ Set<String> deleteFails = getDeleteFails(indexTreads);+ // full throttle thread can+ // have request fails+ checkShardConsistency(!runFullThrottle, true, addFails, deleteFails);++ long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();++ // ensure we have added more than 0 docs+ long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();++ assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0);++ if (VERBOSE)+ System.out.println(+ "control docs:"+ + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()+ + "\n\n");++ // try and make a collection to make sure the overseer has survived the expiration and session+ // loss++ // sometimes we restart zookeeper as well+ if (random().nextBoolean()) {+ // restartZk(1000 * (5 + random().nextInt(4)));+ }++ try (CloudSolrClient client = createCloudClient("collection1", 30000)) {+ createCollection(null, "testcollection", 1, 1, client, null, "conf1");+ }+ List<Integer> numShardsNumReplicas = new ArrayList<>(2);+ numShardsNumReplicas.add(1);+ numShardsNumReplicas.add(1);+ checkForCollection("testcollection", numShardsNumReplicas);++ testSuccessful = true;+ } finally {+ if (!testSuccessful) {+ printLayout();+ }+ }+ }++ private Set<String> getAddFails(List<StoppableIndexingThread> threads) {+ Set<String> addFails = new HashSet<>();+ for (StoppableIndexingThread thread : threads) {+ addFails.addAll(thread.getAddFails());+ }+ return addFails;+ }++ private Set<String> getDeleteFails(List<StoppableIndexingThread> threads) {+ Set<String> deleteFails = new HashSet<>();+ for (StoppableIndexingThread thread : threads) {+ deleteFails.addAll(thread.getDeleteFails());+ }+ return deleteFails;+ }++ // skip the randoms - they can deadlock...+ @Override+ protected void indexr(Object... fields) throws Exception {+ SolrInputDocument doc = getDoc(fields);+ indexDoc(doc);+ }+}diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.javaindex a4c1ab267d1..a75b637af22 100644--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java@@ -14,14 +14,220 @@* See the License for the specific language governing permissions and* limitations under the License.*/+package org.apache.solr.cloud;-/*- * Implementation moved to AbstractChaosMonkeySafeLeaderTestBase as is used by HDFS contrib module test- */-public class ChaosMonkeySafeLeaderTest extends AbstractChaosMonkeySafeLeaderTestBase {+import java.lang.invoke.MethodHandles;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.TimeUnit;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.common.SolrInputDocument;+import org.junit.AfterClass;+import org.junit.BeforeClass;+import org.junit.Test;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ private static final Integer RUN_LENGTH =+ Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));++ @BeforeClass+ public static void beforeSuperClass() {+ schemaString = "schema15.xml"; // we need a string id+ System.setProperty("solr.autoCommit.maxTime", "15000");+ System.clearProperty("solr.httpclient.retries");+ System.clearProperty("solr.retries.on.forward");+ System.clearProperty("solr.retries.to.followers");+ setErrorHook();+ }++ @AfterClass+ public static void afterSuperClass() {+ System.clearProperty("solr.autoCommit.maxTime");+ clearErrorHook();+ }++ protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};+ protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};++ @Override+ public String[] getFieldNames() {+ return fieldNames;+ }+@Override+ public RandVal[] getRandValues() {+ return randVals;+ }+protected String getDirectoryFactory() {return "solr.StandardDirectoryFactory";}++ @Override+ public void distribSetUp() throws Exception {+ useFactory(getDirectoryFactory());+ super.distribSetUp();+ }++ public ChaosMonkeySafeLeaderTest() {+ super();+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1"));+ if (sliceCount == -1) {+ sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1;+ }++ int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1"));+ if (numShards == -1) {+ // we make sure that there's at least one shard with more than one replica+ // so that the ChaosMonkey has something to kill+ numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1;+ }+ fixShardCount(numShards);+ }++ @Test+ public void test() throws Exception {++ handle.clear();+ handle.put("timestamp", SKIPVAL);++ // randomly turn on 1 seconds 'soft' commit+ randomlyEnableAutoSoftCommit();++ tryDelete();++ List<StoppableIndexingThread> threads = new ArrayList<>();+ int threadCount = 2;+ int batchSize = 1;+ if (random().nextBoolean()) {+ batchSize = random().nextInt(98) + 2;+ }++ boolean pauseBetweenUpdates = TEST_NIGHTLY ? random().nextBoolean() : true;+ int maxUpdates = -1;+ if (!pauseBetweenUpdates) {+ maxUpdates = 1000 + random().nextInt(1000);+ } else {+ maxUpdates = 15000;+ }++ for (int i = 0; i < threadCount; i++) {+ StoppableIndexingThread indexThread =+ new StoppableIndexingThread(+ controlClient,+ cloudClient,+ Integer.toString(i),+ true,+ maxUpdates,+ batchSize,+ pauseBetweenUpdates); // random().nextInt(999) + 1+ threads.add(indexThread);+ indexThread.start();+ }++ chaosMonkey.startTheMonkey(false, 500);+ try {+ long runLength;+ if (RUN_LENGTH != -1) {+ runLength = RUN_LENGTH;+ } else {+ int[] runTimes;+ if (TEST_NIGHTLY) {+ runTimes =+ new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000};+ } else {+ runTimes = new int[] {5000, 7000, 15000};+ }+ runLength = runTimes[random().nextInt(runTimes.length - 1)];+ }++ Thread.sleep(runLength);+ } finally {+ chaosMonkey.stopTheMonkey();+ }++ for (StoppableIndexingThread indexThread : threads) {+ indexThread.safeStop();+ }++ // wait for stop...+ for (StoppableIndexingThread indexThread : threads) {+ indexThread.join();+ }++ for (StoppableIndexingThread indexThread : threads) {+ assertEquals(0, indexThread.getFailCount());+ }++ // try and wait for any replications and what not to finish...++ Thread.sleep(2000);++ waitForThingsToLevelOut(3, TimeUnit.MINUTES);++ // even if things were leveled out, a jetty may have just been stopped or something+ // we wait again and wait to level out again to make sure the system is not still in flux++ Thread.sleep(3000);++ waitForThingsToLevelOut(3, TimeUnit.MINUTES);++ checkShardConsistency(batchSize == 1, true);++ if (VERBOSE)+ System.out.println(+ "control docs:"+ + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()+ + "\n\n");++ // try and make a collection to make sure the overseer has survived the expiration and session+ // loss++ // sometimes we restart zookeeper as well+ if (random().nextBoolean()) {+ zkServer.shutdown();+ zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());+ zkServer.run(false);+ }++ try (CloudSolrClient client = createCloudClient("collection1")) {+ createCollection(null, "testcollection", 1, 1, client, null, "conf1");+ }+ List<Integer> numShardsNumReplicas = new ArrayList<>(2);+ numShardsNumReplicas.add(1);+ numShardsNumReplicas.add(1);+ checkForCollection("testcollection", numShardsNumReplicas);+ }++ private void tryDelete() throws Exception {+ long start = System.nanoTime();+ long timeout = start + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS);+ while (System.nanoTime() < timeout) {+ try {+ del("*:*");+ break;+ } catch (SolrServerException e) {+ log.error("cluster may not be up yet", e);+ }+ Thread.sleep(100);+ }+ }++ // skip the randoms - they can deadlock...+ @Override+ protected void indexr(Object... fields) throws Exception {+ SolrInputDocument doc = new SolrInputDocument();+ addFields(doc, fields);+ addFields(doc, "rnd_b", true);+ indexDoc(doc);+ }}diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.javaindex 265cbd4d1ea..d6e00a202ee 100644--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java@@ -17,23 +17,392 @@package org.apache.solr.cloud;+import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.util.ArrayList;+import java.util.Collection;+import java.util.Collections;+import java.util.List;+import java.util.Map;+import java.util.Set;+import org.apache.solr.client.api.model.CoreStatusResponse;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.CoreAdminRequest;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.client.solrj.response.CoreAdminResponse;+import org.apache.solr.client.solrj.response.RequestStatusState;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.cloud.DocCollection;+import org.apache.solr.common.cloud.Replica;+import org.apache.solr.common.cloud.Slice;+import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.embedded.JettySolrRunner;+import org.apache.solr.util.IdUtils;+import org.junit.After;+import org.junit.Before;import org.junit.Test;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;-/**- * Implementation moved to AbstractMoveReplicaTestBase as it is referenced by HDFS contrib module- * tests.- */-public class MoveReplicaTest extends AbstractMoveReplicaTestBase {+public class MoveReplicaTest extends SolrCloudTestCase {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ protected boolean inPlaceMove = random().nextBoolean();+ protected boolean isCollectionApiDistributed = false;++ protected String getConfigSet() {+ return "cloud-dynamic";+ }++ @Before+ public void beforeTest() throws Exception {++ configureCluster(4)+ .addConfig("conf1", configset(getConfigSet()))+ .addConfig("conf2", configset(getConfigSet()))+ .withSolrXml(TEST_PATH().resolve("solr.xml"))+ .configure();++ // If Collection API is distributed let's not wait for Overseer.+ isCollectionApiDistributed =+ new CollectionAdminRequest.RequestApiDistributedProcessing()+ .process(cluster.getSolrClient())+ .getIsCollectionApiDistributed();+ if (isCollectionApiDistributed) {+ return;+ }++ NamedList<Object> overSeerStatus =+ cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());+ JettySolrRunner overseerJetty = null;+ String overseerLeader = (String) overSeerStatus.get("leader");+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {+ if (jetty.getNodeName().equals(overseerLeader)) {+ overseerJetty = jetty;+ break;+ }+ }+ if (overseerJetty == null) {+ fail("no overseer leader!");+ }+ }++ @After+ public void afterTest() throws Exception {+ try {+ shutdownCluster();+ } finally {+ super.tearDown();+ }+ }- @Override@Testpublic void test() throws Exception {- super.test();+ String coll = getTestClass().getSimpleName() + "_coll_" + inPlaceMove;+ if (log.isInfoEnabled()) {+ log.info("total_jettys: {}", cluster.getJettySolrRunners().size());+ }+ int REPLICATION = 2;++ CloudSolrClient cloudClient = cluster.getSolrClient();++ // random create tlog or pull type replicas with nrt+ boolean isTlog = random().nextBoolean();+ CollectionAdminRequest.Create create =+ CollectionAdminRequest.createCollection(+ coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0);+ cloudClient.request(create);++ addDocs(coll, 100);++ Replica replica = getRandomReplica(coll, cloudClient);+ Set<String> liveNodes = cloudClient.getClusterState().getLiveNodes();+ ArrayList<String> l = new ArrayList<>(liveNodes);+ Collections.shuffle(l, random());+ String targetNode = null;+ for (String node : liveNodes) {+ if (!replica.getNodeName().equals(node)) {+ targetNode = node;+ break;+ }+ }+ assertNotNull(targetNode);+ String shardId = null;+ for (Slice slice : cloudClient.getClusterState().getCollection(coll).getSlices()) {+ if (slice.getReplicas().contains(replica)) {+ shardId = slice.getName();+ }+ }++ int sourceNumCores =+ getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name());+ int targetNumCores = getNumOfCores(cloudClient, targetNode, coll, replica.getType().name());++ CollectionAdminRequest.MoveReplica moveReplica =+ createMoveReplicaRequest(coll, replica, targetNode);+ moveReplica.setInPlaceMove(inPlaceMove);+ String asyncId = IdUtils.randomId();+ moveReplica.processAsync(asyncId, cloudClient);+ CollectionAdminRequest.RequestStatus requestStatus =+ CollectionAdminRequest.requestStatus(asyncId);+ // wait for async request success+ boolean success = false;+ for (int i = 0; i < 200; i++) {+ CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);+ if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) {+ success = true;+ break;+ }+ assertNotSame(rsp.getRequestStatus(), RequestStatusState.FAILED);+ Thread.sleep(500);+ }+ assertTrue(success);+ assertEquals(+ "should be one less core on the source node!",+ sourceNumCores - 1,+ getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name()));+ assertEquals(+ "should be one more core on target node!",+ targetNumCores + 1,+ getNumOfCores(cloudClient, targetNode, coll, replica.getType().name()));+ // wait for recovery+ boolean recovered = false;+ for (int i = 0; i < 300; i++) {+ DocCollection collState = getCollectionState(coll);+ log.debug("###### {}", collState);+ Collection<Replica> replicas = collState.getSlice(shardId).getReplicas();+ boolean allActive = true;+ boolean hasLeaders = true;+ if (replicas != null && !replicas.isEmpty()) {+ for (Replica r : replicas) {+ if (!r.getNodeName().equals(targetNode)) {+ continue;+ }+ if (!r.isActive(Collections.singleton(targetNode))) {+ log.info("Not active: {}", r);+ allActive = false;+ }+ }+ } else {+ allActive = false;+ }+ for (Slice slice : collState.getSlices()) {+ if (slice.getLeader() == null) {+ hasLeaders = false;+ }+ }+ if (allActive && hasLeaders) {+ // check the number of active replicas+ assertEquals("total number of replicas", REPLICATION, replicas.size());+ recovered = true;+ break;+ } else {+ log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders);+ Thread.sleep(1000);+ }+ }+ assertTrue("replica never fully recovered", recovered);++ assertEquals(+ 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());++ moveReplica = createMoveReplicaRequest(coll, replica, targetNode, shardId);+ moveReplica.setInPlaceMove(inPlaceMove);+ moveReplica.process(cloudClient);+ checkNumOfCores(cloudClient, replica.getNodeName(), coll, sourceNumCores);+ // wait for recovery+ recovered = false;+ for (int i = 0; i < 300; i++) {+ DocCollection collState = getCollectionState(coll);+ log.debug("###### {}", collState);+ Collection<Replica> replicas = collState.getSlice(shardId).getReplicas();+ boolean allActive = true;+ boolean hasLeaders = true;+ if (replicas != null && !replicas.isEmpty()) {+ for (Replica r : replicas) {+ if (!r.getNodeName().equals(replica.getNodeName())) {+ continue;+ }+ if (!r.isActive(Collections.singleton(replica.getNodeName()))) {+ log.info("Not active yet: {}", r);+ allActive = false;+ }+ }+ } else {+ allActive = false;+ }+ for (Slice slice : collState.getSlices()) {+ if (slice.getLeader() == null) {+ hasLeaders = false;+ }+ }+ if (allActive && hasLeaders) {+ assertEquals("total number of replicas", REPLICATION, replicas.size());+ recovered = true;+ break;+ } else {+ Thread.sleep(1000);+ }+ }+ assertTrue("replica never fully recovered", recovered);++ assertEquals(+ 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());}- @Override@Testpublic void testFailedMove() throws Exception {- super.testFailedMove();+ String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove;+ int REPLICATION = 2;++ CloudSolrClient cloudClient = cluster.getSolrClient();++ // random create tlog or pull type replicas with nrt+ boolean isTlog = random().nextBoolean();+ CollectionAdminRequest.Create create =+ CollectionAdminRequest.createCollection(+ coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0);+ cloudClient.request(create);++ addDocs(coll, 100);++ NamedList<Object> overSeerStatus =+ cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());+ String overseerLeader = (String) overSeerStatus.get("leader");++ // don't kill overseer in this test+ Replica replica;+ int count = 10;+ do {+ replica = getRandomReplica(coll, cloudClient);+ } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0);+ assertNotNull("could not find non-overseer replica???", replica);+ Set<String> liveNodes = cloudClient.getClusterState().getLiveNodes();+ ArrayList<String> l = new ArrayList<>(liveNodes);+ Collections.shuffle(l, random());+ String targetNode = null;+ for (String node : liveNodes) {+ if (!replica.getNodeName().equals(node)+ && (isCollectionApiDistributed || !overseerLeader.equals(node))) {+ targetNode = node;+ break;+ }+ }+ assertNotNull(targetNode);+ CollectionAdminRequest.MoveReplica moveReplica =+ createMoveReplicaRequest(coll, replica, targetNode);+ moveReplica.setInPlaceMove(inPlaceMove);+ // start moving+ String asyncId = IdUtils.randomId();+ moveReplica.processAsync(asyncId, cloudClient);+ // shut down target node+ for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {+ if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) {+ JettySolrRunner j = cluster.stopJettySolrRunner(i);+ cluster.waitForJettyToStop(j);+ break;+ }+ }+ CollectionAdminRequest.RequestStatus requestStatus =+ CollectionAdminRequest.requestStatus(asyncId);+ // wait for async request success+ boolean success = true;+ for (int i = 0; i < 200; i++) {+ CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);+ assertNotSame(+ rsp.getRequestStatus().toString(), rsp.getRequestStatus(), RequestStatusState.COMPLETED);+ if (rsp.getRequestStatus() == RequestStatusState.FAILED) {+ success = false;+ break;+ }+ Thread.sleep(500);+ }+ assertFalse(success);++ if (log.isInfoEnabled()) {+ log.info(+ "--- current collection state: {}", cloudClient.getClusterState().getCollection(coll));+ }+ assertEquals(+ 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());+ }++ private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(+ String coll, Replica replica, String targetNode, String shardId) {+ return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName());+ }++ private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(+ String coll, Replica replica, String targetNode) {+ return new CollectionAdminRequest.MoveReplica(coll, replica.getName(), targetNode);+ }++ private Replica getRandomReplica(String coll, CloudSolrClient cloudClient) throws IOException {+ List<Replica> replicas = cloudClient.getClusterState().getCollection(coll).getReplicas();+ Collections.shuffle(replicas, random());+ return replicas.get(0);+ }++ private void checkNumOfCores(+ CloudSolrClient cloudClient, String nodeName, String collectionName, int expectedCores)+ throws IOException, SolrServerException {+ assertEquals(+ nodeName + " does not have expected number of cores",+ expectedCores,+ getNumOfCores(cloudClient, nodeName, collectionName));+ }++ private int getNumOfCores(CloudSolrClient cloudClient, String nodeName, String collectionName)+ throws IOException, SolrServerException {+ return getNumOfCores(cloudClient, nodeName, collectionName, null);+ }++ private int getNumOfCores(+ CloudSolrClient cloudClient, String nodeName, String collectionName, String replicaType)+ throws IOException, SolrServerException {+ try (SolrClient coreclient =+ getHttpSolrClient(ZkStateReader.from(cloudClient).getBaseUrlForNodeName(nodeName))) {+ CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient);+ if (status.getCoreStatus().size() == 0) {+ return 0;+ }+ if (collectionName == null && replicaType == null) {+ return status.getCoreStatus().size();+ }+ // filter size by collection name+ int size = 0;+ for (Map.Entry<String, CoreStatusResponse.SingleCoreData> coreStatusEntry :+ status.getCoreStatus().entrySet()) {+ if (collectionName != null) {+ String coll = coreStatusEntry.getValue().cloud.collection;+ if (!collectionName.equals(coll)) {+ continue;+ }+ }+ if (replicaType != null) {+ String type = coreStatusEntry.getValue().cloud.replicaType;+ if (!replicaType.equals(type)) {+ continue;+ }+ }+ size++;+ }+ return size;+ }+ }++ protected void addDocs(String collection, int numDocs) throws Exception {+ SolrClient solrClient = cluster.getSolrClient();+ for (int docId = 1; docId <= numDocs; docId++) {+ SolrInputDocument doc = new SolrInputDocument();+ doc.addField("id", docId);+ solrClient.add(collection, doc);+ }+ solrClient.commit(collection);+ Thread.sleep(5000);}}diff --git a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.javaindex b0188069562..4c0ad7af750 100644--- a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java@@ -122,7 +122,7 @@ public class OutOfBoxZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {.withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS).build();try {- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(+ VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true, true, true, true, true, true);} finally {zkClient.close();diff --git a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.javaindex 5810c50fe17..2f3f2201ede 100644--- a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java@@ -153,7 +153,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {try (SolrZkClient zkClient =new SolrZkClientFactoryUsingCompletelyNewProviders(null, null, null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) {- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(+ VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false, false, false, false, false, false);}}@@ -165,7 +165,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPasswordWrong", null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) {- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(+ VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false, false, false, false, false, false);}}@@ -176,7 +176,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPassword", null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) {- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(+ VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true, true, true, true, true, true);}}@@ -188,7 +188,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {new SolrZkClientFactoryUsingCompletelyNewProviders("readonlyACLUsername", "readonlyACLPassword", null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) {- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(+ VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, false, false, false, false, false, false, false, false);}}@@ -204,7 +204,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {new SolrZkClient.Builder().withUrl(zkServer.getZkAddress()).withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) {- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(+ VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false, false, false, false, false, false);}}@@ -220,7 +220,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {new SolrZkClient.Builder().withUrl(zkServer.getZkAddress()).withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) {- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(+ VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false, false, false, false, false, false);}}@@ -236,7 +236,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {new SolrZkClient.Builder().withUrl(zkServer.getZkAddress()).withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) {- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(+ VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true, true, true, true, true, true);}}@@ -252,7 +252,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {new SolrZkClient.Builder().withUrl(zkServer.getZkAddress()).withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) {- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(+ VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, false, false, false, false, false, false, false, false);}}diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.javaindex 6394eb566e0..5ea4794ff4b 100644--- a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java@@ -16,16 +16,143 @@*/package org.apache.solr.cloud;+import java.lang.invoke.MethodHandles;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.TimeUnit;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.apache.CloudLegacySolrClient;+import org.apache.solr.client.solrj.apache.HttpSolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.common.cloud.DocCollection;+import org.apache.solr.common.cloud.Replica;+import org.apache.solr.common.cloud.Slice;+import org.apache.solr.embedded.JettySolrRunner;+import org.junit.After;+import org.junit.BeforeClass;import org.junit.Test;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;-/*- * Implementation moved to AbstractRecoveryZkTestBase as it is used by HDFS contrib tests.- */-public class RecoveryZkTest extends AbstractRecoveryZkTestBase {+public class RecoveryZkTest extends SolrCloudTestCase {++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ @BeforeClass+ public static void setupCluster() throws Exception {+ cluster = configureCluster(2).addConfig("conf", configset("cloud-minimal")).configure();+ }++ private final List<StoppableIndexingThread> threads = new ArrayList<>();++ @After+ public void stopThreads() throws InterruptedException {+ for (StoppableIndexingThread t : threads) {+ t.safeStop();+ }+ for (StoppableIndexingThread t : threads) {+ t.join();+ }+ threads.clear();+ }@Test- @Overridepublic void test() throws Exception {- super.test();++ final String collection = "recoverytest";++ CollectionAdminRequest.createCollection(collection, "conf", 1, 2)+ .process(cluster.getSolrClient());+ waitForState(+ "Expected a collection with one shard and two replicas", collection, clusterShape(1, 2));++ // start a couple indexing threads++ int[] maxDocList = new int[] {300, 700, 1200, 1350, 3000};+ int[] maxDocNightlyList = new int[] {3000, 7000, 12000, 30000, 45000, 60000};++ int maxDoc;+ if (!TEST_NIGHTLY) {+ maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];+ } else {+ maxDoc = maxDocNightlyList[random().nextInt(maxDocList.length - 1)];+ }+ log.info("Indexing {} documents", maxDoc);++ try (SolrClient solrClient =+ cluster.basicSolrClientBuilder().withDefaultCollection(collection).build(); ) {+ final StoppableIndexingThread indexThread =+ new StoppableIndexingThread(null, solrClient, "1", true, maxDoc, 1, true);+ threads.add(indexThread);+ indexThread.start();++ final StoppableIndexingThread indexThread2 =+ new StoppableIndexingThread(null, solrClient, "2", true, maxDoc, 1, true);+ threads.add(indexThread2);+ indexThread2.start();++ // give some time to index...+ int[] waitTimes = new int[] {200, 2000, 3000};+ Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);++ // bring shard replica down+ DocCollection state = getCollectionState(collection);+ Replica leader = state.getLeader("shard1");+ Replica replica = getRandomReplica(state.getSlice("shard1"), (r) -> !leader.equals(r));++ JettySolrRunner jetty = cluster.getReplicaJetty(replica);+ jetty.stop();++ // wait a moment - lets allow some docs to be indexed so replication time is non 0+ Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);++ // bring shard replica up+ jetty.start();++ // make sure replication can start+ Thread.sleep(3000);++ // stop indexing threads+ indexThread.safeStop();+ indexThread2.safeStop();++ indexThread.join();+ indexThread2.join();++ new UpdateRequest().commit(solrClient, collection);++ cluster+ .getZkStateReader()+ .waitForState(collection, 120, TimeUnit.SECONDS, clusterShape(1, 2));++ // test that leader and replica have same doc count+ state = getCollectionState(collection);+ assertShardConsistency(state.getSlice("shard1"), true);+ }+ }++ private void assertShardConsistency(Slice shard, boolean expectDocs) throws Exception {+ List<Replica> replicas = shard.getReplicas(r -> r.getState() == Replica.State.ACTIVE);+ long[] numCounts = new long[replicas.size()];+ int i = 0;+ for (Replica replica : replicas) {+ try (var client =+ new HttpSolrClient.Builder(replica.getBaseUrl())+ .withDefaultCollection(replica.getCoreName())+ .withHttpClient(((CloudLegacySolrClient) cluster.getSolrClient()).getHttpClient())+ .build()) {+ numCounts[i] =+ client.query(new SolrQuery("*:*").add("distrib", "false")).getResults().getNumFound();+ i++;+ }+ }+ for (int j = 1; j < replicas.size(); j++) {+ if (numCounts[j] != numCounts[j - 1])+ fail("Mismatch in counts between replicas"); // TODO improve this!+ if (numCounts[j] == 0 && expectDocs)+ fail("Expected docs on shard " + shard.getName() + " but found none");+ }}}diff --git a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.javaindex 1bc74fff254..22b0084e989 100644--- a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java@@ -16,23 +16,186 @@*/package org.apache.solr.cloud;+import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.TimeUnit;import org.apache.lucene.tests.util.LuceneTestCase.Nightly;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.cloud.ZkStateReader;+import org.junit.AfterClass;+import org.junit.BeforeClass;import org.junit.Test;-/**- * Implementation moved to AbstractRestartWhileUpdatingTestBase because it is used by HDFS contrib- * module tests- */@Nightly-public class RestartWhileUpdatingTest extends AbstractRestartWhileUpdatingTestBase {+public class RestartWhileUpdatingTest extends AbstractFullDistribZkTestBase {++ // private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain";+ private List<StoppableIndexingThread> threads;++ private volatile boolean stopExpire = false;public RestartWhileUpdatingTest() throws Exception {super();+ sliceCount = 1;+ fixShardCount(3);+ schemaString = "schema15.xml"; // we need a string id+ useFactory("solr.StandardDirectoryFactory");}- @Test+ public static String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};+ public static RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};+@Override+ protected String[] getFieldNames() {+ return fieldNames;+ }++ @Override+ protected RandVal[] getRandValues() {+ return randVals;+ }++ @BeforeClass+ public static void beforeRestartWhileUpdatingTest() {+ System.setProperty("leaderVoteWait", "300000");+ System.setProperty("solr.autoCommit.maxTime", "30000");+ System.setProperty("solr.autoSoftCommit.maxTime", "3000");+ // SOLR-13212 // TestInjection.nonGracefullClose = "true:60";+ // SOLR-13189 // TestInjection.failReplicaRequests = "true:03";+ }++ @AfterClass+ public static void afterRestartWhileUpdatingTest() {+ System.clearProperty("leaderVoteWait");+ System.clearProperty("solr.autoCommit.maxTime");+ System.clearProperty("solr.autoSoftCommit.maxTime");+ }++ @Testpublic void test() throws Exception {- super.test();+ handle.clear();+ handle.put("timestamp", SKIPVAL);++ // start a couple indexing threads++ int[] maxDocList = new int[] {5000, 10000};++ int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];++ int numThreads = random().nextInt(4) + 1;++ threads = new ArrayList<>(numThreads);++ Thread expireThread =+ new Thread("expireThread") {+ @Override+ public void run() {+ while (!stopExpire) {+ try {+ Thread.sleep(random().nextInt(15000));+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }++ // try {+ // chaosMonkey.expireRandomSession();+ // } catch (KeeperException e) {+ // throw new RuntimeException(e);+ // } catch (InterruptedException e) {+ // throw new RuntimeException(e);+ // }+ }+ }+ };++ // Currently unused+ // expireThread.start();++ StoppableIndexingThread indexThread;+ for (int i = 0; i < numThreads; i++) {+ indexThread =+ new StoppableIndexingThread(+ controlClient, cloudClient, Integer.toString(i), true, maxDoc, 1, true);+ threads.add(indexThread);+ indexThread.start();+ }++ Thread.sleep(2000);++ int restartTimes = 1; // random().nextInt(4) + 1;;+ for (int i = 0; i < restartTimes; i++) {+ Thread.sleep(random().nextInt(30000));+ stopAndStartAllReplicas();+ Thread.sleep(random().nextInt(30000));+ }++ Thread.sleep(2000);++ // stop indexing threads+ for (StoppableIndexingThread thread : threads) {+ thread.safeStop();+ }+ stopExpire = true;+ expireThread.join();++ Thread.sleep(1000);++ waitForThingsToLevelOut(320, TimeUnit.SECONDS);++ Thread.sleep(2000);++ waitForThingsToLevelOut(30, TimeUnit.SECONDS);++ Thread.sleep(5000);++ waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true);++ for (StoppableIndexingThread thread : threads) {+ thread.join();+ }++ checkShardConsistency(false, false);+ }++ public void stopAndStartAllReplicas() throws Exception, InterruptedException {+ chaosMonkey.stopAll(random().nextInt(1));++ if (random().nextBoolean()) {+ for (StoppableIndexingThread thread : threads) {+ thread.safeStop();+ }+ }+ Thread.sleep(1000);++ chaosMonkey.startAll();+ }++ @Override+ protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException {+ cloudClient.add(doc);+ }++ @Override+ public void distribTearDown() throws Exception {+ // make sure threads have been stopped...+ if (threads != null) {+ for (StoppableIndexingThread thread : threads) {+ thread.safeStop();+ thread.safeStop();+ }+ }++ super.distribTearDown();+ }++ // skip the randoms - they can deadlock...+ @Override+ protected void indexr(Object... fields) throws Exception {+ SolrInputDocument doc = new SolrInputDocument();+ addFields(doc, fields);+ addFields(doc, "rnd_b", true);+ indexDoc(doc);}}diff --git a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.javaindex bdff2a6a477..8f49afba970 100644--- a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java@@ -16,8 +16,285 @@*/package org.apache.solr.cloud;-/**- * Test sync phase that occurs when Leader goes down and a new Leader is elected. Implementation- * moved to AbstractSyncSliceTestBase.java as it is also used by the HDFS contrib tests.- */-public class SyncSliceTest extends AbstractSyncSliceTestBase {}+import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.util.ArrayList;+import java.util.Collection;+import java.util.HashSet;+import java.util.List;+import java.util.Set;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.stream.Collectors;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.SolrRequest;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.apache.HttpSolrClient;+import org.apache.solr.client.solrj.request.GenericSolrRequest;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.cloud.Replica;+import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.params.CollectionParams.CollectionAction;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.util.LogLevel;+import org.junit.Test;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/** Test sync phase that occurs when Leader goes down and a new Leader is elected. */+@LogLevel("org.apache.solr.update.processor.DistributedZkUpdateProcessor=WARN")+public class SyncSliceTest extends AbstractFullDistribZkTestBase {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ private boolean success = false;++ @Override+ public void distribTearDown() throws Exception {+ if (!success) {+ printLayoutOnTearDown = true;+ }+ super.distribTearDown();+ }++ public SyncSliceTest() {+ super();+ sliceCount = 1;+ fixShardCount(TEST_NIGHTLY ? 7 : 4);+ }++ @Test+ public void test() throws Exception {++ handle.clear();+ handle.put("timestamp", SKIPVAL);++ waitForThingsToLevelOut(30, TimeUnit.SECONDS);++ del("*:*");+ List<CloudJettyRunner> skipServers = new ArrayList<>();+ int docId = 0;+ indexDoc(+ skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");++ indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "old haven was blue.");++ skipServers.add(shardToJetty.get("shard1").get(1));++ indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "but the song was fancy.");++ skipServers.add(shardToJetty.get("shard1").get(2));++ indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "under the moon and over the lake");++ commit();++ waitForRecoveriesToFinish(false);++ // shard should be inconsistent+ String shardFailMessage = checkShardConsistency("shard1", true, false);+ assertNotNull(shardFailMessage);++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("action", CollectionAction.SYNCSHARD.toString());+ params.set("collection", "collection1");+ params.set("shard", "shard1");+ final var request =+ new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/collections", params);++ String baseUrl = shardToJetty.get(SHARD1).get(2).jetty.getBaseUrl().toString();++ // we only set the connect timeout, not so timeout+ try (SolrClient baseClient =+ new HttpSolrClient.Builder(baseUrl)+ .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)+ .build()) {+ baseClient.request(request);+ }++ waitForThingsToLevelOut(15, TimeUnit.SECONDS);++ checkShardConsistency(false, true);++ long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();+ assertEquals(4, cloudClientDocs);++ // kill the leader - new leader could have all the docs or be missing one+ CloudJettyRunner leaderJetty = shardToLeaderJetty.get("shard1");++ skipServers = getRandomOtherJetty(leaderJetty, null); // but not the leader++ // this doc won't be on one node+ indexDoc(+ skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");+ commit();++ Set<CloudJettyRunner> jetties = new HashSet<>();+ jetties.addAll(shardToJetty.get("shard1"));+ jetties.remove(leaderJetty);+ assertEquals(getShardCount() - 1, jetties.size());++ leaderJetty.jetty.stop();++ Thread.sleep(3000);++ waitForNoShardInconsistency();++ Thread.sleep(1000);++ checkShardConsistency(false, true);++ cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();+ assertEquals(5, cloudClientDocs);++ CloudJettyRunner deadJetty = leaderJetty;++ // let's get the latest leader+ while (deadJetty.equals(leaderJetty)) {+ updateMappingsFromZk(this.jettys, this.clients);+ leaderJetty = shardToLeaderJetty.get("shard1");+ }++ // bring back dead node+ deadJetty.jetty.start(); // he is not the leader anymore++ waitTillAllNodesActive();++ skipServers = getRandomOtherJetty(leaderJetty, deadJetty);+ skipServers.addAll(getRandomOtherJetty(leaderJetty, deadJetty));+ // skip list should be++ // System.out.println("leader:" + leaderJetty.url);+ // System.out.println("dead:" + deadJetty.url);+ // System.out.println("skip list:" + skipServers);++ // we are skipping 2 nodes+ assertEquals(2, skipServers.size());++ // more docs than can peer sync+ for (int i = 0; i < 300; i++) {+ indexDoc(+ skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");+ }++ commit();+ waitForRecoveriesToFinish(false);++ // shard should be inconsistent+ shardFailMessage = waitTillInconsistent();+ assertNotNull(+ "Test Setup Failure: shard1 should have just been set up to be inconsistent - but it's still consistent. Leader:"+ + leaderJetty.url+ + " Dead Guy:"+ + deadJetty.url+ + "skip list:"+ + skipServers,+ shardFailMessage);++ // good place to test compareResults+ boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient);+ assertTrue("A test that compareResults is working correctly failed", shouldFail);++ jetties = new HashSet<>();+ jetties.addAll(shardToJetty.get("shard1"));+ jetties.remove(leaderJetty);+ assertEquals(getShardCount() - 1, jetties.size());++ // kill the current leader+ leaderJetty.jetty.stop();++ waitForNoShardInconsistency();++ checkShardConsistency(true, true);++ success = true;+ }++ private void waitTillAllNodesActive() throws InterruptedException, TimeoutException {+ ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);++ zkStateReader.waitForState(+ "collection1",+ 3,+ TimeUnit.MINUTES,+ (n, c) -> {+ Collection<Replica> replicas = c.getSlice("shard1").getReplicas();+ Set<String> nodes =+ replicas.stream().map(Replica::getNodeName).collect(Collectors.toSet());+ return replicas.stream().map(Replica::getState).allMatch(Replica.State.ACTIVE::equals)+ && n.containsAll(nodes);+ });+ }++ private String waitTillInconsistent() throws Exception, InterruptedException {+ String shardFailMessage = null;++ shardFailMessage = pollConsistency(shardFailMessage, 0);+ shardFailMessage = pollConsistency(shardFailMessage, 3000);+ shardFailMessage = pollConsistency(shardFailMessage, 5000);+ shardFailMessage = pollConsistency(shardFailMessage, 15000);++ return shardFailMessage;+ }++ private String pollConsistency(String shardFailMessage, int sleep)+ throws InterruptedException, Exception {+ try {+ commit();+ } catch (Throwable t) {+ log.error("commit error", t);+ }+ if (shardFailMessage == null) {+ // try again+ Thread.sleep(sleep);+ shardFailMessage = checkShardConsistency("shard1", true, false);+ }+ return shardFailMessage;+ }++ private List<CloudJettyRunner> getRandomOtherJetty(+ CloudJettyRunner leader, CloudJettyRunner down) {+ List<CloudJettyRunner> skipServers = new ArrayList<>();+ List<CloudJettyRunner> candidates = new ArrayList<>();+ candidates.addAll(shardToJetty.get("shard1"));++ if (leader != null) {+ candidates.remove(leader);+ }++ if (down != null) {+ candidates.remove(down);+ }++ CloudJettyRunner cjetty = candidates.get(random().nextInt(candidates.size()));+ skipServers.add(cjetty);+ return skipServers;+ }++ protected void indexDoc(List<CloudJettyRunner> skipServers, Object... fields)+ throws IOException, SolrServerException {+ SolrInputDocument doc = new SolrInputDocument();++ addFields(doc, fields);+ addFields(doc, "rnd_b", true);++ controlClient.add(doc);++ UpdateRequest ureq = new UpdateRequest();+ ureq.add(doc);+ for (CloudJettyRunner skip : skipServers) {+ ureq.getParams().add("test.distrib.skip.servers", skip.url + "/");+ }+ ureq.process(cloudClient);+ }++ // skip the randoms - they can deadlock...+ @Override+ protected void indexr(Object... fields) throws Exception {+ SolrInputDocument doc = new SolrInputDocument();+ addFields(doc, fields);+ addFields(doc, "rnd_b", true);+ indexDoc(doc);+ }+}diff --git a/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java b/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.javaindex e056c40a893..2fbd5128177 100644--- a/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java@@ -16,26 +16,134 @@*/package org.apache.solr.cloud;+import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.TimeUnit;import org.apache.lucene.tests.util.LuceneTestCase.Nightly;import org.apache.solr.SolrTestCaseJ4.SuppressSSL;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.embedded.JettySolrRunner;+import org.apache.solr.util.TestInjection;+import org.junit.AfterClass;+import org.junit.BeforeClass;import org.junit.Test;@Nightly@SuppressSSL-/*- * Implementation moved to AbstractTlogReplayBufferedWhileIndexingTestBase as it is also used by HDFS contrib- * module tests- */-public class TlogReplayBufferedWhileIndexingTest- extends AbstractTlogReplayBufferedWhileIndexingTestBase {+public class TlogReplayBufferedWhileIndexingTest extends AbstractFullDistribZkTestBase {++ private List<StoppableIndexingThread> threads;public TlogReplayBufferedWhileIndexingTest() throws Exception {super();+ sliceCount = 1;+ fixShardCount(2);+ schemaString = "schema15.xml"; // we need a string id+ }++ @BeforeClass+ public static void beforeRestartWhileUpdatingTest() throws Exception {+ System.setProperty("leaderVoteWait", "300000");+ System.setProperty("solr.autoCommit.maxTime", "10000");+ System.setProperty("solr.autoSoftCommit.maxTime", "3000");+ TestInjection.updateLogReplayRandomPause = "true:10";+ TestInjection.updateRandomPause = "true:10";+ useFactory("solr.StandardDirectoryFactory");+ }++ @AfterClass+ public static void afterRestartWhileUpdatingTest() {+ System.clearProperty("leaderVoteWait");+ System.clearProperty("solr.autoCommit.maxTime");+ System.clearProperty("solr.autoSoftCommit.maxTime");}@Test- @Overridepublic void test() throws Exception {- super.test();+ handle.clear();+ handle.put("timestamp", SKIPVAL);++ waitForRecoveriesToFinish(false);++ int numThreads = 3;++ threads = new ArrayList<>(numThreads);++ ArrayList<JettySolrRunner> allJetty = new ArrayList<>();+ allJetty.addAll(jettys);+ allJetty.remove(shardToLeaderJetty.get("shard1").jetty);+ assert allJetty.size() == 1 : allJetty.size();+ allJetty.get(0).stop();++ StoppableIndexingThread indexThread;+ for (int i = 0; i < numThreads; i++) {+ boolean pauseBetweenUpdates = random().nextBoolean();+ int batchSize = random().nextInt(4) + 1;+ indexThread =+ new StoppableIndexingThread(+ controlClient,+ cloudClient,+ Integer.toString(i),+ true,+ 900,+ batchSize,+ pauseBetweenUpdates);+ threads.add(indexThread);+ indexThread.start();+ }++ Thread.sleep(2000);++ allJetty.get(0).start();++ Thread.sleep(45000);++ waitForThingsToLevelOut(); // we can insert random update delays, so this can take a while,+ // especially when beasting this test++ Thread.sleep(2000);++ waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true);++ for (StoppableIndexingThread thread : threads) {+ thread.safeStop();+ }++ waitForThingsToLevelOut(30, TimeUnit.SECONDS);++ checkShardConsistency(false, false);+ }++ @Override+ protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException {+ cloudClient.add(doc);+ }++ @Override+ public void distribTearDown() throws Exception {+ // make sure threads have been stopped...+ if (threads != null) {+ for (StoppableIndexingThread thread : threads) {+ thread.safeStop();+ }++ for (StoppableIndexingThread thread : threads) {+ thread.join();+ }+ }++ super.distribTearDown();+ }++ // skip the randoms - they can deadlock...+ @Override+ protected void indexr(Object... fields) throws Exception {+ SolrInputDocument doc = new SolrInputDocument();+ addFields(doc, fields);+ addFields(doc, "rnd_b", true);+ indexDoc(doc);}}diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.javaindex 30d3f884b97..7e66787a4a5 100644--- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java@@ -16,12 +16,450 @@*/package org.apache.solr.cloud;+import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.nio.file.Path;+import java.util.Collection;+import java.util.Collections;+import java.util.Random;+import java.util.Set;+import java.util.concurrent.SynchronousQueue;+import java.util.concurrent.ThreadPoolExecutor;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import org.apache.solr.SolrTestCaseJ4.SuppressSSL;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.apache.HttpSolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.cloud.Replica;+import org.apache.solr.common.cloud.Slice;+import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.util.ExecutorUtil;+import org.apache.solr.common.util.SolrNamedThreadFactory;+import org.apache.solr.core.SolrCore;+import org.apache.solr.core.SolrPaths;+import org.apache.solr.embedded.JettySolrRunner;+import org.apache.solr.util.TestInjection;+import org.junit.Test;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;/*** This test simply does a bunch of basic things in solrcloud mode and asserts things work as- * expected. Implementation moved to AbstractUnloadDistributedZkTestBase as it is used by HDFS- * contrib module tests.+ * expected.*/@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")-public class UnloadDistributedZkTest extends AbstractUnloadDistributedZkTestBase {}+public class UnloadDistributedZkTest extends AbstractFullDistribZkTestBase {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ public UnloadDistributedZkTest() {+ super();+ fixShardCount(4); // needs at least 4 servers+ }++ @Override+ protected String getSolrXml() {+ return "solr.xml";+ }++ @Test+ public void test() throws Exception {+ jettys.forEach(+ j -> {+ Set<Path> allowPath = j.getCoreContainer().getAllowPaths();+ allowPath.clear();+ allowPath.add(SolrPaths.ALL_PATH); // Allow non-standard core instance path+ });+ log.info("###Starting testCoreUnloadAndLeaders");+ testCoreUnloadAndLeaders(); // long+ log.info("###Starting testUnloadOfCores");+ testUnloadLotsOfCores(); // long+ log.info("###Starting testUnloadShardAndCollection");+ testUnloadShardAndCollection();+ }++ /**+ * @param url a Solr node base URL. Should <em>not</em> contain a core or collection name.+ */+ private SolrClient newSolrClient(String url) {+ return new HttpSolrClient.Builder(url)+ .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)+ .withSocketTimeout(30000, TimeUnit.MILLISECONDS)+ .build();+ }++ private void checkCoreNamePresenceAndSliceCount(+ String collectionName, String coreName, boolean shouldBePresent, int expectedSliceCount)+ throws Exception {+ ZkStateReader reader = ZkStateReader.from(cloudClient);+ try {+ reader.waitForState(+ collectionName,+ 45,+ TimeUnit.SECONDS,+ c -> {+ final Collection<Slice> slices = (c != null) ? c.getSlices() : Collections.emptyList();+ if (expectedSliceCount == slices.size()) {+ for (Slice slice : slices) {+ for (Replica replica : slice.getReplicas()) {+ if (coreName.equals(replica.get("core"))) {+ return shouldBePresent;+ }+ }+ }+ return !shouldBePresent;+ } else {+ return false;+ }+ });+ } catch (TimeoutException e) {+ printLayout();+ fail(+ "checkCoreNamePresenceAndSliceCount failed:"+ + " collection="+ + collectionName+ + " CoreName="+ + coreName+ + " shouldBePresent="+ + shouldBePresent);+ }+ }++ private void testUnloadShardAndCollection() throws Exception {+ final int numShards = 2;++ final String collection = "test_unload_shard_and_collection";++ final String coreName1 = collection + "_1";+ final String coreName2 = collection + "_2";++ assertEquals(+ 0,+ CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1)+ .setCreateNodeSet("")+ .process(cloudClient)+ .getStatus());+ assertTrue(+ CollectionAdminRequest.addReplicaToShard(collection, "shard1")+ .setCoreName(coreName1)+ .setNode(jettys.get(0).getNodeName())+ .process(cloudClient)+ .isSuccess());++ assertTrue(+ CollectionAdminRequest.addReplicaToShard(collection, "shard2")+ .setCoreName(coreName2)+ .setNode(jettys.get(0).getNodeName())+ .process(cloudClient)+ .isSuccess());++ // does not mean they are active and up yet :*+ waitForRecoveriesToFinish(collection, false);++ final boolean unloadInOrder = random().nextBoolean();+ final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2);+ final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1);++ try (SolrClient adminClient = getHttpSolrClient(buildUrl(jettys.get(0).getLocalPort()))) {+ // now unload one of the two+ Unload unloadCmd = new Unload(false);+ unloadCmd.setCoreName(unloadCmdCoreName1);+ adminClient.request(unloadCmd);++ // there should still be two shards (as of SOLR-5209)+ checkCoreNamePresenceAndSliceCount(+ collection,+ unloadCmdCoreName1,+ false /* shouldBePresent */,+ numShards /* expectedSliceCount */);++ // now unload one of the other+ unloadCmd = new Unload(false);+ unloadCmd.setCoreName(unloadCmdCoreName2);+ adminClient.request(unloadCmd);+ checkCoreNamePresenceAndSliceCount(+ collection,+ unloadCmdCoreName2,+ false /* shouldBePresent */,+ numShards /* expectedSliceCount */);+ }++ // printLayout();+ // the collection should still be present (as of SOLR-5209 replica removal does not cascade to+ // remove the slice and collection)+ getCommonCloudSolrClient();+ assertTrue(+ "No longer found collection " + collection,+ cloudClient.getClusterState().hasCollection(collection));+ }++ protected SolrCore getFirstCore(String collection, JettySolrRunner jetty) {+ SolrCore solrCore = null;+ for (SolrCore core : jetty.getCoreContainer().getCores()) {+ if (core.getName().startsWith(collection)) {+ solrCore = core;+ }+ }+ return solrCore;+ }++ /**+ * @throws Exception on any problem+ */+ private void testCoreUnloadAndLeaders() throws Exception {+ JettySolrRunner jetty1 = jettys.get(0);++ assertEquals(+ 0,+ CollectionAdminRequest.createCollection("unloadcollection", "conf1", 1, 1)+ .setCreateNodeSet(jetty1.getNodeName())+ .process(cloudClient)+ .getStatus());+ getCommonCloudSolrClient();+ ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);++ zkStateReader.forceUpdateCollection("unloadcollection");++ int slices =+ zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size();+ assertEquals(1, slices);+ SolrCore solrCore = getFirstCore("unloadcollection", jetty1);+ String core1DataDir = solrCore.getDataDir();++ assertTrue(+ CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")+ .setCoreName("unloadcollection_shard1_replica2")+ .setNode(jettys.get(1).getNodeName())+ .process(cloudClient)+ .isSuccess());+ zkStateReader.forceUpdateCollection("unloadcollection");+ slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size();+ assertEquals(1, slices);++ waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);++ Replica leader = getLeaderFromZk("unloadcollection", "shard1");++ Random random = random();+ if (random.nextBoolean()) {+ try (SolrClient collectionClient =+ getHttpSolrClient(leader.getBaseUrl(), leader.getCoreName())) {+ // lets try and use the solrj client to index and retrieve a couple+ // documents+ SolrInputDocument doc1 =+ getDoc(id, 6, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");+ SolrInputDocument doc2 =+ getDoc(id, 7, i1, -600, tlong, 600, t1, "humpty dumpy3 sat on a walls");+ SolrInputDocument doc3 =+ getDoc(id, 8, i1, -600, tlong, 600, t1, "humpty dumpy2 sat on a walled");+ collectionClient.add(doc1);+ collectionClient.add(doc2);+ collectionClient.add(doc3);+ collectionClient.commit();+ }+ }++ assertTrue(+ CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")+ .setCoreName("unloadcollection_shard1_replica3")+ .setNode(jettys.get(2).getNodeName())+ .process(cloudClient)+ .isSuccess());++ waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);++ // so that we start with some versions when we reload...+ TestInjection.skipIndexWriterCommitOnClose = true;++ try (SolrClient addClient =+ new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString())+ .withDefaultCollection("unloadcollection_shard1_replica3")+ .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)+ .build()) {++ // add a few docs+ for (int x = 20; x < 100; x++) {+ SolrInputDocument doc1 =+ getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");+ addClient.add(doc1);+ }+ }+ // don't commit so they remain in the tran log+ // collectionClient.commit();++ // unload the leader+ try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) {++ Unload unloadCmd = new Unload(false);+ unloadCmd.setCoreName(leader.getCoreName());+ ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams();++ collectionClient.request(unloadCmd);+ }+ // Thread.currentThread().sleep(500);+ // printLayout();++ int tries = 50;+ while (leader+ .getCoreUrl()+ .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {+ Thread.sleep(100);+ if (tries-- == 0) {+ fail("Leader never changed");+ }+ }++ // ensure there is a leader+ zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);++ try (SolrClient addClient =+ new HttpSolrClient.Builder(jettys.get(1).getBaseUrl().toString())+ .withDefaultCollection("unloadcollection_shard1_replica2")+ .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)+ .withSocketTimeout(90000, TimeUnit.MILLISECONDS)+ .build()) {++ // add a few docs while the leader is down+ for (int x = 101; x < 200; x++) {+ SolrInputDocument doc1 =+ getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");+ addClient.add(doc1);+ }+ }++ assertTrue(+ CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")+ .setCoreName("unloadcollection_shard1_replica4")+ .setNode(jettys.get(3).getNodeName())+ .process(cloudClient)+ .isSuccess());++ waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);++ // unload the leader again+ leader = getLeaderFromZk("unloadcollection", "shard1");+ try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) {++ Unload unloadCmd = new Unload(false);+ unloadCmd.setCoreName(leader.getCoreName());+ collectionClient.request(unloadCmd);+ }+ tries = 50;+ while (leader+ .getCoreUrl()+ .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {+ Thread.sleep(100);+ if (tries-- == 0) {+ fail("Leader never changed");+ }+ }++ zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);++ TestInjection.skipIndexWriterCommitOnClose = false; // set this back+ assertTrue(+ CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")+ .setCoreName(leader.getCoreName())+ .setDataDir(core1DataDir)+ .setNode(leader.getNodeName())+ .process(cloudClient)+ .isSuccess());++ waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);++ long found1, found3;++ try (SolrClient adminClient =+ newSolrClient((jettys.get(1).getBaseUrl() + "/unloadcollection_shard1_replica2"))) {+ adminClient.commit();+ SolrQuery q = new SolrQuery("*:*");+ q.set("distrib", false);+ found1 = adminClient.query(q).getResults().getNumFound();+ }++ try (SolrClient adminClient =+ new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString())+ .withDefaultCollection("unloadcollection_shard1_replica3")+ .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)+ .withSocketTimeout(30000, TimeUnit.MILLISECONDS)+ .build()) {+ adminClient.commit();+ SolrQuery q = new SolrQuery("*:*");+ q.set("distrib", false);+ found3 = adminClient.query(q).getResults().getNumFound();+ }++ try (SolrClient adminClient =+ newSolrClient((jettys.get(3).getBaseUrl() + "/unloadcollection_shard1_replica4"))) {+ adminClient.commit();+ SolrQuery q = new SolrQuery("*:*");+ q.set("distrib", false);+ long found4 = adminClient.query(q).getResults().getNumFound();++ // all 3 shards should now have the same number of docs+ assertEquals(found1, found3);+ assertEquals(found3, found4);+ }+ }++ private void testUnloadLotsOfCores() throws Exception {+ JettySolrRunner jetty = jettys.get(0);+ int shards = TEST_NIGHTLY ? 2 : 1;+ try (final SolrClient adminClient = jetty.newClient(15000, 60000)) {+ int numReplicas = atLeast(3);+ ThreadPoolExecutor executor =+ new ExecutorUtil.MDCAwareThreadPoolExecutor(+ 0,+ Integer.MAX_VALUE,+ 5,+ TimeUnit.SECONDS,+ new SynchronousQueue<>(),+ new SolrNamedThreadFactory("testExecutor"));+ try {+ // create the cores+ BasicDistributedZkTest.createCollectionInOneInstance(+ adminClient, jetty.getNodeName(), executor, "multiunload", shards, numReplicas);+ } finally {+ ExecutorUtil.shutdownAndAwaitTermination(executor);+ }++ if (TEST_NIGHTLY == false) {+ // with nightly tests, we can try doing the unloads before the creates are done+ // it still works, but takes much longer since we end up waiting for a timeout+ waitForRecoveriesToFinish("multiunload", false);+ }++ executor =+ new ExecutorUtil.MDCAwareThreadPoolExecutor(+ 0,+ Integer.MAX_VALUE,+ 5,+ TimeUnit.SECONDS,+ new SynchronousQueue<>(),+ new SolrNamedThreadFactory("testExecutor"));+ try {+ for (int j = 0; j < numReplicas; j++) {+ final int freezeJ = j;+ executor.execute(+ () -> {+ Unload unloadCmd = new Unload(true);+ unloadCmd.setCoreName("multiunload" + freezeJ);+ try {+ adminClient.request(unloadCmd);+ } catch (SolrServerException | IOException e) {+ throw new RuntimeException(e);+ }+ });+ Thread.sleep(random().nextInt(50));+ }+ } finally {+ ExecutorUtil.shutdownAndAwaitTermination(executor);+ }+ }+ }+}diff --git a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.javaindex 82b60068d83..aca50b7bd3a 100644--- a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java@@ -16,5 +16,552 @@*/package org.apache.solr.cloud;-public class VMParamsZkACLAndCredentialsProvidersTest- extends AbstractDigestZkACLAndCredentialsProvidersTestBase {}+import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_FILE_VM_PARAM_NAME;+import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME;+import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME;+import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE;++import java.io.FileWriter;+import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.nio.charset.Charset;+import java.nio.charset.StandardCharsets;+import java.nio.file.Path;+import java.util.Collections;+import java.util.List;+import java.util.Properties;+import java.util.concurrent.TimeUnit;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.common.cloud.DigestZkACLProvider;+import org.apache.solr.common.cloud.DigestZkCredentialsProvider;+import org.apache.solr.common.cloud.SecurityAwareZkACLProvider;+import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.cloud.VMParamsZkCredentialsInjector;+import org.apache.solr.common.cloud.ZkCredentialsInjector;+import org.apache.zookeeper.CreateMode;+import org.apache.zookeeper.KeeperException.NoAuthException;+import org.junit.AfterClass;+import org.junit.BeforeClass;+import org.junit.Test;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ private static final Charset DATA_ENCODING = StandardCharsets.UTF_8;++ private static final String ALL_USERNAME = "connectAndAllACLUsername";+ private static final String ALL_PASSWORD = "connectAndAllACLPassword";+ private static final String READONLY_USERNAME = "readonlyACLUsername";+ private static final String READONLY_PASSWORD = "readonlyACLPassword";++ protected ZkTestServer zkServer;++ protected Path zkDir;++ @BeforeClass+ public static void beforeClass() {+ System.setProperty("solrcloud.skip.autorecovery", "true");+ }++ @AfterClass+ public static void afterClass() {+ System.clearProperty("solrcloud.skip.autorecovery");+ }++ @Override+ public void setUp() throws Exception {+ // TODO: Does all of this setup need to happen for each test case, or can it be done once for+ // the class? (i.e. @BeforeClass) and maybe some minor reset logic in setup, instead of full+ // startup and teardown of a new ZkTestServer in each cycle?+ super.setUp();+ if (log.isInfoEnabled()) {+ log.info("####SETUP_START {}", getTestName());+ }+ createTempDir();++ zkDir = createTempDir().resolve("zookeeper/server1/data");+ log.info("ZooKeeper dataDir:{}", zkDir);+ setSecuritySystemProperties();+ zkServer = new ZkTestServer(zkDir);+ zkServer.run(false);++ System.setProperty("zkHost", zkServer.getZkAddress());++ setDigestZkSystemProps();+ System.setProperty(+ SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,+ AllAndReadonlyCredentialZkCredentialsInjector.class.getName());++ SolrZkClient zkClient =+ new SolrZkClient.Builder()+ .withUrl(zkServer.getZkHost())+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)+ .withConnTimeOut(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)+ .build();++ zkClient.makePath("/solr", false, true);+ zkClient.close();++ zkClient =+ new SolrZkClient.Builder()+ .withUrl(zkServer.getZkAddress())+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)+ .build();+ zkClient.create(+ "/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);+ zkClient.makePath(+ "/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);++ zkClient.create(+ SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH,+ "content".getBytes(DATA_ENCODING),+ CreateMode.PERSISTENT,+ false);+ zkClient.close();++ clearSecuritySystemProperties();++ zkClient =+ new SolrZkClient.Builder()+ .withUrl(zkServer.getZkAddress())+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)+ .build();+ // Currently, no credentials on ZK connection, because those same VM-params are used for adding+ // ACLs, and here we want+ // no (or completely open) ACLs added. Therefore, hack your way into being authorized for+ // creating anyway+ zkClient+ .getCuratorFramework()+ .getZookeeperClient()+ .getZooKeeper()+ .addAuthInfo(+ "digest", (ALL_USERNAME + ":" + ALL_PASSWORD).getBytes(StandardCharsets.UTF_8));+ zkClient.create(+ "/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);+ zkClient.makePath(+ "/unprotectedMakePathNode",+ "content".getBytes(DATA_ENCODING),+ CreateMode.PERSISTENT,+ false);+ zkClient.close();++ setDigestZkSystemProps();+ if (log.isInfoEnabled()) {+ log.info("####SETUP_END {}", getTestName());+ }+ }++ private void setDigestZkSystemProps() {+ System.setProperty(+ SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME,+ DigestZkCredentialsProvider.class.getName());+ System.setProperty(+ SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName());+ }++ @Override+ public void tearDown() throws Exception {+ zkServer.shutdown();++ clearSecuritySystemProperties();++ super.tearDown();+ }++ @Test+ public void testNoCredentials() throws Exception {+ List<TestZkCredentialsInjector> testZkCredentialsInjectors =+ List.of(+ new TestZkCredentialsInjector(NoCredentialZkCredentialsInjector.class),+ new TestZkCredentialsInjector(VMParamsZkCredentialsInjector.class));++ testInjectors(+ testZkCredentialsInjectors,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false);+ }++ @Test+ public void testWrongCredentials() throws Exception {+ List<TestZkCredentialsInjector> testZkCredentialsInjectors =+ List.of(+ new TestZkCredentialsInjector(WrongAllCredentialZkCredentialsInjector.class),+ new TestZkCredentialsInjector(+ VMParamsZkCredentialsInjector.class,+ List.of(+ DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),+ List.of(ALL_USERNAME, ALL_PASSWORD + "Wrong")));++ testInjectors(+ testZkCredentialsInjectors,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false);+ }++ @Test+ public void testAllCredentials() throws Exception {+ List<TestZkCredentialsInjector> testZkCredentialsInjectors =+ List.of(+ new TestZkCredentialsInjector(AllCredentialZkCredentialsInjector.class),+ new TestZkCredentialsInjector(+ VMParamsZkCredentialsInjector.class,+ List.of(+ DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),+ List.of(ALL_USERNAME, ALL_PASSWORD)));++ testInjectors(+ testZkCredentialsInjectors, true, true, true, true, true, true, true, true, true, true);+ }++ @Test+ public void testReadonlyCredentials() throws Exception {+ List<TestZkCredentialsInjector> testZkCredentialsInjectors =+ List.of(+ new TestZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class),+ new TestZkCredentialsInjector(+ VMParamsZkCredentialsInjector.class,+ List.of(+ DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),+ List.of(READONLY_USERNAME, READONLY_PASSWORD)));+ testInjectors(+ testZkCredentialsInjectors,+ true,+ true,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false);+ }++ protected void testInjectors(+ List<TestZkCredentialsInjector> testZkCredentialsInjectors,+ boolean getData,+ boolean list,+ boolean create,+ boolean setData,+ boolean delete,+ boolean secureGet,+ boolean secureList,+ boolean secureCreate,+ boolean secureSet,+ boolean secureDelete)+ throws Exception {+ for (TestZkCredentialsInjector testZkCredentialsInjector : testZkCredentialsInjectors) {+ tearDown();+ setUp();+ testZkCredentialsInjector.setSystemProps();+ try (SolrZkClient zkClient =+ new SolrZkClient.Builder()+ .withUrl(zkServer.getZkAddress())+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)+ .build()) {+ doTest(+ zkClient,+ getData,+ list,+ create,+ setData,+ delete,+ secureGet,+ secureList,+ secureCreate,+ secureSet,+ secureDelete);+ }+ }+ }++ @Test+ public void testRepairACL() throws Exception {+ clearSecuritySystemProperties();+ try (SolrZkClient zkClient =+ new SolrZkClient.Builder()+ .withUrl(zkServer.getZkAddress())+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)+ .build()) {+ // Currently, no credentials on ZK connection, because those same VM-params are used for+ // adding ACLs, and here we want+ // no (or completely open) ACLs added. Therefore, hack your way into being authorized for+ // creating anyway+ zkClient+ .getCuratorFramework()+ .getZookeeperClient()+ .getZooKeeper()+ .addAuthInfo(+ "digest",+ ("connectAndAllACLUsername:connectAndAllACLPassword")+ .getBytes(StandardCharsets.UTF_8));++ zkClient.create(+ "/security.json", "{}".getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, false);+ assertEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false));+ }++ setSecuritySystemProperties();+ try (SolrZkClient zkClient =+ new SolrZkClient.Builder()+ .withUrl(zkServer.getZkAddress())+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)+ .build()) {+ ZkController.createClusterZkNodes(zkClient);+ assertNotEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false));+ }++ useZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class);+ // useReadonlyCredentials();+ try (SolrZkClient zkClient =+ new SolrZkClient.Builder()+ .withUrl(zkServer.getZkAddress())+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)+ .build()) {+ NoAuthException e =+ assertThrows(+ NoAuthException.class, () -> zkClient.getData("/security.json", null, null, false));+ assertEquals("/solr/security.json", e.getPath());+ }+ }++ private void useZkCredentialsInjector(Class<?> zkCredentialsInjectorClass) {+ clearSecuritySystemProperties();+ setDigestZkSystemProps();+ System.setProperty(+ SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,+ zkCredentialsInjectorClass.getName());+ }++ private void setSecuritySystemProperties() {+ System.setProperty(+ SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME,+ DigestZkCredentialsProvider.class.getName());+ System.setProperty(+ SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName());+ System.setProperty(+ SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,+ AllAndReadonlyCredentialZkCredentialsInjector.class.getName());+ }++ private void clearSecuritySystemProperties() {+ System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME);+ System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME);+ System.clearProperty(SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME);+ }++ public static void doTest(+ SolrZkClient zkClient,+ boolean getData,+ boolean list,+ boolean create,+ boolean setData,+ boolean delete,+ boolean secureGet,+ boolean secureList,+ boolean secureCreate,+ boolean secureSet,+ boolean secureDelete)+ throws Exception {+ doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete);+ doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete);+ doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete);+ doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete);+ doTest(+ zkClient,+ SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH,+ secureGet,+ secureList,+ secureCreate,+ secureSet,+ secureDelete);+ }++ protected static void doTest(+ SolrZkClient zkClient,+ String path,+ boolean getData,+ boolean list,+ boolean create,+ boolean setData,+ boolean delete)+ throws Exception {+ doTest(getData, () -> zkClient.getData(path, null, null, false));+ doTest(list, () -> zkClient.getChildren(path, null, false));++ doTest(+ create,+ () -> {+ zkClient.create(path + "/subnode", null, CreateMode.PERSISTENT, false);+ zkClient.delete(path + "/subnode", -1, false);+ });+ doTest(+ create,+ () -> {+ zkClient.makePath(path + "/subnode/subsubnode", false);+ zkClient.delete(path + "/subnode/subsubnode", -1, false);+ zkClient.delete(path + "/subnode", -1, false);+ });++ doTest(setData, () -> zkClient.setData(path, (byte[]) null, false));++ // Actually about the ACLs on /solr, but that is protected+ doTest(delete, () -> zkClient.delete(path, -1, false));+ }++ interface ExceptingRunnable {+ void run() throws Exception;+ }++ private static void doTest(boolean shouldSucceed, ExceptingRunnable action) throws Exception {+ if (shouldSucceed) {+ action.run();+ } else {+ expectThrows(NoAuthException.class, action::run);+ }+ }++ @Test+ public void testVMParamsAllCredentialsFromFile() throws Exception {+ useVMParamsAllCredentialsFromFile();++ try (SolrZkClient zkClient =+ new SolrZkClient.Builder()+ .withUrl(zkServer.getZkAddress())+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)+ .build()) {+ doTest(zkClient, true, true, true, true, true, true, true, true, true, true);+ }+ }++ @Test+ public void testVMParamsReadonlyCredentialsFromFile() throws Exception {+ useVMParamsReadonlyCredentialsFromFile();++ try (SolrZkClient zkClient =+ new SolrZkClient.Builder()+ .withUrl(zkServer.getZkAddress())+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)+ .build()) {+ doTest(zkClient, true, true, false, false, false, false, false, false, false, false);+ }+ }++ private void useVMParamsAllCredentialsFromFile() throws IOException {+ useVMParamsCredentialsFromFile("connectAndAllACLUsername", "connectAndAllACLPassword");+ }++ private void useVMParamsReadonlyCredentialsFromFile() throws IOException {+ useVMParamsCredentialsFromFile("readonlyACLUsername", "readonlyACLPassword");+ }++ private void useVMParamsCredentialsFromFile(String username, String password) throws IOException {+ Properties props = new Properties();+ props.setProperty(+ VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, username);+ props.setProperty(+ VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, password);+ String credsFile = saveCredentialsFile(props);++ useZkCredentialsInjector(VMParamsZkCredentialsInjector.class);+ System.setProperty(DEFAULT_DIGEST_FILE_VM_PARAM_NAME, credsFile);+ }++ private String saveCredentialsFile(Properties props) throws IOException {+ Path tmp = createTempFile("zk-creds", "properties");+ try (FileWriter w = new FileWriter(tmp.toFile(), StandardCharsets.UTF_8)) {+ props.store(w, "test");+ }+ return tmp.toAbsolutePath().toString();+ }++ public static class NoCredentialZkCredentialsInjector implements ZkCredentialsInjector {+ @Override+ public List<ZkCredential> getZkCredentials() {+ return Collections.emptyList();+ }+ }++ public static class AllAndReadonlyCredentialZkCredentialsInjector+ implements ZkCredentialsInjector {+ @Override+ public List<ZkCredential> getZkCredentials() {+ return List.of(+ new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL),+ new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.READ));+ }+ }++ public static class ConnectWithReadonlyCredsInjector implements ZkCredentialsInjector {+ @Override+ public List<ZkCredential> getZkCredentials() {+ return List.of(+ new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.ALL));+ }+ }++ public static class AllCredentialZkCredentialsInjector implements ZkCredentialsInjector {+ @Override+ public List<ZkCredential> getZkCredentials() {+ return List.of(new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL));+ }+ }++ public static class WrongAllCredentialZkCredentialsInjector implements ZkCredentialsInjector {+ @Override+ public List<ZkCredential> getZkCredentials() {+ return List.of(+ new ZkCredential(ALL_USERNAME, ALL_PASSWORD + "Wrong", ZkCredential.Perms.ALL));+ }+ }++ class TestZkCredentialsInjector {+ private final Class<?> zkCredentialsInjectorClass;+ private final List<String> systemPropsKeys;+ private final List<String> systemPropsValues;++ public TestZkCredentialsInjector(Class<?> zkCredentialsInjectorClass) {+ this(zkCredentialsInjectorClass, Collections.emptyList(), Collections.emptyList());+ }++ public TestZkCredentialsInjector(+ Class<?> zkCredentialsInjectorClass,+ List<String> systemPropsKeys,+ List<String> systemPropsValues) {+ this.zkCredentialsInjectorClass = zkCredentialsInjectorClass;+ this.systemPropsKeys = systemPropsKeys;+ this.systemPropsValues = systemPropsValues;+ }++ private void setSystemProps() {+ clearSecuritySystemProperties();+ setDigestZkSystemProps();+ System.setProperty(+ SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,+ zkCredentialsInjectorClass.getName());+ int i = 0;+ for (String key : systemPropsKeys) {+ System.setProperty(key, systemPropsValues.get(i++));+ }+ }+ }+}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.javadeleted file mode 100644index e758dbf26a7..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java+++ /dev/null@@ -1,487 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import static org.apache.solr.cloud.SolrCloudTestCase.configurePrsDefault;--import java.nio.file.Files;-import java.nio.file.Path;-import java.util.concurrent.TimeUnit;-import org.apache.lucene.tests.mockfile.FilterPath;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.request.QueryRequest;-import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.client.solrj.request.UpdateRequest;-import org.apache.solr.client.solrj.response.QueryResponse;-import org.apache.solr.common.SolrException;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.cloud.ZkNodeProps;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.embedded.JettySolrRunner;-import org.apache.solr.handler.BackupStatusChecker;-import org.apache.solr.handler.ReplicationHandler;-import org.junit.BeforeClass;-import org.junit.Test;--/**- * This test simply does a bunch of basic things in solrcloud mode and asserts things work as- * expected.- */-// Backups do checksum validation against a footer value not present in 'SimpleText'[email protected]({"SimpleText"})-public abstract class AbstractBasicDistributedZk2TestBase extends AbstractFullDistribZkTestBase {- private static final String SHARD2 = "shard2";- private static final String SHARD1 = "shard1";- private static final String ONE_NODE_COLLECTION = "onenodecollection";- private final boolean onlyLeaderIndexes = random().nextBoolean();-- public AbstractBasicDistributedZk2TestBase() {- super();- // we need DVs on point fields to compute stats & facets- if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP))- System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");-- sliceCount = 2;- }-- @Override- protected boolean useTlogReplicas() {- return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug- // and it's TestInjection use- }-- @BeforeClass- public static void _setPrsDefault() {- configurePrsDefault();- }-- @Test- @ShardsFixed(num = 4)- public void test() throws Exception {- boolean testFinished = false;- try {- handle.clear();- handle.put("timestamp", SKIPVAL);-- testNodeWithoutCollectionForwarding();-- indexr(- id,- 1,- i1,- 100,- tlong,- 100,- t1,- "now is the time for all good men",- "foo_f",- 1.414f,- "foo_b",- "true",- "foo_d",- 1.414d);-- commit();-- // make sure we are in a steady state...- waitForRecoveriesToFinish(false);-- assertDocCounts(false);-- indexAbunchOfDocs();-- // check again- waitForRecoveriesToFinish(false);-- commit();-- assertDocCounts(VERBOSE);- checkQueries();-- assertDocCounts(VERBOSE);-- query("q", "*:*", "sort", "n_tl1 desc");-- bringDownShardIndexSomeDocsAndRecover();-- query("q", "*:*", "sort", "n_tl1 desc");-- // test adding another replica to a shard - it should do a- // recovery/replication to pick up the index from the leader- addNewReplica();-- long docId = testUpdateAndDelete();-- // index a bad doc...- expectThrows(SolrException.class, () -> indexr(t1, "a doc with no id"));-- // TODO: bring this to its own method?- // try indexing to a leader that has no replicas up- ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);- ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, SHARD2);-- String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP);- chaosMonkey.stopShardExcept(SHARD2, nodeName);-- SolrClient client = getClient(nodeName);-- index_specific(client, "id", docId + 1, t1, "what happens here?");-- // expire a session...- CloudJettyRunner cloudJetty = shardToJetty.get(SHARD1).get(0);- chaosMonkey.expireSession(cloudJetty.jetty);- // Wait until the jetty is reconnected, otherwise the following index command could fail- cloudJetty- .jetty- .getCoreContainer()- .getZkController()- .getZkClient()- .getCuratorFramework()- .blockUntilConnected(50, TimeUnit.MILLISECONDS);-- indexr("id", docId + 1, t1, "slip this doc in");-- waitForRecoveriesToFinish(false);-- checkShardConsistency(SHARD1);- checkShardConsistency(SHARD2);-- testFinished = true;- } finally {- if (!testFinished) {- printLayoutOnTearDown = true;- }- }- }-- private void testNodeWithoutCollectionForwarding() throws Exception {- assertEquals(- 0,- CollectionAdminRequest.createCollection(ONE_NODE_COLLECTION, "conf1", 1, 1)- .setCreateNodeSet("")- .process(cloudClient)- .getStatus());- assertTrue(- CollectionAdminRequest.addReplicaToShard(ONE_NODE_COLLECTION, "shard1")- .setCoreName(ONE_NODE_COLLECTION + "core")- .process(cloudClient)- .isSuccess());-- waitForCollection(ZkStateReader.from(cloudClient), ONE_NODE_COLLECTION, 1);- waitForRecoveriesToFinish(ONE_NODE_COLLECTION, ZkStateReader.from(cloudClient), false);-- ZkStateReader.from(cloudClient).getLeaderRetry(ONE_NODE_COLLECTION, SHARD1, 30000);-- int docs = 2;- for (JettySolrRunner jetty : jettys) {- final String clientUrl = getBaseUrl(jetty);- addAndQueryDocs(clientUrl, docs);- docs += 2;- }- }-- // 2 docs added every call- private void addAndQueryDocs(final String baseUrl, int docs) throws Exception {-- SolrQuery query = new SolrQuery("*:*");-- try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollection")) {- // add a doc- client.add(sdoc("id", docs));- client.commit();-- QueryResponse results = client.query(query);- assertEquals(docs - 1, results.getResults().getNumFound());-- SolrInputDocument doc = new SolrInputDocument();- doc.addField("id", docs + 1);- client.add(doc);- client.commit();-- query = new SolrQuery("*:*");- query.set("rows", 0);- results = client.query(query);- assertEquals(docs, results.getResults().getNumFound());- }- }-- private long testUpdateAndDelete() throws Exception {- long docId = 99999999L;- indexr("id", docId, t1, "originalcontent");-- commit();-- ModifiableSolrParams params = new ModifiableSolrParams();- params.add("q", t1 + ":originalcontent");- QueryResponse results = clients.get(0).query(params);- assertEquals(1, results.getResults().getNumFound());-- // update doc- indexr("id", docId, t1, "updatedcontent");-- commit();-- results = clients.get(0).query(params);- assertEquals(0, results.getResults().getNumFound());-- params.set("q", t1 + ":updatedcontent");-- results = clients.get(0).query(params);- assertEquals(1, results.getResults().getNumFound());-- UpdateRequest uReq = new UpdateRequest();- // uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN);- uReq.deleteById(Long.toString(docId)).process(clients.get(0));-- commit();-- results = clients.get(0).query(params);- assertEquals(0, results.getResults().getNumFound());- return docId;- }-- private void bringDownShardIndexSomeDocsAndRecover() throws Exception {- SolrQuery query = new SolrQuery("*:*");- query.set("distrib", false);-- commit();-- long deadShardCount =- shardToJetty.get(SHARD2).get(0).client.solrClient.query(query).getResults().getNumFound();-- query("q", "*:*", "sort", "n_tl1 desc");-- int oldLiveNodes =- ZkStateReader.from(cloudClient)- .getZkClient()- .getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true)- .size();-- assertEquals(5, oldLiveNodes);-- // kill a shard- CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0);-- // ensure shard is dead- expectThrows(- SolrServerException.class,- "This server should be down and this update should have failed",- () -> index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1, "specific doc!"));-- commit();-- query("q", "*:*", "sort", "n_tl1 desc");-- // long cloudClientDocs = cloudClient.query(new- // SolrQuery("*:*")).getResults().getNumFound();- // System.out.println("clouddocs:" + cloudClientDocs);-- // try to index to a living shard at shard2-- long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();-- ZkStateReader.from(cloudClient).getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 60000);-- try {- index_specific(- shardToJetty.get(SHARD1).get(1).client.solrClient,- id,- 1000,- i1,- 108,- t1,- "specific doc!");- } catch (Exception e) {- // wait and try again- Thread.sleep(4000);- index_specific(- shardToJetty.get(SHARD1).get(1).client.solrClient,- id,- 1000,- i1,- 108,- t1,- "specific doc!");- }-- commit();-- checkShardConsistency(true, false);-- query("q", "*:*", "sort", "n_tl1 desc");-- long numFound2 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();-- assertEquals(numFound1 + 1, numFound2);-- SolrInputDocument doc = new SolrInputDocument();- doc.addField("id", 1001);-- controlClient.add(doc);-- // try adding a doc with CloudSolrServer- UpdateRequest ureq = new UpdateRequest();- ureq.add(doc);- // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);-- try {- ureq.process(cloudClient);- } catch (SolrServerException e) {- // try again- Thread.sleep(3500);- ureq.process(cloudClient);- }-- commit();-- query("q", "*:*", "sort", "n_tl1 desc");-- long numFound3 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();-- // lets just check that the one doc since last commit made it in...- assertEquals(numFound2 + 1, numFound3);-- // test debugging- testDebugQueries();-- if (VERBOSE) {- System.err.println(controlClient.query(new SolrQuery("*:*")).getResults().getNumFound());-- for (SolrClient client : clients) {- try {- SolrQuery q = new SolrQuery("*:*");- q.set("distrib", false);- System.err.println(client.query(q).getResults().getNumFound());- } catch (Exception e) {-- }- }- }- // TODO: This test currently fails because debug info is obtained only- // on shards with matches.- // query("q","matchesnothing","fl","*,score", "debugQuery", "true");-- // this should trigger a recovery phase on deadShard- deadShard.jetty.start();-- // make sure we have published we are recovering- Thread.sleep(1500);-- waitForRecoveriesToFinish(false);-- deadShardCount =- shardToJetty.get(SHARD1).get(0).client.solrClient.query(query).getResults().getNumFound();- // if we properly recovered, we should now have the couple missing docs that- // came in while shard was down- checkShardConsistency(true, false);-- // recover over 100 docs so we do more than just peer sync (replicate recovery)- chaosMonkey.stopJetty(deadShard);-- for (int i = 0; i < 226; i++) {- doc = new SolrInputDocument();- doc.addField("id", 2000 + i);- controlClient.add(doc);- ureq = new UpdateRequest();- ureq.add(doc);- // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);- ureq.process(cloudClient);- }- commit();-- Thread.sleep(1500);-- deadShard.jetty.start();-- // make sure we have published we are recovering- Thread.sleep(1500);-- waitForThingsToLevelOut(1, TimeUnit.MINUTES);-- Thread.sleep(500);-- waitForRecoveriesToFinish(false);-- checkShardConsistency(true, false);-- // try a backup command- try (final SolrClient client =- getHttpSolrClient((String) shardToJetty.get(SHARD2).get(0).info.get("base_url"))) {- final String backupName = "the_backup";- ModifiableSolrParams params = new ModifiableSolrParams();- params.set("qt", ReplicationHandler.PATH);- params.set("command", "backup");- params.set("name", backupName);- final Path location = FilterPath.unwrap(createTempDir()).toRealPath();- // Allow non-standard location outside SOLR_HOME- jettys.forEach(j -> j.getCoreContainer().getAllowPaths().add(location));- params.set("location", location.toString());-- QueryRequest request = new QueryRequest(params);- client.request(request, DEFAULT_TEST_COLLECTION_NAME);-- final BackupStatusChecker backupStatus =- new BackupStatusChecker(client, "/" + DEFAULT_TEST_COLLECTION_NAME + "/replication");- final String backupDirName = backupStatus.waitForBackupSuccess(backupName, 30);- assertTrue(- "Backup dir does not exist: " + backupDirName,- Files.exists(location.resolve(backupDirName)));- }- }-- private void addNewReplica() throws Exception {-- waitForRecoveriesToFinish(false);-- // new server should be part of first shard- // how many docs are on the new shard?- for (CloudJettyRunner cjetty : shardToJetty.get(SHARD1)) {- if (VERBOSE)- System.err.println(- "shard1 total:"- + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());- }- for (CloudJettyRunner cjetty : shardToJetty.get(SHARD2)) {- if (VERBOSE)- System.err.println(- "shard2 total:"- + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());- }-- checkShardConsistency(SHARD1);- checkShardConsistency(SHARD2);-- assertDocCounts(VERBOSE);- }-- private void testDebugQueries() throws Exception {- handle.put("explain", SKIPVAL);- handle.put("debug", UNORDERED);- handle.put("time", SKIPVAL);- handle.put("track", SKIP);- query("q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true");- query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true");- query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING);- query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS);- query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY);- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.javadeleted file mode 100644index 24d0689c5a6..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java+++ /dev/null@@ -1,1737 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import java.io.IOException;-import java.io.InputStream;-import java.lang.invoke.MethodHandles;-import java.nio.charset.StandardCharsets;-import java.util.ArrayList;-import java.util.Collections;-import java.util.HashMap;-import java.util.HashSet;-import java.util.List;-import java.util.Map;-import java.util.Set;-import java.util.concurrent.Callable;-import java.util.concurrent.CompletionService;-import java.util.concurrent.CountDownLatch;-import java.util.concurrent.ExecutorCompletionService;-import java.util.concurrent.Future;-import java.util.concurrent.SynchronousQueue;-import java.util.concurrent.ThreadPoolExecutor;-import java.util.concurrent.TimeUnit;-import java.util.concurrent.TimeoutException;-import java.util.concurrent.atomic.AtomicInteger;-import java.util.concurrent.atomic.AtomicLong;-import java.util.concurrent.atomic.AtomicReference;-import org.apache.lucene.util.IOUtils;-import org.apache.solr.JSONTestUtil;-import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.SolrRequest;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.apache.HttpSolrClient;-import org.apache.solr.client.solrj.request.AbstractUpdateRequest;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;-import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;-import org.apache.solr.client.solrj.request.GenericSolrRequest;-import org.apache.solr.client.solrj.request.QueryRequest;-import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.client.solrj.request.StreamingUpdateRequest;-import org.apache.solr.client.solrj.request.UpdateRequest;-import org.apache.solr.client.solrj.response.CollectionAdminResponse;-import org.apache.solr.client.solrj.response.FacetField;-import org.apache.solr.client.solrj.response.Group;-import org.apache.solr.client.solrj.response.GroupCommand;-import org.apache.solr.client.solrj.response.GroupResponse;-import org.apache.solr.client.solrj.response.InputStreamResponseParser;-import org.apache.solr.client.solrj.response.QueryResponse;-import org.apache.solr.client.solrj.response.UpdateResponse;-import org.apache.solr.cloud.api.collections.CollectionHandlingUtils;-import org.apache.solr.common.SolrDocument;-import org.apache.solr.common.SolrDocumentList;-import org.apache.solr.common.SolrException;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.cloud.ClusterState;-import org.apache.solr.common.cloud.DocCollection;-import org.apache.solr.common.cloud.Replica;-import org.apache.solr.common.cloud.Slice;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.params.CollectionParams.CollectionAction;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.params.SolrParams;-import org.apache.solr.common.params.UpdateParams;-import org.apache.solr.common.util.ExecutorUtil;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.common.util.SolrNamedThreadFactory;-import org.apache.solr.embedded.JettySolrRunner;-import org.apache.solr.util.TestInjection;-import org.apache.solr.util.TestInjection.Hook;-import org.junit.BeforeClass;-import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--/**- * This test simply does a bunch of basic things in solrcloud mode and asserts things work as- * expected.- */-public abstract class AbstractBasicDistributedZkTestBase extends AbstractFullDistribZkTestBase {-- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- private static final String DEFAULT_COLLECTION = "collection1";-- private final boolean onlyLeaderIndexes = random().nextBoolean();-- String tsort = "t_sortable";-- private Map<String, List<SolrClient>> otherCollectionClients = new HashMap<>();-- private String oneInstanceCollection = "oneInstanceCollection";- private String oneInstanceCollection2 = "oneInstanceCollection2";-- private AtomicInteger nodeCounter = new AtomicInteger();-- CompletionService<Object> completionService;- Set<Future<Object>> pending;-- private static Hook newSearcherHook =- new Hook() {- volatile CountDownLatch latch;- AtomicReference<String> collection = new AtomicReference<>();-- @Override- public void newSearcher(String collectionName) {- String c = collection.get();- if (c != null && c.equals(collectionName)) {- log.info("Hook detected newSearcher");- try {- latch.countDown();- } catch (NullPointerException e) {-- }- }- }-- @Override- public void waitForSearcher(- String collection, int cnt, int timeoutms, boolean failOnTimeout)- throws InterruptedException {- latch = new CountDownLatch(cnt);- this.collection.set(collection);- boolean timeout = !latch.await(timeoutms, TimeUnit.MILLISECONDS);- if (timeout && failOnTimeout) {- fail("timed out waiting for new searcher event " + latch.getCount());- }- }- };-- public AbstractBasicDistributedZkTestBase() {- // we need DVs on point fields to compute stats & facets- if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP))- System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");-- sliceCount = 2;- completionService = new ExecutorCompletionService<>(executor);- pending = new HashSet<>();- }-- @BeforeClass- public static void beforeBDZKTClass() {- TestInjection.newSearcherHook(newSearcherHook);- }-- @Override- protected boolean useTlogReplicas() {- // TODO: tlog replicas makes commits take way to long due to what is likely a bug and it's- // TestInjection use- return false;- }-- @Override- protected void setDistributedParams(ModifiableSolrParams params) {-- if (r.nextBoolean()) {- // don't set shards, let that be figured out from the cloud state- } else {- // use shard ids rather than physical locations- StringBuilder sb = new StringBuilder();- for (int i = 0; i < getShardCount(); i++) {- if (i > 0) sb.append(',');- sb.append("shard" + (i + 3));- }- params.set("shards", sb.toString());- }- }-- @Test- @ShardsFixed(num = 4)- protected void test() throws Exception {- // setLoggingLevel(null);-- ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);- // make sure we have leaders for each shard- for (int j = 1; j < sliceCount; j++) {- zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);- } // make sure we again have leaders for each shard-- waitForRecoveriesToFinish(false);-- handle.clear();- handle.put("timestamp", SKIPVAL);-- del("*:*");- queryAndCompareShards(params("q", "*:*", "distrib", "false", "sanity_check", "is_empty"));-- // ask every individual replica of every shard to update+commit the same doc id- // with an incrementing counter on each update+commit- int foo_i_counter = 0;- for (SolrClient client : clients) {- foo_i_counter++;- indexDoc(- client,- params("commit", "true"), // SOLR-4923- sdoc(id, 1, i1, 100, tlong, 100, "foo_i", foo_i_counter));- // after every update+commit, check all the shards consistency- queryAndCompareShards(- params("q", "id:1", "distrib", "false", "sanity_check", "non_distrib_id_1_lookup"));- queryAndCompareShards(- params(- "q", "id:1",- "sanity_check", "distrib_id_1_lookup"));- }-- indexr(- id,- 1,- i1,- 100,- tlong,- 100,- t1,- "now is the time for all good men",- "foo_f",- 1.414f,- "foo_b",- "true",- "foo_d",- 1.414d,- tsort,- "now is the time for all good men");- indexr(- id,- 2,- i1,- 50,- tlong,- 50,- t1,- "to come to the aid of their country.",- tsort,- "to come to the aid of their country.");- indexr(id, 3, i1, 2, tlong, 2, t1, "how now brown cow", tsort, "how now brown cow");- indexr(- id,- 4,- i1,- -100,- tlong,- 101,- t1,- "the quick fox jumped over the lazy dog",- tsort,- "the quick fox jumped over the lazy dog");- indexr(- id,- 5,- i1,- 500,- tlong,- 500,- t1,- "the quick fox jumped way over the lazy dog",- tsort,- "the quick fox jumped over the lazy dog");- indexr(- id,- 6,- i1,- -600,- tlong,- 600,- t1,- "humpty dumpy sat on a wall",- tsort,- "the quick fox jumped over the lazy dog");- indexr(- id,- 7,- i1,- 123,- tlong,- 123,- t1,- "humpty dumpy had a great fall",- tsort,- "the quick fox jumped over the lazy dog");- indexr(- id,- 8,- i1,- 876,- tlong,- 876,- t1,- "all the kings horses and all the kings men",- tsort,- "all the kings horses and all the kings men");- indexr(- id,- 9,- i1,- 7,- tlong,- 7,- t1,- "couldn't put humpty together again",- tsort,- "the quick fox jumped over the lazy dog");- indexr(id, 10, i1, 4321, tlong, 4321, t1, "this too shall pass", tsort, "this too shall pass");- indexr(- id,- 11,- i1,- -987,- tlong,- 987,- t1,- "An eye for eye only ends up making the whole world blind.",- tsort,- "An eye for eye only ends up making the whole world blind.");- indexr(- id,- 12,- i1,- 379,- tlong,- 379,- t1,- "Great works are performed, not by strength, but by perseverance.",- tsort,- "Great works are performed, not by strength, but by perseverance.");- indexr(- id,- 13,- i1,- 232,- tlong,- 232,- t1,- "no eggs on wall, lesson learned",- oddField,- "odd man out",- tsort,- "no eggs on wall, lesson learned");-- indexr(- id,- 14,- "SubjectTerms_mfacet",- new String[] {"mathematical models", "mathematical analysis"});- indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});- indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});- String[] vals = new String[100];- for (int i = 0; i < 100; i++) {- vals[i] = "test " + i;- }- indexr(id, 17, "SubjectTerms_mfacet", vals);-- for (int i = 100; i < 150; i++) {- indexr(id, i);- }-- commit();-- testTokenizedGrouping();- testSortableTextFaceting();- testSortableTextSorting();- testSortableTextGrouping();-- queryAndCompareShards(- params(- "q", "*:*",- "sort", "id desc",- "distrib", "false",- "sanity_check", "is_empty"));-- // random value sort- for (String f : fieldNames) {- query(false, new String[] {"q", "*:*", "sort", f + " desc"});- query(false, new String[] {"q", "*:*", "sort", f + " asc"});- }-- // these queries should be exactly ordered and scores should exactly match- query(false, new String[] {"q", "*:*", "sort", i1 + " desc"});- query(false, new String[] {"q", "*:*", "sort", i1 + " asc"});- query(false, new String[] {"q", "*:*", "sort", i1 + " desc", "fl", "*,score"});- query(false, new String[] {"q", "*:*", "sort", "n_tl1 asc", "fl", "*,score"});- query(false, new String[] {"q", "*:*", "sort", "n_tl1 desc"});- handle.put("maxScore", SKIPVAL);- query(- false,- new String[] {"q", "{!func}" + i1}); // does not expect maxScore. So if it comes ,ignore it.- // JavaBinCodec.writeSolrDocumentList()- // is agnostic of request params.- handle.remove("maxScore");- query(- false,- new String[] {- "q", "{!func}" + i1, "fl", "*,score"- }); // even scores should match exactly here-- handle.put("highlighting", UNORDERED);- handle.put("response", UNORDERED);-- handle.put("maxScore", SKIPVAL);- query(false, new String[] {"q", "quick"});- query(false, new String[] {"q", "all", "fl", "id", "start", "0"});- query(- false,- new String[] {"q", "all", "fl", "foofoofoo", "start", "0"}); // no fields in returned docs- query(false, new String[] {"q", "all", "fl", "id", "start", "100"});-- handle.put("score", SKIPVAL);- query(false, new String[] {"q", "quick", "fl", "*,score"});- query(false, new String[] {"q", "all", "fl", "*,score", "start", "1"});- query(false, new String[] {"q", "all", "fl", "*,score", "start", "100"});-- query(- false,- new String[] {- "q", "now their fox sat had put", "fl", "*,score", "hl", "true", "hl.fl", t1- });-- query(- false,- new String[] {- "q", "now their fox sat had put", "fl", "foofoofoo", "hl", "true", "hl.fl", t1- });-- query(false, new String[] {"q", "matchesnothing", "fl", "*,score"});-- query(false, new Object[] {"q", "*:*", "rows", 100, "facet", "true", "facet.field", t1});- query(- false,- new Object[] {- "q",- "*:*",- "rows",- 100,- "facet",- "true",- "facet.field",- t1,- "facet.limit",- -1,- "facet.sort",- "count"- });- query(- false,- new Object[] {- "q",- "*:*",- "rows",- 100,- "facet",- "true",- "facet.field",- t1,- "facet.limit",- -1,- "facet.sort",- "count",- "facet.mincount",- 2- });- query(- false,- new Object[] {- "q",- "*:*",- "rows",- 100,- "facet",- "true",- "facet.field",- t1,- "facet.limit",- -1,- "facet.sort",- "index"- });- query(- false,- new Object[] {- "q",- "*:*",- "rows",- 100,- "facet",- "true",- "facet.field",- t1,- "facet.limit",- -1,- "facet.sort",- "index",- "facet.mincount",- 2- });- query(- false,- new Object[] {- "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.limit", 1- });- query(- false,- new Object[] {- "q",- "*:*",- "rows",- 100,- "facet",- "true",- "facet.query",- "quick",- "facet.query",- "all",- "facet.query",- "*:*"- });- query(- false,- new Object[] {- "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.offset", 1- });- query(- false,- new Object[] {- "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.mincount", 2- });-- // test faceting multiple things at once- query(- false,- new Object[] {- "q",- "*:*",- "rows",- 100,- "facet",- "true",- "facet.query",- "quick",- "facet.query",- "all",- "facet.query",- "*:*",- "facet.field",- t1- });-- // test filter tagging, facet exclusion, and naming (multi-select facet support)- query(- false,- new Object[] {- "q",- "*:*",- "rows",- 100,- "facet",- "true",- "facet.query",- "{!key=myquick}quick",- "facet.query",- "{!key=myall ex=a}all",- "facet.query",- "*:*",- "facet.field",- "{!key=mykey ex=a}" + t1,- "facet.field",- "{!key=other ex=b}" + t1,- "facet.field",- "{!key=again ex=a,b}" + t1,- "facet.field",- t1,- "fq",- "{!tag=a}id_i1:[1 TO 7]",- "fq",- "{!tag=b}id_i1:[3 TO 9]"- });- query(- false,- new Object[] {- "q",- "*:*",- "facet",- "true",- "facet.field",- "{!ex=t1}SubjectTerms_mfacet",- "fq",- "{!tag=t1}SubjectTerms_mfacet:(test 1)",- "facet.limit",- "10",- "facet.mincount",- "1"- });-- // test field that is valid in schema but missing in all shards- query(- false,- new Object[] {- "q", "*:*", "rows", 100, "facet", "true", "facet.field", missingField, "facet.mincount", 2- });- // test field that is valid in schema and missing in some shards- query(- false,- new Object[] {- "q", "*:*", "rows", 100, "facet", "true", "facet.field", oddField, "facet.mincount", 2- });-- query(- false, new Object[] {"q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", i1});-- /* TODO: the failure may come back in "exception"- try {- // test error produced for field that is invalid for schema- query("q","*:*", "rows",100, "facet","true", "facet.field",invalidField, "facet.mincount",2);- TestCase.fail("SolrServerException expected for invalid field that is not in schema");- } catch (SolrServerException ex) {- // expected- }- */-- // Try to get better coverage for refinement queries by turning off over requesting.- // This makes it much more likely that we may not get the top facet values and hence- // we turn of that checking.- handle.put("facet_fields", SKIPVAL);- query(- false,- new Object[] {- "q",- "*:*",- "rows",- 0,- "facet",- "true",- "facet.field",- t1,- "facet.limit",- 5,- "facet.shard.limit",- 5- });- // check a complex key name- query(- false,- new Object[] {- "q",- "*:*",- "rows",- 0,- "facet",- "true",- "facet.field",- "{!key='a b/c \\' \\} foo'}" + t1,- "facet.limit",- 5,- "facet.shard.limit",- 5- });- handle.remove("facet_fields");-- // index the same document to two servers and make sure things- // don't blow up.- if (clients.size() >= 2) {- index(id, 100, i1, 107, t1, "oh no, a duplicate!");- for (int i = 0; i < clients.size(); i++) {- index_specific(i, id, 100, i1, 107, t1, "oh no, a duplicate!");- }- commit();- query(false, new Object[] {"q", "duplicate", "hl", "true", "hl.fl", t1});- query(false, new Object[] {"q", "fox duplicate horses", "hl", "true", "hl.fl", t1});- query(false, new Object[] {"q", "*:*", "rows", 100});- }-- // test debugging- handle.put("explain", SKIPVAL);- handle.put("debug", UNORDERED);- handle.put("time", SKIPVAL);- handle.put("track", SKIP);- query(- false,- new Object[] {- "q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true"- });- query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"});- query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING});- query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS});- query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY});-- // try add commitWithin- long before = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();- for (SolrClient client : clients) {- assertEquals(- "unexpected pre-commitWithin document count on node: "- + ((HttpSolrClient) client).getBaseURL(),- before,- client.query(new SolrQuery("*:*")).getResults().getNumFound());- }-- ModifiableSolrParams params = new ModifiableSolrParams();- params.set("commitWithin", 10);- add(cloudClient, params, List.of(getDoc("id", 300), getDoc("id", 301)));-- newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);-- ClusterState clusterState = getCommonCloudSolrClient().getClusterState();- DocCollection dColl = clusterState.getCollection(DEFAULT_COLLECTION);-- assertSliceCounts("should have found 2 docs, 300 and 301", before + 2, dColl);-- // try deleteById commitWithin- UpdateRequest deleteByIdReq = new UpdateRequest();- deleteByIdReq.deleteById("300");- deleteByIdReq.setCommitWithin(10);- deleteByIdReq.process(cloudClient);-- newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);-- assertSliceCounts("deleteById commitWithin did not work", before + 1, dColl);-- // try deleteByQuery commitWithin- UpdateRequest deleteByQueryReq = new UpdateRequest();- deleteByQueryReq.deleteByQuery("id:301");- deleteByQueryReq.setCommitWithin(10);- deleteByQueryReq.process(cloudClient);-- newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);-- assertSliceCounts("deleteByQuery commitWithin did not work", before, dColl);-- // TODO: This test currently fails because debug info is obtained only- // on shards with matches.- // query("q","matchesnothing","fl","*,score", "debugQuery", "true");-- // would be better if these where all separate tests - but much, much- // slower- doOptimisticLockingAndUpdating();- testShardParamVariations();- testMultipleCollections();- testANewCollectionInOneInstance();- testSearchByCollectionName();- testUpdateByCollectionName();- testANewCollectionInOneInstanceWithManualShardAssignement();- testNumberOfCommitsWithCommitAfterAdd();-- testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit");- testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit");-- testStopAndStartCoresInOneInstance();- }-- private void testSortableTextFaceting() throws Exception {- SolrQuery query = new SolrQuery("*:*");- query.addFacetField(tsort);- query.setFacetMissing(false);- QueryResponse resp = queryRandomShard(query);- List<FacetField> ffs = resp.getFacetFields();- for (FacetField ff : ffs) {- if (ff.getName().equals(tsort) == false) continue;- for (FacetField.Count count : ff.getValues()) {- long num = count.getCount();- switch (count.getName()) {- case "all the kings horses and all the kings men":- case "An eye for eye only ends up making the whole world blind.":- case "Great works are performed, not by strength, but by perseverance.":- case "how now brown cow":- case "no eggs on wall, lesson learned":- case "now is the time for all good men":- case "this too shall pass":- case "to come to the aid of their country.":- assertEquals("Should have exactly one facet count for field " + ff.getName(), 1, num);- break;- case "the quick fox jumped over the lazy dog":- assertEquals("Should have 5 docs for the lazy dog", 5, num);- break;- default:- fail("No case for facet '" + ff.getName() + "'");- }- }- }- }-- private void testSortableTextSorting() throws Exception {- SolrQuery query = new SolrQuery("*:*");- query.addSort(tsort, SolrQuery.ORDER.desc);- query.addField("*");- query.addField("eoe_sortable");- query.addField(tsort);- QueryResponse resp = queryRandomShard(query);-- SolrDocumentList docs = resp.getResults();-- String title = docs.get(0).getFieldValue(tsort).toString();- for (SolrDocument doc : docs) {- assertTrue(- "Docs should be back in sorted order, descending",- title.compareTo(doc.getFieldValue(tsort).toString()) >= 0);- title = doc.getFieldValue(tsort).toString();- }- }-- private void testSortableTextGrouping() throws Exception {- SolrQuery query = new SolrQuery("*:*");- query.add("group", "true");- query.add("group.field", tsort);- QueryResponse resp = queryRandomShard(query);- GroupResponse groupResp = resp.getGroupResponse();- List<GroupCommand> grpCmds = groupResp.getValues();- for (GroupCommand grpCmd : grpCmds) {- if (grpCmd.getName().equals(tsort) == false) continue;- for (Group grp : grpCmd.getValues()) {- long count = grp.getResult().getNumFound();- if (grp.getGroupValue() == null)- continue; // Don't count the groups without an entry as the numnber is variable- switch (grp.getGroupValue()) {- case "all the kings horses and all the kings men":- case "An eye for eye only ends up making the whole world blind.":- case "Great works are performed, not by strength, but by perseverance.":- case "how now brown cow":- case "no eggs on wall, lesson learned":- case "now is the time for all good men":- case "this too shall pass":- case "to come to the aid of their country.":- assertEquals(- "Should have exactly one facet count for field " + grpCmd.getName(), 1, count);- break;- case "the quick fox jumped over the lazy dog":- assertEquals("Should have 5 docs for the lazy dog", 5, count);- break;- default:- fail("No case for facet '" + grpCmd.getName() + "'");- }- }- }- }-- private void testTokenizedGrouping() throws Exception {- SolrException ex =- expectThrows(- SolrException.class,- () -> {- query(false, new String[] {"q", "*:*", "group", "true", "group.field", t1});- });- assertTrue(- "Expected error from server that SortableTextFields are required",- ex.getMessage()- .contains(- "Sorting on a tokenized field that is not a SortableTextField is not supported in cloud mode"));- }-- private void assertSliceCounts(String msg, long expected, DocCollection dColl) throws Exception {- long found = checkSlicesSameCounts(dColl);-- if (found != expected) {- // we get one do over in a bad race- Thread.sleep(1000);- found = checkSlicesSameCounts(dColl);- }-- assertEquals(msg, expected, checkSlicesSameCounts(dColl));- }-- // Ensure that total docs found is the expected number.- private void waitForDocCount(long expectedNumFound, long waitMillis, String failureMessage)- throws Exception {- AtomicLong total = new AtomicLong(-1);- try {- ZkStateReader.from(getCommonCloudSolrClient())- .waitForState(- DEFAULT_COLLECTION,- waitMillis,- TimeUnit.MILLISECONDS,- (n, c) -> {- long docTotal;- try {- docTotal = checkSlicesSameCounts(c);- } catch (SolrServerException | IOException e) {- throw new RuntimeException(e);- }- total.set(docTotal);- if (docTotal == expectedNumFound) {- return true;- }- return false;- });- } catch (TimeoutException | InterruptedException e) {-- }- // We could fail here if we broke out of the above because we exceeded the time allowed.- assertEquals(failureMessage, expectedNumFound, total.get());-- // This should be redundant, but it caught a test error after all.- for (SolrClient client : clients) {- assertEquals(- failureMessage,- expectedNumFound,- client.query(new SolrQuery("*:*")).getResults().getNumFound());- }- }-- // Insure that counts are the same for all replicas in each shard- // Return the total doc count for the query.- private long checkSlicesSameCounts(DocCollection dColl) throws SolrServerException, IOException {- long docTotal = 0; // total number of documents found counting only one replica per slice.- for (Slice slice : dColl.getActiveSlices()) {- long sliceDocCount = -1;- for (Replica rep : slice.getReplicas()) {- try (SolrClient one = getHttpSolrClient(rep)) {- SolrQuery query = new SolrQuery("*:*");- query.setDistrib(false);- QueryResponse resp = one.query(query);- long hits = resp.getResults().getNumFound();- if (sliceDocCount == -1) {- sliceDocCount = hits;- docTotal += hits;- } else {- if (hits != sliceDocCount) {- return -1;- }- }- }- }- }- return docTotal;- }-- private void testShardParamVariations() throws Exception {- SolrQuery query = new SolrQuery("*:*");- Map<String, Long> shardCounts = new HashMap<>();-- for (String shard : shardToJetty.keySet()) {- // every client should give the same numDocs for this shard- // shffle the clients in a diff order for each shard- List<SolrClient> solrclients = new ArrayList<>(this.clients);- Collections.shuffle(solrclients, random());- for (SolrClient client : solrclients) {- query.set("shards", shard);- long numDocs = client.query(query).getResults().getNumFound();- assertTrue("numDocs < 0 for shard " + shard + " via " + client, 0 <= numDocs);- if (!shardCounts.containsKey(shard)) {- shardCounts.put(shard, numDocs);- }- assertEquals(- "inconsitent numDocs for shard " + shard + " via " + client,- shardCounts.get(shard).longValue(),- numDocs);-- List<CloudJettyRunner> replicaJetties = new ArrayList<>(shardToJetty.get(shard));- Collections.shuffle(replicaJetties, random());-- // each replica should also give the same numDocs- ArrayList<String> replicaAlts = new ArrayList<>(replicaJetties.size() * 2);- for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) {- String replica = replicaJetty.url;- query.set("shards", replica);-- // replicas already shuffled, use this in the alternative check below- if (0 == random().nextInt(3) || replicaAlts.size() < 2) {- replicaAlts.add(replica);- }-- numDocs = client.query(query).getResults().getNumFound();- assertTrue("numDocs < 0 for replica " + replica + " via " + client, 0 <= numDocs);- assertEquals(- "inconsitent numDocs for shard "- + shard- + " in replica "- + replica- + " via "- + client,- shardCounts.get(shard).longValue(),- numDocs);- }-- // any combination of replica alternatives should give same numDocs- String replicas = String.join("|", replicaAlts);- query.set("shards", replicas);- numDocs = client.query(query).getResults().getNumFound();- assertTrue("numDocs < 0 for replicas " + replicas + " via " + client, 0 <= numDocs);- assertEquals(- "inconsitent numDocs for replicas " + replicas + " via " + client,- shardCounts.get(shard).longValue(),- numDocs);- }- }-- // sums of multiple shards should add up regardless of how we- // query those shards or which client we use- long randomShardCountsExpected = 0;- ArrayList<String> randomShards = new ArrayList<>(shardCounts.size());- for (Map.Entry<String, Long> shardData : shardCounts.entrySet()) {- if (random().nextBoolean() || randomShards.size() < 2) {- String shard = shardData.getKey();- randomShardCountsExpected += shardData.getValue();- if (random().nextBoolean()) {- // use shard id- randomShards.add(shard);- } else {- // use some set explicit replicas- ArrayList<String> replicas = new ArrayList<>(7);- for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) {- if (0 == random().nextInt(3) || 0 == replicas.size()) {- replicas.add(replicaJetty.url);- }- }- Collections.shuffle(replicas, random());- randomShards.add(String.join("|", replicas));- }- }- }- String randShards = String.join(",", randomShards);- query.set("shards", randShards);- for (SolrClient client : this.clients) {- assertEquals(- "numDocs for " + randShards + " via " + client,- randomShardCountsExpected,- client.query(query).getResults().getNumFound());- }-- // total num docs must match sum of every shard's numDocs- query = new SolrQuery("*:*");- long totalShardNumDocs = 0;- for (Long c : shardCounts.values()) {- totalShardNumDocs += c;- }- for (SolrClient client : clients) {- assertEquals(- "sum of shard numDocs on client: " + client,- totalShardNumDocs,- client.query(query).getResults().getNumFound());- }- assertTrue("total numDocs <= 0, WTF? Test is useless", 0 < totalShardNumDocs);- }-- private void testStopAndStartCoresInOneInstance() throws Exception {- JettySolrRunner jetty = jettys.get(0);- try (final SolrClient httpSolrClient = (HttpSolrClient) jetty.newClient(15000, 60000)) {- ThreadPoolExecutor executor = null;- try {- executor =- new ExecutorUtil.MDCAwareThreadPoolExecutor(- 0,- Integer.MAX_VALUE,- 5,- TimeUnit.SECONDS,- new SynchronousQueue<Runnable>(),- new SolrNamedThreadFactory("testExecutor"));- int cnt = 3;-- // create the cores- createCollectionInOneInstance(- httpSolrClient, jetty.getNodeName(), executor, "multiunload2", 1, cnt);- } finally {- if (executor != null) {- ExecutorUtil.shutdownAndAwaitTermination(executor);- }- }- }-- cloudJettys.get(0).jetty.stop();- printLayout();-- cloudJettys.get(0).jetty.start();- ZkStateReader.from(cloudClient).forceUpdateCollection("multiunload2");- try {- ZkStateReader.from(cloudClient).getLeaderRetry("multiunload2", "shard1", 30000);- } catch (SolrException e) {- printLayout();- throw e;- }-- printLayout();- }-- /** Create a collection in single node */- public static void createCollectionInOneInstance(- final SolrClient client,- String nodeName,- ThreadPoolExecutor executor,- final String collection,- final int numShards,- int numReplicas) {- assertNotNull(nodeName);- try {- assertEquals(- 0,- CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1)- .setCreateNodeSet("")- .process(client)- .getStatus());- } catch (SolrServerException | IOException e) {- throw new RuntimeException(e);- }- for (int i = 0; i < numReplicas; i++) {- final int freezeI = i;- executor.execute(- () -> {- try {- assertTrue(- CollectionAdminRequest.addReplicaToShard(- collection, "shard" + ((freezeI % numShards) + 1))- .setCoreName(collection + freezeI)- .setNode(nodeName)- .process(client)- .isSuccess());- } catch (SolrServerException | IOException e) {- throw new RuntimeException(e);- }- });- }- }-- @Override- protected CollectionAdminResponse createCollection(- Map<String, List<Integer>> collectionInfos,- String collectionName,- String configSetName,- int numShards,- int numReplicas,- SolrClient client,- String createNodeSetStr)- throws SolrServerException, IOException {- // TODO: Use CollectionAdminRequest for this test- ModifiableSolrParams params = new ModifiableSolrParams();- params.set("action", CollectionAction.CREATE.toString());-- params.set(CollectionHandlingUtils.NUM_SLICES, numShards);- params.set(ZkStateReader.REPLICATION_FACTOR, numReplicas);- if (createNodeSetStr != null)- params.set(CollectionHandlingUtils.CREATE_NODE_SET, createNodeSetStr);-- int clientIndex = clients.size() > 1 ? random().nextInt(2) : 0;- List<Integer> list = new ArrayList<>();- list.add(numShards);- list.add(numReplicas);- if (collectionInfos != null) {- collectionInfos.put(collectionName, list);- }- params.set("name", collectionName);- params.set("collection.configName", configSetName);- QueryRequest request = new QueryRequest(params);- request.setPath("/admin/collections");-- CollectionAdminResponse res = new CollectionAdminResponse();- if (client == null) {- final String baseUrl = ((HttpSolrClient) clients.get(clientIndex)).getBaseURL();-- try (SolrClient aClient = createNewSolrClient("", baseUrl)) {- res.setResponse(aClient.request(request));- }- } else {- res.setResponse(client.request(request));- }- return res;- }-- @Override- protected Replica getLeaderFromZk(String collection, String slice) {- ClusterState clusterState = getCommonCloudSolrClient().getClusterState();- Replica leader = clusterState.getCollection(collection).getLeader(slice);- if (leader == null) {- throw new RuntimeException("Could not find leader:" + collection + " " + slice);- }- return leader;- }-- /**- * Expects a RegexReplaceProcessorFactories in the chain which will "double up" the values in two- * (stored) string fields.- *- * <p>If the values are "double-doubled" or "not-doubled" then we know the processor was not run- * the appropriate number of times- */- private void testUpdateProcessorsRunOnlyOnce(final String chain) throws Exception {-- final String fieldA = "regex_dup_A_s";- final String fieldB = "regex_dup_B_s";- final String val = "x";- final String expected = "x_x";- final ModifiableSolrParams updateParams = new ModifiableSolrParams();- updateParams.add(UpdateParams.UPDATE_CHAIN, chain);-- final int numLoops = atLeast(50);-- for (int i = 1; i < numLoops; i++) {- // add doc to random client- SolrClient updateClient = clients.get(random().nextInt(clients.size()));- SolrInputDocument doc = new SolrInputDocument();- addFields(doc, id, i, fieldA, val, fieldB, val);- UpdateResponse ures = add(updateClient, updateParams, doc);- assertEquals(chain + ": update failed", 0, ures.getStatus());- ures = updateClient.commit();- assertEquals(chain + ": commit failed", 0, ures.getStatus());- }-- // query for each doc, and check both fields to ensure the value is correct- for (int i = 1; i < numLoops; i++) {- final String query = id + ":" + i;- QueryResponse qres = queryRandomShard(new SolrQuery(query));- assertEquals(chain + ": query failed: " + query, 0, qres.getStatus());- assertEquals(- chain + ": didn't find correct # docs with query: " + query,- 1,- qres.getResults().getNumFound());- SolrDocument doc = qres.getResults().get(0);-- for (String field : new String[] {fieldA, fieldB}) {- assertEquals(- chain + ": doc#" + i + " has wrong value for " + field,- expected,- doc.getFirstValue(field));- }- }- }-- // cloud level test mainly needed just to make sure that versions and errors are propagated- // correctly- private void doOptimisticLockingAndUpdating() throws Exception {- log.info("### STARTING doOptimisticLockingAndUpdating");- printLayout();-- final SolrInputDocument sd = sdoc("id", 1000, "_version_", -1);- indexDoc(sd);-- ignoreException("version conflict");- for (SolrClient client : clients) {- SolrException e = expectThrows(SolrException.class, () -> client.add(sd));- assertEquals(409, e.code());- }- unIgnoreException("version conflict");-- // TODO: test deletes. SolrJ needs a good way to pass version for delete...-- final SolrInputDocument sd2 = sdoc("id", 1000, "foo_i", 5);- clients.get(0).add(sd2);-- List<Integer> expected = new ArrayList<>();- int val = 0;- for (SolrClient client : clients) {- val += 10;- client.add(sdoc("id", 1000, "val_i", map("add", val), "foo_i", val));- expected.add(val);- }-- QueryRequest qr = new QueryRequest(params("qt", "/get", "id", "1000"));- for (SolrClient client : clients) {- val += 10;- NamedList<?> rsp = client.request(qr);- String match = JSONTestUtil.matchObj("/val_i", rsp.get("doc"), expected);- if (match != null) throw new RuntimeException(match);- }- }-- private void testNumberOfCommitsWithCommitAfterAdd() throws SolrServerException, IOException {- log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd");- long startCommits = getNumCommits((HttpSolrClient) clients.get(0));-- NamedList<Object> result =- clients- .get(0)- .request(- new StreamingUpdateRequest(- "/update", getFile("books_numeric_ids.csv"), "application/csv")- .setCommitWithin(900000)- .setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true));-- long endCommits = getNumCommits((HttpSolrClient) clients.get(0));- assertEquals(startCommits + 1L, endCommits);- }-- private Long getNumCommits(HttpSolrClient sourceClient) throws SolrServerException, IOException {- String collection = sourceClient.getDefaultCollection();- try (SolrClient client =- new HttpSolrClient.Builder(sourceClient.getBaseURL())- .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)- .withSocketTimeout(60000, TimeUnit.MILLISECONDS)- .build()) {- var req =- new GenericSolrRequest(- SolrRequest.METHOD.GET,- "/admin/metrics",- SolrRequest.SolrRequestType.ADMIN,- SolrParams.of("wt", "prometheus"));- req.setResponseParser(new InputStreamResponseParser("prometheus"));-- NamedList<Object> resp = client.request(req);- try (InputStream in = (InputStream) resp.get("stream")) {- String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);- String metricName = "solr_core_update_commit_ops";-- return (long)- output- .lines()- .filter(- l ->- l.startsWith(metricName)- && l.contains("collection=\"" + collection + "\"")- && l.contains("ops=\"commits\""))- .mapToDouble(s -> Double.parseDouble(s.substring(s.lastIndexOf(" "))))- .sum();- }- }- }-- private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception {- log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement");- assertEquals(- 0,- CollectionAdminRequest.createCollection(oneInstanceCollection2, "conf1", 2, 2)- .setCreateNodeSet("")- .process(cloudClient)- .getStatus());-- List<SolrClient> collectionClients = new ArrayList<>();- for (int i = 0; i < 4; i++) {- CollectionAdminResponse resp =- CollectionAdminRequest.addReplicaToShard(oneInstanceCollection2, "shard" + ((i % 2) + 1))- .setNode(jettys.get(0).getNodeName())- .process(cloudClient);- for (String coreName : resp.getCollectionCoresStatus().keySet()) {- collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString()));- }- }-- SolrClient client1 = collectionClients.get(0);- SolrClient client2 = collectionClients.get(1);- SolrClient client3 = collectionClients.get(2);- SolrClient client4 = collectionClients.get(3);-- // no one should be recovering- waitForRecoveriesToFinish(- oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()), false, true);-- assertAllActive(oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()));-- // printLayout();-- // TODO: enable when we don't falsely get slice1...- // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000);- // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000);- client2.add(getDoc(id, "1"));- client3.add(getDoc(id, "2"));- client4.add(getDoc(id, "3"));-- client1.commit();- SolrQuery query = new SolrQuery("*:*");- query.set("distrib", false);- long oneDocs = client1.query(query).getResults().getNumFound();- long twoDocs = client2.query(query).getResults().getNumFound();- long threeDocs = client3.query(query).getResults().getNumFound();- long fourDocs = client4.query(query).getResults().getNumFound();-- query.set("collection", oneInstanceCollection2);- query.set("distrib", true);- long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound();-- // System.out.println("1:" + oneDocs);- // System.out.println("2:" + twoDocs);- // System.out.println("3:" + threeDocs);- // System.out.println("4:" + fourDocs);- // System.out.println("All Docs:" + allDocs);-- // assertEquals(oneDocs, threeDocs);- // assertEquals(twoDocs, fourDocs);- // assertNotSame(oneDocs, twoDocs);- assertEquals(3, allDocs);-- // we added a role of none on these creates - check for it- ZkStateReader zkStateReader = ZkStateReader.from(getCommonCloudSolrClient());- zkStateReader.forceUpdateCollection(oneInstanceCollection2);- Map<String, Slice> slices =- zkStateReader.getClusterState().getCollection(oneInstanceCollection2).getSlicesMap();- assertNotNull(slices);-- Replica leader =- getCommonCloudSolrClient()- .getClusterState()- .getCollection(oneInstanceCollection2)- .getLeader("shard1");-- // now test that unloading a core gets us a new leader- try (SolrClient unloadClient =- new HttpSolrClient.Builder(jettys.get(0).getBaseUrl().toString())- .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)- .withSocketTimeout(60000, TimeUnit.MILLISECONDS)- .build()) {- Unload unloadCmd = new Unload(true);- unloadCmd.setCoreName(leader.getCoreName());-- String leaderUrl = leader.getCoreUrl();-- testExecutor.execute(- new Runnable() {-- @Override- public void run() {- try {- unloadClient.request(unloadCmd);- } catch (SolrServerException e) {- throw new RuntimeException(e);- } catch (IOException e) {- throw new RuntimeException(e);- }- }- });-- try {- ZkStateReader.from(getCommonCloudSolrClient())- .waitForState(- oneInstanceCollection2,- 20000,- TimeUnit.MILLISECONDS,- (n, c) -> {- try {- if (leaderUrl.equals(- zkStateReader.getLeaderUrl(oneInstanceCollection2, "shard1", 10000))) {- return false;- }- } catch (InterruptedException e) {- throw new RuntimeException(e);- }- return true;- });- } catch (TimeoutException | InterruptedException e) {- fail("Leader never changed");- }- }-- IOUtils.close(collectionClients);- }-- private void testSearchByCollectionName() throws SolrServerException, IOException {- log.info("### STARTING testSearchByCollectionName");- SolrClient client = clients.get(0);- final String baseUrl = ((HttpSolrClient) client).getBaseURL();-- // the cores each have different names, but if we add the collection name to the url- // we should get mapped to the right core- try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) {- SolrQuery query = new SolrQuery("*:*");- long oneDocs = client1.query(query).getResults().getNumFound();- assertEquals(3, oneDocs);- }- }-- private void testUpdateByCollectionName() throws SolrServerException, IOException {- log.info("### STARTING testUpdateByCollectionName");- SolrClient client = clients.get(0);- final String baseUrl = ((HttpSolrClient) client).getBaseURL();-- // the cores each have different names, but if we add the collection name to the url- // we should get mapped to the right core- // test hitting an update url- try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) {- client1.commit();- }- }-- private void testANewCollectionInOneInstance() throws Exception {- log.info("### STARTING testANewCollectionInOneInstance");- CollectionAdminResponse response =- CollectionAdminRequest.createCollection(oneInstanceCollection, "conf1", 2, 2)- .setCreateNodeSet(jettys.get(0).getNodeName())- .process(cloudClient);- assertEquals(0, response.getStatus());- List<SolrClient> collectionClients = new ArrayList<>();- for (String coreName : response.getCollectionCoresStatus().keySet()) {- collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString()));- }-- SolrClient client1 = collectionClients.get(0);- SolrClient client2 = collectionClients.get(1);- SolrClient client3 = collectionClients.get(2);- SolrClient client4 = collectionClients.get(3);-- waitForRecoveriesToFinish(- oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()), false);- assertAllActive(oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()));-- client2.add(getDoc(id, "1"));- client3.add(getDoc(id, "2"));- client4.add(getDoc(id, "3"));-- client1.commit();- SolrQuery query = new SolrQuery("*:*");- query.set("distrib", false);- long oneDocs = client1.query(query).getResults().getNumFound();- long twoDocs = client2.query(query).getResults().getNumFound();- long threeDocs = client3.query(query).getResults().getNumFound();- long fourDocs = client4.query(query).getResults().getNumFound();-- query.set("collection", oneInstanceCollection);- query.set("distrib", true);- long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound();-- // System.out.println("1:" + oneDocs);- // System.out.println("2:" + twoDocs);- // System.out.println("3:" + threeDocs);- // System.out.println("4:" + fourDocs);- // System.out.println("All Docs:" + allDocs);-- assertEquals(3, allDocs);- IOUtils.close(collectionClients);- }-- private void createSolrCore(- final String collection,- List<SolrClient> collectionClients,- final String baseUrl,- final int num,- final String shardId) {- Callable<Object> call =- () -> {- try (SolrClient client = getHttpSolrClient(baseUrl)) {- // client.setConnectionTimeout(15000);- Create createCmd = new Create();- createCmd.setCoreName(collection + num);- createCmd.setCollection(collection);-- if (random().nextBoolean()) {- // sometimes we use an explicit core node name- createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet());- }-- if (shardId == null) {- createCmd.setNumShards(2);- }- createCmd.setDataDir(getDataDir(createTempDir(collection).toString()));- if (shardId != null) {- createCmd.setShardId(shardId);- }- client.request(createCmd);- } catch (Exception e) {- log.error("error creating core", e);- // fail- }- return null;- };-- pending.add(completionService.submit(call));-- collectionClients.add(createNewSolrClient(collection + num, baseUrl));- }-- private void testMultipleCollections() throws Exception {- log.info("### STARTING testMultipleCollections");- // create another 2 collections and search across them- createNewCollection("collection2");- createNewCollection("collection3");-- while (pending != null && pending.size() > 0) {-- Future<Object> future = completionService.take();- if (future == null) return;- pending.remove(future);- }-- indexDoc("collection2", getDoc(id, "10000000"));- indexDoc("collection2", getDoc(id, "10000001"));- indexDoc("collection2", getDoc(id, "10000003"));-- getSolrClient("collection2").add(getDoc(id, "10000004"));-- indexDoc("collection3", getDoc(id, "20000000"));- indexDoc("collection3", getDoc(id, "20000001"));-- getSolrClient("collection3").add(getDoc(id, "10000005"));-- otherCollectionClients.get("collection2").get(0).commit();- otherCollectionClients.get("collection3").get(0).commit();-- SolrClient clientForCollection1 = getSolrClient("collection1");- long collection1Docs =- clientForCollection1.query(new SolrQuery("*:*")).getResults().getNumFound();-- long collection2Docs =- otherCollectionClients- .get("collection2")- .get(0)- .query(new SolrQuery("*:*"))- .getResults()- .getNumFound();-- long collection3Docs =- otherCollectionClients- .get("collection3")- .get(0)- .query(new SolrQuery("*:*"))- .getResults()- .getNumFound();-- SolrQuery query = new SolrQuery("*:*");- query.set("collection", "collection2,collection3");- long found = clients.get(0).query(query).getResults().getNumFound();- assertEquals(collection2Docs + collection3Docs, found);-- query = new SolrQuery("*:*");- query.set("collection", "collection1,collection2,collection3");- found = clients.get(0).query(query).getResults().getNumFound();- assertEquals(collection1Docs + collection2Docs + collection3Docs, found);-- // try to search multiple with cloud client- found = getCommonCloudSolrClient().query(query).getResults().getNumFound();- assertEquals(collection1Docs + collection2Docs + collection3Docs, found);-- query.set("collection", "collection2,collection3");- found = getCommonCloudSolrClient().query(query).getResults().getNumFound();- assertEquals(collection2Docs + collection3Docs, found);-- query.set("collection", "collection3");- found = getCommonCloudSolrClient().query(query).getResults().getNumFound();- assertEquals(collection3Docs, found);-- query.remove("collection");- found = getCommonCloudSolrClient().query(query).getResults().getNumFound();- assertEquals(collection1Docs, found);-- assertEquals(collection3Docs, collection2Docs - 1);- }-- protected void indexDoc(String collection, SolrInputDocument doc)- throws IOException, SolrServerException {- List<SolrClient> clients = otherCollectionClients.get(collection);- int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size();- SolrClient client = clients.get(which);- client.add(doc);- }-- @SuppressWarnings({"unchecked"})- private void createNewCollection(final String collection) throws InterruptedException {- try {- assertEquals(- 0,- CollectionAdminRequest.createCollection(collection, "conf1", 2, 1)- .setCreateNodeSet("")- .process(cloudClient)- .getStatus());- } catch (Exception e) {- log.error("error creating collection", e);- // fails- }- final List<SolrClient> collectionClients = new ArrayList<>();- otherCollectionClients.put(collection, collectionClients);- int unique = 0;- for (final JettySolrRunner runner : jettys) {- unique++;- final int frozeUnique = unique;- Callable<Object> call =- () -> {- try {- assertTrue(- CollectionAdminRequest.addReplicaToShard(- collection, "shard" + ((frozeUnique % 2) + 1))- .setNode(runner.getNodeName())- .process(cloudClient)- .isSuccess());- } catch (Exception e) {- log.error("error adding replica", e);- // fails- }- return null;- };-- collectionClients.add(createNewSolrClient(collection, runner.getBaseUrl().toString()));- pending.add(completionService.submit(call));- while (pending != null && pending.size() > 0) {-- Future<Object> future = completionService.take();- if (future == null) return;- pending.remove(future);- }- }- }-- @Override- protected SolrClient createNewSolrClient(String collection, String baseUrl) {-- SolrClient client = getHttpSolrClient(baseUrl, collection);-- return client;- }-- /**- * @param collection the name of a collection or core to set as the "default" on the created- * client.- * @param baseUrl the "base" URL of a Solr node. Should <em>not</em> contain a collection or core- * name.- * @param connectionTimeoutMillis the HTTP connection timeout in milliseconds- * @param socketTimeoutMillis the HTTP socket-read timeout in milliseconds- */- protected SolrClient createNewSolrClient(- String collection, String baseUrl, int connectionTimeoutMillis, int socketTimeoutMillis) {-- SolrClient client =- new HttpSolrClient.Builder(baseUrl)- .withDefaultCollection(collection)- .withConnectionTimeout(connectionTimeoutMillis, TimeUnit.MILLISECONDS)- .withSocketTimeout(socketTimeoutMillis, TimeUnit.MILLISECONDS)- .build();-- return client;- }-- @Override- protected QueryResponse queryRandomShard(ModifiableSolrParams params)- throws SolrServerException, IOException {-- if (r.nextBoolean()) return super.queryRandomShard(params);-- if (r.nextBoolean()) params.set("collection", DEFAULT_COLLECTION);-- QueryResponse rsp = getCommonCloudSolrClient().query(params);- return rsp;- }-- @Override- public void distribTearDown() throws Exception {- super.distribTearDown();- if (otherCollectionClients != null) {- for (List<SolrClient> clientList : otherCollectionClients.values()) {- IOUtils.close(clientList);- }- }- otherCollectionClients = null;- List<Runnable> tasks = executor.shutdownNow();- assertTrue(tasks.isEmpty());- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.javadeleted file mode 100644index 6d55d1f560b..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java+++ /dev/null@@ -1,338 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import java.util.ArrayList;-import java.util.HashSet;-import java.util.List;-import java.util.Set;-import java.util.concurrent.TimeUnit;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.solr.client.solrj.apache.CloudLegacySolrClient;-import org.apache.solr.client.solrj.impl.CloudSolrClient;-import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.cloud.ZkStateReader;-import org.junit.AfterClass;-import org.junit.BeforeClass;-import org.junit.Test;--public abstract class AbstractChaosMonkeyNothingIsSafeTestBase- extends AbstractFullDistribZkTestBase {- private static final int FAIL_TOLERANCE = 100;-- private static final Integer RUN_LENGTH =- Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));-- private final boolean onlyLeaderIndexes = random().nextBoolean();-- @BeforeClass- public static void beforeSuperClass() {- schemaString = "schema15.xml"; // we need a string id- System.setProperty("solr.autoCommit.maxTime", "15000");- System.clearProperty("solr.httpclient.retries");- System.clearProperty("solr.retries.on.forward");- System.clearProperty("solr.retries.to.followers");- setErrorHook();- }-- @AfterClass- public static void afterSuperClass() {- System.clearProperty("solr.autoCommit.maxTime");- clearErrorHook();- }-- @Override- protected void destroyServers() throws Exception {-- super.destroyServers();- }-- protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};- protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};-- private int clientSoTimeout = 60000;-- private volatile FullThrottleStoppableIndexingThread ftIndexThread;-- private final boolean runFullThrottle;-- @Override- public String[] getFieldNames() {- return fieldNames;- }-- @Override- public RandVal[] getRandValues() {- return randVals;- }-- @Override- public void distribSetUp() throws Exception {- super.distribSetUp();- // can help to hide this when testing and looking at logs- // ignoreException("shard update error");- useFactory("solr.StandardDirectoryFactory");- }-- @Override- public void distribTearDown() throws Exception {- try {- ftIndexThread.safeStop();- } catch (NullPointerException e) {- // okay- }- super.distribTearDown();- }-- public AbstractChaosMonkeyNothingIsSafeTestBase() {- super();- sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1"));- if (sliceCount == -1) {- sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1;- }-- int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1"));- if (numShards == -1) {- // we make sure that there's at least one shard with more than one replica- // so that the ChaosMonkey has something to kill- numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1;- }- fixShardCount(numShards);-- // TODO: we only do this sometimes so that we can sometimes compare against control,- // it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer- runFullThrottle = random().nextBoolean();- }-- @Override- protected boolean useTlogReplicas() {- return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug- // and it's TestInjection use- }-- @Override- protected CloudSolrClient createCloudClient(String defaultCollection) {- return this.createCloudClient(defaultCollection, this.clientSoTimeout);- }-- protected CloudSolrClient createCloudClient(String defaultCollection, int socketTimeout) {-- return getCloudSolrClient(- zkServer.getZkAddress(), defaultCollection, random().nextBoolean(), 30000, socketTimeout);- }-- @Test- @SuppressWarnings({"try"})- public void test() throws Exception {- // None of the operations used here are particularly costly, so this should work.- // Using this low timeout will also help us catch index stalling.- clientSoTimeout = 5000;-- boolean testSuccessful = false;- try (CloudSolrClient ourCloudClient = createCloudClient(DEFAULT_COLLECTION)) {- handle.clear();- handle.put("timestamp", SKIPVAL);- ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);- // make sure we have leaders for each shard- for (int j = 1; j < sliceCount; j++) {- zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);- } // make sure we again have leaders for each shard-- waitForRecoveriesToFinish(false);-- // we cannot do delete by query- // as it's not supported for recovery- del("*:*");-- List<StoppableThread> threads = new ArrayList<>();- List<StoppableIndexingThread> indexTreads = new ArrayList<>();- int threadCount = TEST_NIGHTLY ? 3 : 1;- int i = 0;- for (i = 0; i < threadCount; i++) {- StoppableIndexingThread indexThread =- new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true);- threads.add(indexThread);- indexTreads.add(indexThread);- indexThread.start();- }-- threadCount = 1;- i = 0;- for (i = 0; i < threadCount; i++) {- StoppableSearchThread searchThread = new StoppableSearchThread(cloudClient);- threads.add(searchThread);- searchThread.start();- }-- if (runFullThrottle) {- ftIndexThread =- new FullThrottleStoppableIndexingThread(- ((CloudLegacySolrClient) cloudClient).getHttpClient(),- controlClient,- cloudClient,- clients,- "ft1",- true,- this.clientSoTimeout);- ftIndexThread.start();- }-- chaosMonkey.startTheMonkey(true, 10000);- try {- long runLength;- if (RUN_LENGTH != -1) {- runLength = RUN_LENGTH;- } else {- int[] runTimes;- if (TEST_NIGHTLY) {- runTimes =- new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000};- } else {- runTimes = new int[] {5000, 7000, 15000};- }- runLength = runTimes[random().nextInt(runTimes.length - 1)];- }-- Thread.sleep(runLength);- } finally {- chaosMonkey.stopTheMonkey();- }-- // ideally this should go into chaosMonkey- restartZk(1000 * (5 + random().nextInt(4)));-- if (runFullThrottle) {- ftIndexThread.safeStop();- }-- for (StoppableThread indexThread : threads) {- indexThread.safeStop();- }-- // start any downed jetties to be sure we still will end up with a leader per shard...-- // wait for stop...- for (StoppableThread indexThread : threads) {- indexThread.join();- }-- // try and wait for any replications and what not to finish...-- Thread.sleep(2000);-- // wait until there are no recoveries...- waitForThingsToLevelOut();-- // make sure we again have leaders for each shard- for (int j = 1; j < sliceCount; j++) {- zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000);- }-- commit();-- // TODO: assert we didn't kill everyone-- zkStateReader.updateLiveNodes();- assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0);-- // we expect full throttle fails, but cloud client should not easily fail- for (StoppableThread indexThread : threads) {- if (indexThread instanceof StoppableIndexingThread- && !(indexThread instanceof FullThrottleStoppableIndexingThread)) {- int failCount = ((StoppableIndexingThread) indexThread).getFailCount();- assertFalse(- "There were too many update fails ("- + failCount- + " > "- + FAIL_TOLERANCE- + ") - we expect it can happen, but shouldn't easily",- failCount > FAIL_TOLERANCE);- }- }-- waitForThingsToLevelOut(20, TimeUnit.SECONDS);-- commit();-- Set<String> addFails = getAddFails(indexTreads);- Set<String> deleteFails = getDeleteFails(indexTreads);- // full throttle thread can- // have request fails- checkShardConsistency(!runFullThrottle, true, addFails, deleteFails);-- long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();-- // ensure we have added more than 0 docs- long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();-- assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0);-- if (VERBOSE)- System.out.println(- "control docs:"- + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()- + "\n\n");-- // try and make a collection to make sure the overseer has survived the expiration and session- // loss-- // sometimes we restart zookeeper as well- if (random().nextBoolean()) {- // restartZk(1000 * (5 + random().nextInt(4)));- }-- try (CloudSolrClient client = createCloudClient("collection1", 30000)) {- createCollection(null, "testcollection", 1, 1, client, null, "conf1");- }- List<Integer> numShardsNumReplicas = new ArrayList<>(2);- numShardsNumReplicas.add(1);- numShardsNumReplicas.add(1);- checkForCollection("testcollection", numShardsNumReplicas);-- testSuccessful = true;- } finally {- if (!testSuccessful) {- printLayout();- }- }- }-- private Set<String> getAddFails(List<StoppableIndexingThread> threads) {- Set<String> addFails = new HashSet<>();- for (StoppableIndexingThread thread : threads) {- addFails.addAll(thread.getAddFails());- }- return addFails;- }-- private Set<String> getDeleteFails(List<StoppableIndexingThread> threads) {- Set<String> deleteFails = new HashSet<>();- for (StoppableIndexingThread thread : threads) {- deleteFails.addAll(thread.getDeleteFails());- }- return deleteFails;- }-- // skip the randoms - they can deadlock...- @Override- protected void indexr(Object... fields) throws Exception {- SolrInputDocument doc = getDoc(fields);- indexDoc(doc);- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.javadeleted file mode 100644index 5d94eae05e3..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java+++ /dev/null@@ -1,231 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.cloud;--import java.lang.invoke.MethodHandles;-import java.util.ArrayList;-import java.util.List;-import java.util.concurrent.TimeUnit;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.impl.CloudSolrClient;-import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.common.SolrInputDocument;-import org.junit.AfterClass;-import org.junit.BeforeClass;-import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--public abstract class AbstractChaosMonkeySafeLeaderTestBase extends AbstractFullDistribZkTestBase {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- private static final Integer RUN_LENGTH =- Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));-- @BeforeClass- public static void beforeSuperClass() {- schemaString = "schema15.xml"; // we need a string id- System.setProperty("solr.autoCommit.maxTime", "15000");- System.clearProperty("solr.httpclient.retries");- System.clearProperty("solr.retries.on.forward");- System.clearProperty("solr.retries.to.followers");- setErrorHook();- }-- @AfterClass- public static void afterSuperClass() {- System.clearProperty("solr.autoCommit.maxTime");- clearErrorHook();- }-- protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};- protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};-- @Override- public String[] getFieldNames() {- return fieldNames;- }-- @Override- public RandVal[] getRandValues() {- return randVals;- }-- protected abstract String getDirectoryFactory();-- @Override- public void distribSetUp() throws Exception {- useFactory(getDirectoryFactory());- super.distribSetUp();- }-- public AbstractChaosMonkeySafeLeaderTestBase() {- super();- sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1"));- if (sliceCount == -1) {- sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1;- }-- int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1"));- if (numShards == -1) {- // we make sure that there's at least one shard with more than one replica- // so that the ChaosMonkey has something to kill- numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1;- }- fixShardCount(numShards);- }-- @Test- public void test() throws Exception {-- handle.clear();- handle.put("timestamp", SKIPVAL);-- // randomly turn on 1 seconds 'soft' commit- randomlyEnableAutoSoftCommit();-- tryDelete();-- List<StoppableIndexingThread> threads = new ArrayList<>();- int threadCount = 2;- int batchSize = 1;- if (random().nextBoolean()) {- batchSize = random().nextInt(98) + 2;- }-- boolean pauseBetweenUpdates = TEST_NIGHTLY ? random().nextBoolean() : true;- int maxUpdates = -1;- if (!pauseBetweenUpdates) {- maxUpdates = 1000 + random().nextInt(1000);- } else {- maxUpdates = 15000;- }-- for (int i = 0; i < threadCount; i++) {- StoppableIndexingThread indexThread =- new StoppableIndexingThread(- controlClient,- cloudClient,- Integer.toString(i),- true,- maxUpdates,- batchSize,- pauseBetweenUpdates); // random().nextInt(999) + 1- threads.add(indexThread);- indexThread.start();- }-- chaosMonkey.startTheMonkey(false, 500);- try {- long runLength;- if (RUN_LENGTH != -1) {- runLength = RUN_LENGTH;- } else {- int[] runTimes;- if (TEST_NIGHTLY) {- runTimes =- new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000};- } else {- runTimes = new int[] {5000, 7000, 15000};- }- runLength = runTimes[random().nextInt(runTimes.length - 1)];- }-- Thread.sleep(runLength);- } finally {- chaosMonkey.stopTheMonkey();- }-- for (StoppableIndexingThread indexThread : threads) {- indexThread.safeStop();- }-- // wait for stop...- for (StoppableIndexingThread indexThread : threads) {- indexThread.join();- }-- for (StoppableIndexingThread indexThread : threads) {- assertEquals(0, indexThread.getFailCount());- }-- // try and wait for any replications and what not to finish...-- Thread.sleep(2000);-- waitForThingsToLevelOut(3, TimeUnit.MINUTES);-- // even if things were leveled out, a jetty may have just been stopped or something- // we wait again and wait to level out again to make sure the system is not still in flux-- Thread.sleep(3000);-- waitForThingsToLevelOut(3, TimeUnit.MINUTES);-- checkShardConsistency(batchSize == 1, true);-- if (VERBOSE)- System.out.println(- "control docs:"- + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()- + "\n\n");-- // try and make a collection to make sure the overseer has survived the expiration and session- // loss-- // sometimes we restart zookeeper as well- if (random().nextBoolean()) {- zkServer.shutdown();- zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());- zkServer.run(false);- }-- try (CloudSolrClient client = createCloudClient("collection1")) {- createCollection(null, "testcollection", 1, 1, client, null, "conf1");- }- List<Integer> numShardsNumReplicas = new ArrayList<>(2);- numShardsNumReplicas.add(1);- numShardsNumReplicas.add(1);- checkForCollection("testcollection", numShardsNumReplicas);- }-- private void tryDelete() throws Exception {- long start = System.nanoTime();- long timeout = start + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS);- while (System.nanoTime() < timeout) {- try {- del("*:*");- break;- } catch (SolrServerException e) {- log.error("cluster may not be up yet", e);- }- Thread.sleep(100);- }- }-- // skip the randoms - they can deadlock...- @Override- protected void indexr(Object... fields) throws Exception {- SolrInputDocument doc = new SolrInputDocument();- addFields(doc, fields);- addFields(doc, "rnd_b", true);- indexDoc(doc);- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.javadeleted file mode 100644index 31bfdb837c6..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java+++ /dev/null@@ -1,567 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_FILE_VM_PARAM_NAME;-import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME;-import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME;-import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE;--import java.io.FileWriter;-import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.nio.charset.Charset;-import java.nio.charset.StandardCharsets;-import java.nio.file.Path;-import java.util.Collections;-import java.util.List;-import java.util.Properties;-import java.util.concurrent.TimeUnit;-import org.apache.solr.SolrTestCaseJ4;-import org.apache.solr.common.cloud.DigestZkACLProvider;-import org.apache.solr.common.cloud.DigestZkCredentialsProvider;-import org.apache.solr.common.cloud.SecurityAwareZkACLProvider;-import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.VMParamsZkCredentialsInjector;-import org.apache.solr.common.cloud.ZkCredentialsInjector;-import org.apache.zookeeper.CreateMode;-import org.apache.zookeeper.KeeperException.NoAuthException;-import org.junit.AfterClass;-import org.junit.BeforeClass;-import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--public class AbstractDigestZkACLAndCredentialsProvidersTestBase extends SolrTestCaseJ4 {-- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- private static final Charset DATA_ENCODING = StandardCharsets.UTF_8;-- private static final String ALL_USERNAME = "connectAndAllACLUsername";- private static final String ALL_PASSWORD = "connectAndAllACLPassword";- private static final String READONLY_USERNAME = "readonlyACLUsername";- private static final String READONLY_PASSWORD = "readonlyACLPassword";-- protected ZkTestServer zkServer;-- protected Path zkDir;-- @BeforeClass- public static void beforeClass() {- System.setProperty("solrcloud.skip.autorecovery", "true");- }-- @AfterClass- public static void afterClass() {- System.clearProperty("solrcloud.skip.autorecovery");- }-- @Override- public void setUp() throws Exception {- // TODO: Does all of this setup need to happen for each test case, or can it be done once for- // the class? (i.e. @BeforeClass) and maybe some minor reset logic in setup, instead of full- // startup and teardown of a new ZkTestServer in each cycle?- super.setUp();- if (log.isInfoEnabled()) {- log.info("####SETUP_START {}", getTestName());- }- createTempDir();-- zkDir = createTempDir().resolve("zookeeper/server1/data");- log.info("ZooKeeper dataDir:{}", zkDir);- setSecuritySystemProperties();- zkServer = new ZkTestServer(zkDir);- zkServer.run(false);-- System.setProperty("zkHost", zkServer.getZkAddress());-- setDigestZkSystemProps();- System.setProperty(- SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,- AllAndReadonlyCredentialZkCredentialsInjector.class.getName());-- SolrZkClient zkClient =- new SolrZkClient.Builder()- .withUrl(zkServer.getZkHost())- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)- .withConnTimeOut(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)- .build();-- zkClient.makePath("/solr", false, true);- zkClient.close();-- zkClient =- new SolrZkClient.Builder()- .withUrl(zkServer.getZkAddress())- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)- .build();- zkClient.create(- "/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);- zkClient.makePath(- "/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);-- zkClient.create(- SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH,- "content".getBytes(DATA_ENCODING),- CreateMode.PERSISTENT,- false);- zkClient.close();-- clearSecuritySystemProperties();-- zkClient =- new SolrZkClient.Builder()- .withUrl(zkServer.getZkAddress())- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)- .build();- // Currently, no credentials on ZK connection, because those same VM-params are used for adding- // ACLs, and here we want- // no (or completely open) ACLs added. Therefore, hack your way into being authorized for- // creating anyway- zkClient- .getCuratorFramework()- .getZookeeperClient()- .getZooKeeper()- .addAuthInfo(- "digest", (ALL_USERNAME + ":" + ALL_PASSWORD).getBytes(StandardCharsets.UTF_8));- zkClient.create(- "/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);- zkClient.makePath(- "/unprotectedMakePathNode",- "content".getBytes(DATA_ENCODING),- CreateMode.PERSISTENT,- false);- zkClient.close();-- setDigestZkSystemProps();- if (log.isInfoEnabled()) {- log.info("####SETUP_END {}", getTestName());- }- }-- private void setDigestZkSystemProps() {- System.setProperty(- SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME,- DigestZkCredentialsProvider.class.getName());- System.setProperty(- SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName());- }-- @Override- public void tearDown() throws Exception {- zkServer.shutdown();-- clearSecuritySystemProperties();-- super.tearDown();- }-- @Test- public void testNoCredentials() throws Exception {- List<TestZkCredentialsInjector> testZkCredentialsInjectors =- List.of(- new TestZkCredentialsInjector(NoCredentialZkCredentialsInjector.class),- new TestZkCredentialsInjector(VMParamsZkCredentialsInjector.class));-- testInjectors(- testZkCredentialsInjectors,- false,- false,- false,- false,- false,- false,- false,- false,- false,- false);- }-- @Test- public void testWrongCredentials() throws Exception {- List<TestZkCredentialsInjector> testZkCredentialsInjectors =- List.of(- new TestZkCredentialsInjector(WrongAllCredentialZkCredentialsInjector.class),- new TestZkCredentialsInjector(- VMParamsZkCredentialsInjector.class,- List.of(- DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),- List.of(ALL_USERNAME, ALL_PASSWORD + "Wrong")));-- testInjectors(- testZkCredentialsInjectors,- false,- false,- false,- false,- false,- false,- false,- false,- false,- false);- }-- @Test- public void testAllCredentials() throws Exception {- List<TestZkCredentialsInjector> testZkCredentialsInjectors =- List.of(- new TestZkCredentialsInjector(AllCredentialZkCredentialsInjector.class),- new TestZkCredentialsInjector(- VMParamsZkCredentialsInjector.class,- List.of(- DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),- List.of(ALL_USERNAME, ALL_PASSWORD)));-- testInjectors(- testZkCredentialsInjectors, true, true, true, true, true, true, true, true, true, true);- }-- @Test- public void testReadonlyCredentials() throws Exception {- List<TestZkCredentialsInjector> testZkCredentialsInjectors =- List.of(- new TestZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class),- new TestZkCredentialsInjector(- VMParamsZkCredentialsInjector.class,- List.of(- DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),- List.of(READONLY_USERNAME, READONLY_PASSWORD)));- testInjectors(- testZkCredentialsInjectors,- true,- true,- false,- false,- false,- false,- false,- false,- false,- false);- }-- protected void testInjectors(- List<TestZkCredentialsInjector> testZkCredentialsInjectors,- boolean getData,- boolean list,- boolean create,- boolean setData,- boolean delete,- boolean secureGet,- boolean secureList,- boolean secureCreate,- boolean secureSet,- boolean secureDelete)- throws Exception {- for (TestZkCredentialsInjector testZkCredentialsInjector : testZkCredentialsInjectors) {- tearDown();- setUp();- testZkCredentialsInjector.setSystemProps();- try (SolrZkClient zkClient =- new SolrZkClient.Builder()- .withUrl(zkServer.getZkAddress())- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)- .build()) {- doTest(- zkClient,- getData,- list,- create,- setData,- delete,- secureGet,- secureList,- secureCreate,- secureSet,- secureDelete);- }- }- }-- @Test- public void testRepairACL() throws Exception {- clearSecuritySystemProperties();- try (SolrZkClient zkClient =- new SolrZkClient.Builder()- .withUrl(zkServer.getZkAddress())- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)- .build()) {- // Currently, no credentials on ZK connection, because those same VM-params are used for- // adding ACLs, and here we want- // no (or completely open) ACLs added. Therefore, hack your way into being authorized for- // creating anyway- zkClient- .getCuratorFramework()- .getZookeeperClient()- .getZooKeeper()- .addAuthInfo(- "digest",- ("connectAndAllACLUsername:connectAndAllACLPassword")- .getBytes(StandardCharsets.UTF_8));-- zkClient.create(- "/security.json", "{}".getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, false);- assertEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false));- }-- setSecuritySystemProperties();- try (SolrZkClient zkClient =- new SolrZkClient.Builder()- .withUrl(zkServer.getZkAddress())- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)- .build()) {- ZkController.createClusterZkNodes(zkClient);- assertNotEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false));- }-- useZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class);- // useReadonlyCredentials();- try (SolrZkClient zkClient =- new SolrZkClient.Builder()- .withUrl(zkServer.getZkAddress())- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)- .build()) {- NoAuthException e =- assertThrows(- NoAuthException.class, () -> zkClient.getData("/security.json", null, null, false));- assertEquals("/solr/security.json", e.getPath());- }- }-- private void useZkCredentialsInjector(Class<?> zkCredentialsInjectorClass) {- clearSecuritySystemProperties();- setDigestZkSystemProps();- System.setProperty(- SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,- zkCredentialsInjectorClass.getName());- }-- private void setSecuritySystemProperties() {- System.setProperty(- SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME,- DigestZkCredentialsProvider.class.getName());- System.setProperty(- SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName());- System.setProperty(- SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,- AllAndReadonlyCredentialZkCredentialsInjector.class.getName());- }-- private void clearSecuritySystemProperties() {- System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME);- System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME);- System.clearProperty(SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME);- }-- public static void doTest(- SolrZkClient zkClient,- boolean getData,- boolean list,- boolean create,- boolean setData,- boolean delete,- boolean secureGet,- boolean secureList,- boolean secureCreate,- boolean secureSet,- boolean secureDelete)- throws Exception {- doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete);- doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete);- doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete);- doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete);- doTest(- zkClient,- SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH,- secureGet,- secureList,- secureCreate,- secureSet,- secureDelete);- }-- protected static void doTest(- SolrZkClient zkClient,- String path,- boolean getData,- boolean list,- boolean create,- boolean setData,- boolean delete)- throws Exception {- doTest(getData, () -> zkClient.getData(path, null, null, false));- doTest(list, () -> zkClient.getChildren(path, null, false));-- doTest(- create,- () -> {- zkClient.create(path + "/subnode", null, CreateMode.PERSISTENT, false);- zkClient.delete(path + "/subnode", -1, false);- });- doTest(- create,- () -> {- zkClient.makePath(path + "/subnode/subsubnode", false);- zkClient.delete(path + "/subnode/subsubnode", -1, false);- zkClient.delete(path + "/subnode", -1, false);- });-- doTest(setData, () -> zkClient.setData(path, (byte[]) null, false));-- // Actually about the ACLs on /solr, but that is protected- doTest(delete, () -> zkClient.delete(path, -1, false));- }-- interface ExceptingRunnable {- void run() throws Exception;- }-- private static void doTest(boolean shouldSucceed, ExceptingRunnable action) throws Exception {- if (shouldSucceed) {- action.run();- } else {- expectThrows(NoAuthException.class, action::run);- }- }-- @Test- public void testVMParamsAllCredentialsFromFile() throws Exception {- useVMParamsAllCredentialsFromFile();-- try (SolrZkClient zkClient =- new SolrZkClient.Builder()- .withUrl(zkServer.getZkAddress())- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)- .build()) {- doTest(zkClient, true, true, true, true, true, true, true, true, true, true);- }- }-- @Test- public void testVMParamsReadonlyCredentialsFromFile() throws Exception {- useVMParamsReadonlyCredentialsFromFile();-- try (SolrZkClient zkClient =- new SolrZkClient.Builder()- .withUrl(zkServer.getZkAddress())- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)- .build()) {- doTest(zkClient, true, true, false, false, false, false, false, false, false, false);- }- }-- private void useVMParamsAllCredentialsFromFile() throws IOException {- useVMParamsCredentialsFromFile("connectAndAllACLUsername", "connectAndAllACLPassword");- }-- private void useVMParamsReadonlyCredentialsFromFile() throws IOException {- useVMParamsCredentialsFromFile("readonlyACLUsername", "readonlyACLPassword");- }-- private void useVMParamsCredentialsFromFile(String username, String password) throws IOException {- Properties props = new Properties();- props.setProperty(- VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, username);- props.setProperty(- VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, password);- String credsFile = saveCredentialsFile(props);-- useZkCredentialsInjector(VMParamsZkCredentialsInjector.class);- System.setProperty(DEFAULT_DIGEST_FILE_VM_PARAM_NAME, credsFile);- }-- private String saveCredentialsFile(Properties props) throws IOException {- Path tmp = createTempFile("zk-creds", "properties");- try (FileWriter w = new FileWriter(tmp.toFile(), StandardCharsets.UTF_8)) {- props.store(w, "test");- }- return tmp.toAbsolutePath().toString();- }-- public static class NoCredentialZkCredentialsInjector implements ZkCredentialsInjector {- @Override- public List<ZkCredential> getZkCredentials() {- return Collections.emptyList();- }- }-- public static class AllAndReadonlyCredentialZkCredentialsInjector- implements ZkCredentialsInjector {- @Override- public List<ZkCredential> getZkCredentials() {- return List.of(- new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL),- new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.READ));- }- }-- public static class ConnectWithReadonlyCredsInjector implements ZkCredentialsInjector {- @Override- public List<ZkCredential> getZkCredentials() {- return List.of(- new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.ALL));- }- }-- public static class AllCredentialZkCredentialsInjector implements ZkCredentialsInjector {- @Override- public List<ZkCredential> getZkCredentials() {- return List.of(new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL));- }- }-- public static class WrongAllCredentialZkCredentialsInjector implements ZkCredentialsInjector {- @Override- public List<ZkCredential> getZkCredentials() {- return List.of(- new ZkCredential(ALL_USERNAME, ALL_PASSWORD + "Wrong", ZkCredential.Perms.ALL));- }- }-- class TestZkCredentialsInjector {- private final Class<?> zkCredentialsInjectorClass;- private final List<String> systemPropsKeys;- private final List<String> systemPropsValues;-- public TestZkCredentialsInjector(Class<?> zkCredentialsInjectorClass) {- this(zkCredentialsInjectorClass, Collections.emptyList(), Collections.emptyList());- }-- public TestZkCredentialsInjector(- Class<?> zkCredentialsInjectorClass,- List<String> systemPropsKeys,- List<String> systemPropsValues) {- this.zkCredentialsInjectorClass = zkCredentialsInjectorClass;- this.systemPropsKeys = systemPropsKeys;- this.systemPropsValues = systemPropsValues;- }-- private void setSystemProps() {- clearSecuritySystemProperties();- setDigestZkSystemProps();- System.setProperty(- SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,- zkCredentialsInjectorClass.getName());- int i = 0;- for (String key : systemPropsKeys) {- System.setProperty(key, systemPropsValues.get(i++));- }- }- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.javaindex e123fa6d53f..5a24a1f2f4d 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java@@ -133,6 +133,12 @@ public abstract class AbstractFullDistribZkTestBase extends BaseDistributedSearcprotected volatile ZkTestServer zkServer;private final AtomicInteger homeCount = new AtomicInteger();+ @BeforeClass+ public static void beforeThisClass() throws Exception {+ // Only For Manual Testing: this will force an fs based dir factory+ // useFactory(null);+ }+@BeforeClasspublic static void beforeFullSolrCloudTest() {}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.javadeleted file mode 100644index da9ed3da1fe..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java+++ /dev/null@@ -1,410 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.cloud;--import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.util.ArrayList;-import java.util.Collection;-import java.util.Collections;-import java.util.List;-import java.util.Map;-import java.util.Set;-import org.apache.solr.client.api.model.CoreStatusResponse;-import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.impl.CloudSolrClient;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.request.CoreAdminRequest;-import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.client.solrj.response.CoreAdminResponse;-import org.apache.solr.client.solrj.response.RequestStatusState;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.cloud.DocCollection;-import org.apache.solr.common.cloud.Replica;-import org.apache.solr.common.cloud.Slice;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.embedded.JettySolrRunner;-import org.apache.solr.util.IdUtils;-import org.junit.After;-import org.junit.Before;-import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--public abstract class AbstractMoveReplicaTestBase extends SolrCloudTestCase {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- // used by MoveReplicaHDFSTest- protected boolean inPlaceMove = true;- protected boolean isCollectionApiDistributed = false;-- protected String getConfigSet() {- return "cloud-dynamic";- }-- @Before- public void beforeTest() throws Exception {- inPlaceMove = true;-- configureCluster(4)- .addConfig("conf1", configset(getConfigSet()))- .addConfig("conf2", configset(getConfigSet()))- .withSolrXml(TEST_PATH().resolve("solr.xml"))- .configure();-- // If Collection API is distributed let's not wait for Overseer.- isCollectionApiDistributed =- new CollectionAdminRequest.RequestApiDistributedProcessing()- .process(cluster.getSolrClient())- .getIsCollectionApiDistributed();- if (isCollectionApiDistributed) {- return;- }-- NamedList<Object> overSeerStatus =- cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());- JettySolrRunner overseerJetty = null;- String overseerLeader = (String) overSeerStatus.get("leader");- for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {- if (jetty.getNodeName().equals(overseerLeader)) {- overseerJetty = jetty;- break;- }- }- if (overseerJetty == null) {- fail("no overseer leader!");- }- }-- @After- public void afterTest() throws Exception {- try {- shutdownCluster();- } finally {- super.tearDown();- }- }-- @Test- public void test() throws Exception {- String coll = getTestClass().getSimpleName() + "_coll_" + inPlaceMove;- if (log.isInfoEnabled()) {- log.info("total_jettys: {}", cluster.getJettySolrRunners().size());- }- int REPLICATION = 2;-- CloudSolrClient cloudClient = cluster.getSolrClient();-- // random create tlog or pull type replicas with nrt- boolean isTlog = random().nextBoolean();- CollectionAdminRequest.Create create =- CollectionAdminRequest.createCollection(- coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0);- cloudClient.request(create);-- addDocs(coll, 100);-- Replica replica = getRandomReplica(coll, cloudClient);- Set<String> liveNodes = cloudClient.getClusterState().getLiveNodes();- ArrayList<String> l = new ArrayList<>(liveNodes);- Collections.shuffle(l, random());- String targetNode = null;- for (String node : liveNodes) {- if (!replica.getNodeName().equals(node)) {- targetNode = node;- break;- }- }- assertNotNull(targetNode);- String shardId = null;- for (Slice slice : cloudClient.getClusterState().getCollection(coll).getSlices()) {- if (slice.getReplicas().contains(replica)) {- shardId = slice.getName();- }- }-- int sourceNumCores =- getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name());- int targetNumCores = getNumOfCores(cloudClient, targetNode, coll, replica.getType().name());-- CollectionAdminRequest.MoveReplica moveReplica =- createMoveReplicaRequest(coll, replica, targetNode);- moveReplica.setInPlaceMove(inPlaceMove);- String asyncId = IdUtils.randomId();- moveReplica.processAsync(asyncId, cloudClient);- CollectionAdminRequest.RequestStatus requestStatus =- CollectionAdminRequest.requestStatus(asyncId);- // wait for async request success- boolean success = false;- for (int i = 0; i < 200; i++) {- CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);- if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) {- success = true;- break;- }- assertNotSame(rsp.getRequestStatus(), RequestStatusState.FAILED);- Thread.sleep(500);- }- assertTrue(success);- assertEquals(- "should be one less core on the source node!",- sourceNumCores - 1,- getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name()));- assertEquals(- "should be one more core on target node!",- targetNumCores + 1,- getNumOfCores(cloudClient, targetNode, coll, replica.getType().name()));- // wait for recovery- boolean recovered = false;- for (int i = 0; i < 300; i++) {- DocCollection collState = getCollectionState(coll);- log.debug("###### {}", collState);- Collection<Replica> replicas = collState.getSlice(shardId).getReplicas();- boolean allActive = true;- boolean hasLeaders = true;- if (replicas != null && !replicas.isEmpty()) {- for (Replica r : replicas) {- if (!r.getNodeName().equals(targetNode)) {- continue;- }- if (!r.isActive(Collections.singleton(targetNode))) {- log.info("Not active: {}", r);- allActive = false;- }- }- } else {- allActive = false;- }- for (Slice slice : collState.getSlices()) {- if (slice.getLeader() == null) {- hasLeaders = false;- }- }- if (allActive && hasLeaders) {- // check the number of active replicas- assertEquals("total number of replicas", REPLICATION, replicas.size());- recovered = true;- break;- } else {- log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders);- Thread.sleep(1000);- }- }- assertTrue("replica never fully recovered", recovered);-- assertEquals(- 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());-- moveReplica = createMoveReplicaRequest(coll, replica, targetNode, shardId);- moveReplica.setInPlaceMove(inPlaceMove);- moveReplica.process(cloudClient);- checkNumOfCores(cloudClient, replica.getNodeName(), coll, sourceNumCores);- // wait for recovery- recovered = false;- for (int i = 0; i < 300; i++) {- DocCollection collState = getCollectionState(coll);- log.debug("###### {}", collState);- Collection<Replica> replicas = collState.getSlice(shardId).getReplicas();- boolean allActive = true;- boolean hasLeaders = true;- if (replicas != null && !replicas.isEmpty()) {- for (Replica r : replicas) {- if (!r.getNodeName().equals(replica.getNodeName())) {- continue;- }- if (!r.isActive(Collections.singleton(replica.getNodeName()))) {- log.info("Not active yet: {}", r);- allActive = false;- }- }- } else {- allActive = false;- }- for (Slice slice : collState.getSlices()) {- if (slice.getLeader() == null) {- hasLeaders = false;- }- }- if (allActive && hasLeaders) {- assertEquals("total number of replicas", REPLICATION, replicas.size());- recovered = true;- break;- } else {- Thread.sleep(1000);- }- }- assertTrue("replica never fully recovered", recovered);-- assertEquals(- 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());- }-- @Test- public void testFailedMove() throws Exception {- String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove;- int REPLICATION = 2;-- CloudSolrClient cloudClient = cluster.getSolrClient();-- // random create tlog or pull type replicas with nrt- boolean isTlog = random().nextBoolean();- CollectionAdminRequest.Create create =- CollectionAdminRequest.createCollection(- coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0);- cloudClient.request(create);-- addDocs(coll, 100);-- NamedList<Object> overSeerStatus =- cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());- String overseerLeader = (String) overSeerStatus.get("leader");-- // don't kill overseer in this test- Replica replica;- int count = 10;- do {- replica = getRandomReplica(coll, cloudClient);- } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0);- assertNotNull("could not find non-overseer replica???", replica);- Set<String> liveNodes = cloudClient.getClusterState().getLiveNodes();- ArrayList<String> l = new ArrayList<>(liveNodes);- Collections.shuffle(l, random());- String targetNode = null;- for (String node : liveNodes) {- if (!replica.getNodeName().equals(node)- && (isCollectionApiDistributed || !overseerLeader.equals(node))) {- targetNode = node;- break;- }- }- assertNotNull(targetNode);- CollectionAdminRequest.MoveReplica moveReplica =- createMoveReplicaRequest(coll, replica, targetNode);- moveReplica.setInPlaceMove(inPlaceMove);- // start moving- String asyncId = IdUtils.randomId();- moveReplica.processAsync(asyncId, cloudClient);- // shut down target node- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {- if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) {- JettySolrRunner j = cluster.stopJettySolrRunner(i);- cluster.waitForJettyToStop(j);- break;- }- }- CollectionAdminRequest.RequestStatus requestStatus =- CollectionAdminRequest.requestStatus(asyncId);- // wait for async request success- boolean success = true;- for (int i = 0; i < 200; i++) {- CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);- assertNotSame(- rsp.getRequestStatus().toString(), rsp.getRequestStatus(), RequestStatusState.COMPLETED);- if (rsp.getRequestStatus() == RequestStatusState.FAILED) {- success = false;- break;- }- Thread.sleep(500);- }- assertFalse(success);-- if (log.isInfoEnabled()) {- log.info(- "--- current collection state: {}", cloudClient.getClusterState().getCollection(coll));- }- assertEquals(- 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());- }-- private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(- String coll, Replica replica, String targetNode, String shardId) {- return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName());- }-- private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(- String coll, Replica replica, String targetNode) {- return new CollectionAdminRequest.MoveReplica(coll, replica.getName(), targetNode);- }-- private Replica getRandomReplica(String coll, CloudSolrClient cloudClient) throws IOException {- List<Replica> replicas = cloudClient.getClusterState().getCollection(coll).getReplicas();- Collections.shuffle(replicas, random());- return replicas.get(0);- }-- private void checkNumOfCores(- CloudSolrClient cloudClient, String nodeName, String collectionName, int expectedCores)- throws IOException, SolrServerException {- assertEquals(- nodeName + " does not have expected number of cores",- expectedCores,- getNumOfCores(cloudClient, nodeName, collectionName));- }-- private int getNumOfCores(CloudSolrClient cloudClient, String nodeName, String collectionName)- throws IOException, SolrServerException {- return getNumOfCores(cloudClient, nodeName, collectionName, null);- }-- private int getNumOfCores(- CloudSolrClient cloudClient, String nodeName, String collectionName, String replicaType)- throws IOException, SolrServerException {- try (SolrClient coreclient =- getHttpSolrClient(ZkStateReader.from(cloudClient).getBaseUrlForNodeName(nodeName))) {- CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient);- if (status.getCoreStatus().size() == 0) {- return 0;- }- if (collectionName == null && replicaType == null) {- return status.getCoreStatus().size();- }- // filter size by collection name- int size = 0;- for (Map.Entry<String, CoreStatusResponse.SingleCoreData> coreStatusEntry :- status.getCoreStatus().entrySet()) {- if (collectionName != null) {- String coll = coreStatusEntry.getValue().cloud.collection;- if (!collectionName.equals(coll)) {- continue;- }- }- if (replicaType != null) {- String type = coreStatusEntry.getValue().cloud.replicaType;- if (!replicaType.equals(type)) {- continue;- }- }- size++;- }- return size;- }- }-- protected void addDocs(String collection, int numDocs) throws Exception {- SolrClient solrClient = cluster.getSolrClient();- for (int docId = 1; docId <= numDocs; docId++) {- SolrInputDocument doc = new SolrInputDocument();- doc.addField("id", docId);- solrClient.add(collection, doc);- }- solrClient.commit(collection);- Thread.sleep(5000);- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.javadeleted file mode 100644index 545a9486c41..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java+++ /dev/null@@ -1,158 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import java.lang.invoke.MethodHandles;-import java.util.ArrayList;-import java.util.List;-import java.util.concurrent.TimeUnit;-import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.apache.CloudLegacySolrClient;-import org.apache.solr.client.solrj.apache.HttpSolrClient;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.client.solrj.request.UpdateRequest;-import org.apache.solr.common.cloud.DocCollection;-import org.apache.solr.common.cloud.Replica;-import org.apache.solr.common.cloud.Slice;-import org.apache.solr.embedded.JettySolrRunner;-import org.junit.After;-import org.junit.BeforeClass;-import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--public abstract class AbstractRecoveryZkTestBase extends SolrCloudTestCase {-- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- @BeforeClass- public static void setupCluster() throws Exception {- cluster = configureCluster(2).addConfig("conf", configset("cloud-minimal")).configure();- }-- private final List<StoppableIndexingThread> threads = new ArrayList<>();-- @After- public void stopThreads() throws InterruptedException {- for (StoppableIndexingThread t : threads) {- t.safeStop();- }- for (StoppableIndexingThread t : threads) {- t.join();- }- threads.clear();- }-- @Test- public void test() throws Exception {-- final String collection = "recoverytest";-- CollectionAdminRequest.createCollection(collection, "conf", 1, 2)- .process(cluster.getSolrClient());- waitForState(- "Expected a collection with one shard and two replicas", collection, clusterShape(1, 2));-- // start a couple indexing threads-- int[] maxDocList = new int[] {300, 700, 1200, 1350, 3000};- int[] maxDocNightlyList = new int[] {3000, 7000, 12000, 30000, 45000, 60000};-- int maxDoc;- if (!TEST_NIGHTLY) {- maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];- } else {- maxDoc = maxDocNightlyList[random().nextInt(maxDocList.length - 1)];- }- log.info("Indexing {} documents", maxDoc);-- try (SolrClient solrClient =- cluster.basicSolrClientBuilder().withDefaultCollection(collection).build(); ) {- final StoppableIndexingThread indexThread =- new StoppableIndexingThread(null, solrClient, "1", true, maxDoc, 1, true);- threads.add(indexThread);- indexThread.start();-- final StoppableIndexingThread indexThread2 =- new StoppableIndexingThread(null, solrClient, "2", true, maxDoc, 1, true);- threads.add(indexThread2);- indexThread2.start();-- // give some time to index...- int[] waitTimes = new int[] {200, 2000, 3000};- Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);-- // bring shard replica down- DocCollection state = getCollectionState(collection);- Replica leader = state.getLeader("shard1");- Replica replica = getRandomReplica(state.getSlice("shard1"), (r) -> !leader.equals(r));-- JettySolrRunner jetty = cluster.getReplicaJetty(replica);- jetty.stop();-- // wait a moment - lets allow some docs to be indexed so replication time is non 0- Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);-- // bring shard replica up- jetty.start();-- // make sure replication can start- Thread.sleep(3000);-- // stop indexing threads- indexThread.safeStop();- indexThread2.safeStop();-- indexThread.join();- indexThread2.join();-- new UpdateRequest().commit(solrClient, collection);-- cluster- .getZkStateReader()- .waitForState(collection, 120, TimeUnit.SECONDS, clusterShape(1, 2));-- // test that leader and replica have same doc count- state = getCollectionState(collection);- assertShardConsistency(state.getSlice("shard1"), true);- }- }-- private void assertShardConsistency(Slice shard, boolean expectDocs) throws Exception {- List<Replica> replicas = shard.getReplicas(r -> r.getState() == Replica.State.ACTIVE);- long[] numCounts = new long[replicas.size()];- int i = 0;- for (Replica replica : replicas) {- try (var client =- new HttpSolrClient.Builder(replica.getBaseUrl())- .withDefaultCollection(replica.getCoreName())- .withHttpClient(((CloudLegacySolrClient) cluster.getSolrClient()).getHttpClient())- .build()) {- numCounts[i] =- client.query(new SolrQuery("*:*").add("distrib", "false")).getResults().getNumFound();- i++;- }- }- for (int j = 1; j < replicas.size(); j++) {- if (numCounts[j] != numCounts[j - 1])- fail("Mismatch in counts between replicas"); // TODO improve this!- if (numCounts[j] == 0 && expectDocs)- fail("Expected docs on shard " + shard.getName() + " but found none");- }- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.javadeleted file mode 100644index 38733b30fbe..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java+++ /dev/null@@ -1,199 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import java.io.IOException;-import java.util.ArrayList;-import java.util.List;-import java.util.concurrent.TimeUnit;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.cloud.ZkStateReader;-import org.junit.AfterClass;-import org.junit.BeforeClass;-import org.junit.Test;--public abstract class AbstractRestartWhileUpdatingTestBase extends AbstractFullDistribZkTestBase {-- // private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain";- private List<StoppableIndexingThread> threads;-- private volatile boolean stopExpire = false;-- public AbstractRestartWhileUpdatingTestBase() throws Exception {- super();- sliceCount = 1;- fixShardCount(3);- schemaString = "schema15.xml"; // we need a string id- useFactory("solr.StandardDirectoryFactory");- }-- public static String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};- public static RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};-- @Override- protected String[] getFieldNames() {- return fieldNames;- }-- @Override- protected RandVal[] getRandValues() {- return randVals;- }-- @BeforeClass- public static void beforeRestartWhileUpdatingTest() {- System.setProperty("leaderVoteWait", "300000");- System.setProperty("solr.autoCommit.maxTime", "30000");- System.setProperty("solr.autoSoftCommit.maxTime", "3000");- // SOLR-13212 // TestInjection.nonGracefullClose = "true:60";- // SOLR-13189 // TestInjection.failReplicaRequests = "true:03";- }-- @AfterClass- public static void afterRestartWhileUpdatingTest() {- System.clearProperty("leaderVoteWait");- System.clearProperty("solr.autoCommit.maxTime");- System.clearProperty("solr.autoSoftCommit.maxTime");- }-- @Test- public void test() throws Exception {- handle.clear();- handle.put("timestamp", SKIPVAL);-- // start a couple indexing threads-- int[] maxDocList = new int[] {5000, 10000};-- int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];-- int numThreads = random().nextInt(4) + 1;-- threads = new ArrayList<>(numThreads);-- Thread expireThread =- new Thread("expireThread") {- @Override- public void run() {- while (!stopExpire) {- try {- Thread.sleep(random().nextInt(15000));- } catch (InterruptedException e) {- throw new RuntimeException(e);- }-- // try {- // chaosMonkey.expireRandomSession();- // } catch (KeeperException e) {- // throw new RuntimeException(e);- // } catch (InterruptedException e) {- // throw new RuntimeException(e);- // }- }- }- };-- // Currently unused- // expireThread.start();-- StoppableIndexingThread indexThread;- for (int i = 0; i < numThreads; i++) {- indexThread =- new StoppableIndexingThread(- controlClient, cloudClient, Integer.toString(i), true, maxDoc, 1, true);- threads.add(indexThread);- indexThread.start();- }-- Thread.sleep(2000);-- int restartTimes = 1; // random().nextInt(4) + 1;;- for (int i = 0; i < restartTimes; i++) {- Thread.sleep(random().nextInt(30000));- stopAndStartAllReplicas();- Thread.sleep(random().nextInt(30000));- }-- Thread.sleep(2000);-- // stop indexing threads- for (StoppableIndexingThread thread : threads) {- thread.safeStop();- }- stopExpire = true;- expireThread.join();-- Thread.sleep(1000);-- waitForThingsToLevelOut(320, TimeUnit.SECONDS);-- Thread.sleep(2000);-- waitForThingsToLevelOut(30, TimeUnit.SECONDS);-- Thread.sleep(5000);-- waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true);-- for (StoppableIndexingThread thread : threads) {- thread.join();- }-- checkShardConsistency(false, false);- }-- public void stopAndStartAllReplicas() throws Exception, InterruptedException {- chaosMonkey.stopAll(random().nextInt(1));-- if (random().nextBoolean()) {- for (StoppableIndexingThread thread : threads) {- thread.safeStop();- }- }- Thread.sleep(1000);-- chaosMonkey.startAll();- }-- @Override- protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException {- cloudClient.add(doc);- }-- @Override- public void distribTearDown() throws Exception {- // make sure threads have been stopped...- if (threads != null) {- for (StoppableIndexingThread thread : threads) {- thread.safeStop();- thread.safeStop();- }- }-- super.distribTearDown();- }-- // skip the randoms - they can deadlock...- @Override- protected void indexr(Object... fields) throws Exception {- SolrInputDocument doc = new SolrInputDocument();- addFields(doc, fields);- addFields(doc, "rnd_b", true);- indexDoc(doc);- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.javadeleted file mode 100644index c0c41f2eb22..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java+++ /dev/null@@ -1,300 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.util.ArrayList;-import java.util.Collection;-import java.util.HashSet;-import java.util.List;-import java.util.Set;-import java.util.concurrent.TimeUnit;-import java.util.concurrent.TimeoutException;-import java.util.stream.Collectors;-import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.SolrRequest;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.apache.HttpSolrClient;-import org.apache.solr.client.solrj.request.GenericSolrRequest;-import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.client.solrj.request.UpdateRequest;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.cloud.Replica;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.params.CollectionParams.CollectionAction;-import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.util.LogLevel;-import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--/** Test sync phase that occurs when Leader goes down and a new Leader is elected. */-@LogLevel("org.apache.solr.update.processor.DistributedZkUpdateProcessor=WARN")-public abstract class AbstractSyncSliceTestBase extends AbstractFullDistribZkTestBase {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- private boolean success = false;-- @Override- public void distribTearDown() throws Exception {- if (!success) {- printLayoutOnTearDown = true;- }- super.distribTearDown();- }-- public AbstractSyncSliceTestBase() {- super();- sliceCount = 1;- fixShardCount(TEST_NIGHTLY ? 7 : 4);- }-- @Test- public void test() throws Exception {-- handle.clear();- handle.put("timestamp", SKIPVAL);-- waitForThingsToLevelOut(30, TimeUnit.SECONDS);-- del("*:*");- List<CloudJettyRunner> skipServers = new ArrayList<>();- int docId = 0;- indexDoc(- skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");-- indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "old haven was blue.");-- skipServers.add(shardToJetty.get("shard1").get(1));-- indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "but the song was fancy.");-- skipServers.add(shardToJetty.get("shard1").get(2));-- indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "under the moon and over the lake");-- commit();-- waitForRecoveriesToFinish(false);-- // shard should be inconsistent- String shardFailMessage = checkShardConsistency("shard1", true, false);- assertNotNull(shardFailMessage);-- ModifiableSolrParams params = new ModifiableSolrParams();- params.set("action", CollectionAction.SYNCSHARD.toString());- params.set("collection", "collection1");- params.set("shard", "shard1");- final var request =- new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/collections", params);-- String baseUrl = shardToJetty.get(SHARD1).get(2).jetty.getBaseUrl().toString();-- // we only set the connect timeout, not so timeout- try (SolrClient baseClient =- new HttpSolrClient.Builder(baseUrl)- .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)- .build()) {- baseClient.request(request);- }-- waitForThingsToLevelOut(15, TimeUnit.SECONDS);-- checkShardConsistency(false, true);-- long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();- assertEquals(4, cloudClientDocs);-- // kill the leader - new leader could have all the docs or be missing one- CloudJettyRunner leaderJetty = shardToLeaderJetty.get("shard1");-- skipServers = getRandomOtherJetty(leaderJetty, null); // but not the leader-- // this doc won't be on one node- indexDoc(- skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");- commit();-- Set<CloudJettyRunner> jetties = new HashSet<>();- jetties.addAll(shardToJetty.get("shard1"));- jetties.remove(leaderJetty);- assertEquals(getShardCount() - 1, jetties.size());-- leaderJetty.jetty.stop();-- Thread.sleep(3000);-- waitForNoShardInconsistency();-- Thread.sleep(1000);-- checkShardConsistency(false, true);-- cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();- assertEquals(5, cloudClientDocs);-- CloudJettyRunner deadJetty = leaderJetty;-- // let's get the latest leader- while (deadJetty.equals(leaderJetty)) {- updateMappingsFromZk(this.jettys, this.clients);- leaderJetty = shardToLeaderJetty.get("shard1");- }-- // bring back dead node- deadJetty.jetty.start(); // he is not the leader anymore-- waitTillAllNodesActive();-- skipServers = getRandomOtherJetty(leaderJetty, deadJetty);- skipServers.addAll(getRandomOtherJetty(leaderJetty, deadJetty));- // skip list should be-- // System.out.println("leader:" + leaderJetty.url);- // System.out.println("dead:" + deadJetty.url);- // System.out.println("skip list:" + skipServers);-- // we are skipping 2 nodes- assertEquals(2, skipServers.size());-- // more docs than can peer sync- for (int i = 0; i < 300; i++) {- indexDoc(- skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");- }-- commit();- waitForRecoveriesToFinish(false);-- // shard should be inconsistent- shardFailMessage = waitTillInconsistent();- assertNotNull(- "Test Setup Failure: shard1 should have just been set up to be inconsistent - but it's still consistent. Leader:"- + leaderJetty.url- + " Dead Guy:"- + deadJetty.url- + "skip list:"- + skipServers,- shardFailMessage);-- // good place to test compareResults- boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient);- assertTrue("A test that compareResults is working correctly failed", shouldFail);-- jetties = new HashSet<>();- jetties.addAll(shardToJetty.get("shard1"));- jetties.remove(leaderJetty);- assertEquals(getShardCount() - 1, jetties.size());-- // kill the current leader- leaderJetty.jetty.stop();-- waitForNoShardInconsistency();-- checkShardConsistency(true, true);-- success = true;- }-- private void waitTillAllNodesActive() throws InterruptedException, TimeoutException {- ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);-- zkStateReader.waitForState(- "collection1",- 3,- TimeUnit.MINUTES,- (n, c) -> {- Collection<Replica> replicas = c.getSlice("shard1").getReplicas();- Set<String> nodes =- replicas.stream().map(Replica::getNodeName).collect(Collectors.toSet());- return replicas.stream().map(Replica::getState).allMatch(Replica.State.ACTIVE::equals)- && n.containsAll(nodes);- });- }-- private String waitTillInconsistent() throws Exception, InterruptedException {- String shardFailMessage = null;-- shardFailMessage = pollConsistency(shardFailMessage, 0);- shardFailMessage = pollConsistency(shardFailMessage, 3000);- shardFailMessage = pollConsistency(shardFailMessage, 5000);- shardFailMessage = pollConsistency(shardFailMessage, 15000);-- return shardFailMessage;- }-- private String pollConsistency(String shardFailMessage, int sleep)- throws InterruptedException, Exception {- try {- commit();- } catch (Throwable t) {- log.error("commit error", t);- }- if (shardFailMessage == null) {- // try again- Thread.sleep(sleep);- shardFailMessage = checkShardConsistency("shard1", true, false);- }- return shardFailMessage;- }-- private List<CloudJettyRunner> getRandomOtherJetty(- CloudJettyRunner leader, CloudJettyRunner down) {- List<CloudJettyRunner> skipServers = new ArrayList<>();- List<CloudJettyRunner> candidates = new ArrayList<>();- candidates.addAll(shardToJetty.get("shard1"));-- if (leader != null) {- candidates.remove(leader);- }-- if (down != null) {- candidates.remove(down);- }-- CloudJettyRunner cjetty = candidates.get(random().nextInt(candidates.size()));- skipServers.add(cjetty);- return skipServers;- }-- protected void indexDoc(List<CloudJettyRunner> skipServers, Object... fields)- throws IOException, SolrServerException {- SolrInputDocument doc = new SolrInputDocument();-- addFields(doc, fields);- addFields(doc, "rnd_b", true);-- controlClient.add(doc);-- UpdateRequest ureq = new UpdateRequest();- ureq.add(doc);- for (CloudJettyRunner skip : skipServers) {- ureq.getParams().add("test.distrib.skip.servers", skip.url + "/");- }- ureq.process(cloudClient);- }-- // skip the randoms - they can deadlock...- @Override- protected void indexr(Object... fields) throws Exception {- SolrInputDocument doc = new SolrInputDocument();- addFields(doc, fields);- addFields(doc, "rnd_b", true);- indexDoc(doc);- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.javadeleted file mode 100644index ee73ea8e817..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java+++ /dev/null@@ -1,146 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import java.io.IOException;-import java.util.ArrayList;-import java.util.List;-import java.util.concurrent.TimeUnit;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.embedded.JettySolrRunner;-import org.apache.solr.util.TestInjection;-import org.junit.AfterClass;-import org.junit.BeforeClass;-import org.junit.Test;--public abstract class AbstractTlogReplayBufferedWhileIndexingTestBase- extends AbstractFullDistribZkTestBase {-- private List<StoppableIndexingThread> threads;-- public AbstractTlogReplayBufferedWhileIndexingTestBase() throws Exception {- super();- sliceCount = 1;- fixShardCount(2);- schemaString = "schema15.xml"; // we need a string id- }-- @BeforeClass- public static void beforeRestartWhileUpdatingTest() throws Exception {- System.setProperty("leaderVoteWait", "300000");- System.setProperty("solr.autoCommit.maxTime", "10000");- System.setProperty("solr.autoSoftCommit.maxTime", "3000");- TestInjection.updateLogReplayRandomPause = "true:10";- TestInjection.updateRandomPause = "true:10";- useFactory("solr.StandardDirectoryFactory");- }-- @AfterClass- public static void afterRestartWhileUpdatingTest() {- System.clearProperty("leaderVoteWait");- System.clearProperty("solr.autoCommit.maxTime");- System.clearProperty("solr.autoSoftCommit.maxTime");- }-- @Test- public void test() throws Exception {- handle.clear();- handle.put("timestamp", SKIPVAL);-- waitForRecoveriesToFinish(false);-- int numThreads = 3;-- threads = new ArrayList<>(numThreads);-- ArrayList<JettySolrRunner> allJetty = new ArrayList<>();- allJetty.addAll(jettys);- allJetty.remove(shardToLeaderJetty.get("shard1").jetty);- assert allJetty.size() == 1 : allJetty.size();- allJetty.get(0).stop();-- StoppableIndexingThread indexThread;- for (int i = 0; i < numThreads; i++) {- boolean pauseBetweenUpdates = random().nextBoolean();- int batchSize = random().nextInt(4) + 1;- indexThread =- new StoppableIndexingThread(- controlClient,- cloudClient,- Integer.toString(i),- true,- 900,- batchSize,- pauseBetweenUpdates);- threads.add(indexThread);- indexThread.start();- }-- Thread.sleep(2000);-- allJetty.get(0).start();-- Thread.sleep(45000);-- waitForThingsToLevelOut(); // we can insert random update delays, so this can take a while,- // especially when beasting this test-- Thread.sleep(2000);-- waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true);-- for (StoppableIndexingThread thread : threads) {- thread.safeStop();- }-- waitForThingsToLevelOut(30, TimeUnit.SECONDS);-- checkShardConsistency(false, false);- }-- @Override- protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException {- cloudClient.add(doc);- }-- @Override- public void distribTearDown() throws Exception {- // make sure threads have been stopped...- if (threads != null) {- for (StoppableIndexingThread thread : threads) {- thread.safeStop();- }-- for (StoppableIndexingThread thread : threads) {- thread.join();- }- }-- super.distribTearDown();- }-- // skip the randoms - they can deadlock...- @Override- protected void indexr(Object... fields) throws Exception {- SolrInputDocument doc = new SolrInputDocument();- addFields(doc, fields);- addFields(doc, "rnd_b", true);- indexDoc(doc);- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.javadeleted file mode 100644index 6477af7aac6..00000000000--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java+++ /dev/null@@ -1,463 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.nio.file.Path;-import java.util.Collection;-import java.util.Collections;-import java.util.Random;-import java.util.Set;-import java.util.concurrent.SynchronousQueue;-import java.util.concurrent.ThreadPoolExecutor;-import java.util.concurrent.TimeUnit;-import java.util.concurrent.TimeoutException;-import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.apache.HttpSolrClient;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;-import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.cloud.Replica;-import org.apache.solr.common.cloud.Slice;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.util.ExecutorUtil;-import org.apache.solr.common.util.SolrNamedThreadFactory;-import org.apache.solr.core.SolrCore;-import org.apache.solr.core.SolrPaths;-import org.apache.solr.embedded.JettySolrRunner;-import org.apache.solr.util.TestInjection;-import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--/**- * This test simply does a bunch of basic things in solrcloud mode and asserts things work as- * expected.- */-public abstract class AbstractUnloadDistributedZkTestBase extends AbstractFullDistribZkTestBase {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- public AbstractUnloadDistributedZkTestBase() {- super();- fixShardCount(4); // needs at least 4 servers- }-- @Override- protected String getSolrXml() {- return "solr.xml";- }-- @Test- public void test() throws Exception {- jettys.forEach(- j -> {- Set<Path> allowPath = j.getCoreContainer().getAllowPaths();- allowPath.clear();- allowPath.add(SolrPaths.ALL_PATH); // Allow non-standard core instance path- });- log.info("###Starting testCoreUnloadAndLeaders");- testCoreUnloadAndLeaders(); // long- log.info("###Starting testUnloadOfCores");- testUnloadLotsOfCores(); // long- log.info("###Starting testUnloadShardAndCollection");- testUnloadShardAndCollection();- }-- /**- * @param url a Solr node base URL. Should <em>not</em> contain a core or collection name.- */- private SolrClient newSolrClient(String url) {- return new HttpSolrClient.Builder(url)- .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)- .withSocketTimeout(30000, TimeUnit.MILLISECONDS)- .build();- }-- private void checkCoreNamePresenceAndSliceCount(- String collectionName, String coreName, boolean shouldBePresent, int expectedSliceCount)- throws Exception {- ZkStateReader reader = ZkStateReader.from(cloudClient);- try {- reader.waitForState(- collectionName,- 45,- TimeUnit.SECONDS,- c -> {- final Collection<Slice> slices = (c != null) ? c.getSlices() : Collections.emptyList();- if (expectedSliceCount == slices.size()) {- for (Slice slice : slices) {- for (Replica replica : slice.getReplicas()) {- if (coreName.equals(replica.get("core"))) {- return shouldBePresent;- }- }- }- return !shouldBePresent;- } else {- return false;- }- });- } catch (TimeoutException e) {- printLayout();- fail(- "checkCoreNamePresenceAndSliceCount failed:"- + " collection="- + collectionName- + " CoreName="- + coreName- + " shouldBePresent="- + shouldBePresent);- }- }-- private void testUnloadShardAndCollection() throws Exception {- final int numShards = 2;-- final String collection = "test_unload_shard_and_collection";-- final String coreName1 = collection + "_1";- final String coreName2 = collection + "_2";-- assertEquals(- 0,- CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1)- .setCreateNodeSet("")- .process(cloudClient)- .getStatus());- assertTrue(- CollectionAdminRequest.addReplicaToShard(collection, "shard1")- .setCoreName(coreName1)- .setNode(jettys.get(0).getNodeName())- .process(cloudClient)- .isSuccess());-- assertTrue(- CollectionAdminRequest.addReplicaToShard(collection, "shard2")- .setCoreName(coreName2)- .setNode(jettys.get(0).getNodeName())- .process(cloudClient)- .isSuccess());-- // does not mean they are active and up yet :*- waitForRecoveriesToFinish(collection, false);-- final boolean unloadInOrder = random().nextBoolean();- final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2);- final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1);-- try (SolrClient adminClient = getHttpSolrClient(buildUrl(jettys.get(0).getLocalPort()))) {- // now unload one of the two- Unload unloadCmd = new Unload(false);- unloadCmd.setCoreName(unloadCmdCoreName1);- adminClient.request(unloadCmd);-- // there should still be two shards (as of SOLR-5209)- checkCoreNamePresenceAndSliceCount(- collection,- unloadCmdCoreName1,- false /* shouldBePresent */,- numShards /* expectedSliceCount */);-- // now unload one of the other- unloadCmd = new Unload(false);- unloadCmd.setCoreName(unloadCmdCoreName2);- adminClient.request(unloadCmd);- checkCoreNamePresenceAndSliceCount(- collection,- unloadCmdCoreName2,- false /* shouldBePresent */,- numShards /* expectedSliceCount */);- }-- // printLayout();- // the collection should still be present (as of SOLR-5209 replica removal does not cascade to- // remove the slice and collection)- getCommonCloudSolrClient();- assertTrue(- "No longer found collection " + collection,- cloudClient.getClusterState().hasCollection(collection));- }-- protected SolrCore getFirstCore(String collection, JettySolrRunner jetty) {- SolrCore solrCore = null;- for (SolrCore core : jetty.getCoreContainer().getCores()) {- if (core.getName().startsWith(collection)) {- solrCore = core;- }- }- return solrCore;- }-- /**- * @throws Exception on any problem- */- private void testCoreUnloadAndLeaders() throws Exception {- JettySolrRunner jetty1 = jettys.get(0);-- assertEquals(- 0,- CollectionAdminRequest.createCollection("unloadcollection", "conf1", 1, 1)- .setCreateNodeSet(jetty1.getNodeName())- .process(cloudClient)- .getStatus());- getCommonCloudSolrClient();- ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);-- zkStateReader.forceUpdateCollection("unloadcollection");-- int slices =- zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size();- assertEquals(1, slices);- SolrCore solrCore = getFirstCore("unloadcollection", jetty1);- String core1DataDir = solrCore.getDataDir();-- assertTrue(- CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")- .setCoreName("unloadcollection_shard1_replica2")- .setNode(jettys.get(1).getNodeName())- .process(cloudClient)- .isSuccess());- zkStateReader.forceUpdateCollection("unloadcollection");- slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size();- assertEquals(1, slices);-- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);-- Replica leader = getLeaderFromZk("unloadcollection", "shard1");-- Random random = random();- if (random.nextBoolean()) {- try (SolrClient collectionClient =- getHttpSolrClient(leader.getBaseUrl(), leader.getCoreName())) {- // lets try and use the solrj client to index and retrieve a couple- // documents- SolrInputDocument doc1 =- getDoc(id, 6, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");- SolrInputDocument doc2 =- getDoc(id, 7, i1, -600, tlong, 600, t1, "humpty dumpy3 sat on a walls");- SolrInputDocument doc3 =- getDoc(id, 8, i1, -600, tlong, 600, t1, "humpty dumpy2 sat on a walled");- collectionClient.add(doc1);- collectionClient.add(doc2);- collectionClient.add(doc3);- collectionClient.commit();- }- }-- assertTrue(- CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")- .setCoreName("unloadcollection_shard1_replica3")- .setNode(jettys.get(2).getNodeName())- .process(cloudClient)- .isSuccess());-- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);-- // so that we start with some versions when we reload...- TestInjection.skipIndexWriterCommitOnClose = true;-- try (SolrClient addClient =- new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString())- .withDefaultCollection("unloadcollection_shard1_replica3")- .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)- .build()) {-- // add a few docs- for (int x = 20; x < 100; x++) {- SolrInputDocument doc1 =- getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");- addClient.add(doc1);- }- }- // don't commit so they remain in the tran log- // collectionClient.commit();-- // unload the leader- try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) {-- Unload unloadCmd = new Unload(false);- unloadCmd.setCoreName(leader.getCoreName());- ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams();-- collectionClient.request(unloadCmd);- }- // Thread.currentThread().sleep(500);- // printLayout();-- int tries = 50;- while (leader- .getCoreUrl()- .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {- Thread.sleep(100);- if (tries-- == 0) {- fail("Leader never changed");- }- }-- // ensure there is a leader- zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);-- try (SolrClient addClient =- new HttpSolrClient.Builder(jettys.get(1).getBaseUrl().toString())- .withDefaultCollection("unloadcollection_shard1_replica2")- .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)- .withSocketTimeout(90000, TimeUnit.MILLISECONDS)- .build()) {-- // add a few docs while the leader is down- for (int x = 101; x < 200; x++) {- SolrInputDocument doc1 =- getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");- addClient.add(doc1);- }- }-- assertTrue(- CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")- .setCoreName("unloadcollection_shard1_replica4")- .setNode(jettys.get(3).getNodeName())- .process(cloudClient)- .isSuccess());-- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);-- // unload the leader again- leader = getLeaderFromZk("unloadcollection", "shard1");- try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) {-- Unload unloadCmd = new Unload(false);- unloadCmd.setCoreName(leader.getCoreName());- collectionClient.request(unloadCmd);- }- tries = 50;- while (leader- .getCoreUrl()- .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {- Thread.sleep(100);- if (tries-- == 0) {- fail("Leader never changed");- }- }-- zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);-- TestInjection.skipIndexWriterCommitOnClose = false; // set this back- assertTrue(- CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")- .setCoreName(leader.getCoreName())- .setDataDir(core1DataDir)- .setNode(leader.getNodeName())- .process(cloudClient)- .isSuccess());-- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);-- long found1, found3;-- try (SolrClient adminClient =- newSolrClient((jettys.get(1).getBaseUrl() + "/unloadcollection_shard1_replica2"))) {- adminClient.commit();- SolrQuery q = new SolrQuery("*:*");- q.set("distrib", false);- found1 = adminClient.query(q).getResults().getNumFound();- }-- try (SolrClient adminClient =- new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString())- .withDefaultCollection("unloadcollection_shard1_replica3")- .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)- .withSocketTimeout(30000, TimeUnit.MILLISECONDS)- .build()) {- adminClient.commit();- SolrQuery q = new SolrQuery("*:*");- q.set("distrib", false);- found3 = adminClient.query(q).getResults().getNumFound();- }-- try (SolrClient adminClient =- newSolrClient((jettys.get(3).getBaseUrl() + "/unloadcollection_shard1_replica4"))) {- adminClient.commit();- SolrQuery q = new SolrQuery("*:*");- q.set("distrib", false);- long found4 = adminClient.query(q).getResults().getNumFound();-- // all 3 shards should now have the same number of docs- assertEquals(found1, found3);- assertEquals(found3, found4);- }- }-- private void testUnloadLotsOfCores() throws Exception {- JettySolrRunner jetty = jettys.get(0);- int shards = TEST_NIGHTLY ? 2 : 1;- try (final SolrClient adminClient = jetty.newClient(15000, 60000)) {- int numReplicas = atLeast(3);- ThreadPoolExecutor executor =- new ExecutorUtil.MDCAwareThreadPoolExecutor(- 0,- Integer.MAX_VALUE,- 5,- TimeUnit.SECONDS,- new SynchronousQueue<>(),- new SolrNamedThreadFactory("testExecutor"));- try {- // create the cores- AbstractBasicDistributedZkTestBase.createCollectionInOneInstance(- adminClient, jetty.getNodeName(), executor, "multiunload", shards, numReplicas);- } finally {- ExecutorUtil.shutdownAndAwaitTermination(executor);- }-- if (TEST_NIGHTLY == false) {- // with nightly tests, we can try doing the unloads before the creates are done- // it still works, but takes much longer since we end up waiting for a timeout- waitForRecoveriesToFinish("multiunload", false);- }-- executor =- new ExecutorUtil.MDCAwareThreadPoolExecutor(- 0,- Integer.MAX_VALUE,- 5,- TimeUnit.SECONDS,- new SynchronousQueue<>(),- new SolrNamedThreadFactory("testExecutor"));- try {- for (int j = 0; j < numReplicas; j++) {- final int freezeJ = j;- executor.execute(- () -> {- Unload unloadCmd = new Unload(true);- unloadCmd.setCoreName("multiunload" + freezeJ);- try {- adminClient.request(unloadCmd);- } catch (SolrServerException | IOException e) {- throw new RuntimeException(e);- }- });- Thread.sleep(random().nextInt(50));- }- } finally {- ExecutorUtil.shutdownAndAwaitTermination(executor);- }- }- }-}diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.javaindex 79dc6bde071..5dc62e94ed2 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java@@ -47,7 +47,6 @@ import org.apache.solr.common.cloud.ImplicitDocRouter;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.params.CoreAdminParams;import org.junit.AfterClass;import org.junit.BeforeClass;@@ -173,7 +172,6 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa}testBackupAndRestore(getCollectionName());- testConfigBackupOnly("conf1", getCollectionName());testInvalidPath(getCollectionName());}@@ -230,22 +228,6 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa}}- /**- * This test validates the backup of collection configuration using {@linkplain- * CollectionAdminParams#NO_INDEX_BACKUP_STRATEGY}.- *- * @param configName The config name for the collection to be backed up.- * @param collectionName The name of the collection to be backed up.- * @throws Exception in case of errors.- */- protected void testConfigBackupOnly(String configName, String collectionName) throws Exception {- // This is deliberately no-op since we want to run this test only for one of the backup- // repository- // implementation (mainly to avoid redundant test execution). Currently HDFS backup repository- // test- // implements this.- }-// This test verifies the system behavior when the backup location cluster property is configured// with an invalid// value for the specified repository (and the default backup location is not configured in