Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

Continue cleanups post removing HDFS from Solr (#3913)

Eric Pugh98cf180ba24b
diff --git a/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java
index fdd48350f7a..c1ef93b790e 100644
--- a/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java
@@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;
public class BlockDirectory extends FilterDirectory implements ShutdownAwareDirectory {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final long BLOCK_SHIFT = Integer.getInteger("solr.hdfs.blockcache.blockshift", 13);
+ public static final long BLOCK_SHIFT = Integer.getInteger("solr.blockcache.blockshift", 13);
public static final int BLOCK_SIZE = 1 << BLOCK_SHIFT;
public static final long BLOCK_MOD = BLOCK_SIZE - 1L;
diff --git a/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java b/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java
index 79d44d0fc6b..df3f1a058b2 100644
--- a/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java
@@ -28,7 +28,7 @@ import org.apache.lucene.store.IndexOutput;
public abstract class CustomBufferedIndexInput extends IndexInput {
public static final int BUFFER_SIZE =
- Integer.getInteger("solr.hdfs.readbuffer.size.default", 32768);
+ Integer.getInteger("solr.blockcache.readbuffer.size.default", 32768);
private int bufferSize = BUFFER_SIZE;
diff --git a/solr/core/src/java/org/apache/solr/blockcache/Metrics.java b/solr/core/src/java/org/apache/solr/blockcache/Metrics.java
index ed820fbb500..4070cd557c2 100644
--- a/solr/core/src/java/org/apache/solr/blockcache/Metrics.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/Metrics.java
@@ -157,12 +157,12 @@ public class Metrics extends SolrCacheBase implements SolrInfoBean {
@Override
public String getName() {
- return "hdfsBlockCache";
+ return "blockCache";
}
@Override
public String getDescription() {
- return "Provides metrics for the HdfsDirectoryFactory BlockCache.";
+ return "Provides metrics for the BlockCache.";
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
index 48f065f537e..9bcae03e812 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
@@ -154,8 +154,8 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {
replica.getBool(ZkStateReader.SHARED_STORAGE_PROP, false) && dataDir != null;
if (isSharedFS && inPlaceMove) {
- log.debug("-- moveHdfsReplica");
- moveHdfsReplica(
+ log.debug("-- moveSharedFsReplica");
+ moveSharedFsReplica(
clusterState,
results,
dataDir.toString(),
@@ -179,7 +179,7 @@ public class MoveReplicaCmd implements CollApiCmds.CollectionApiCommand {
}
}
- private void moveHdfsReplica(
+ private void moveSharedFsReplica(
ClusterState clusterState,
NamedList<Object> results,
String dataDir,
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
index 464c47cc53e..8ecd634b757 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@ -446,6 +446,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
// `init(UpdateHandler, SolrCore` is never actually called concurrently in application code
// (`TestHdfsUpdateLog.testFSThreadSafety()`, introduced by SOLR-7113, seems to be the only
// place that requires true thread safety from this method?).
+ // HDFS was removed in Solr 10, and therefore the test referenced is gone as well.
if (debug) {
log.debug(
"UpdateHandler init: tlogDir={}, next id={} this is a reopen or double init ... nothing else to do.",
@@ -509,8 +510,8 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
/**
* Resolves any relative path wrt the highest core-scoped level (whatever that means for a
- * particular implementation). For most filesystems, this will be the core instanceDir, but there
- * are other cases; e.g., HdfsUpdateLog will resolve paths relative to the core dataDir.
+ * particular implementation). For most filesystems, this will be the core instanceDir, but that
+ * is not a hard and fast rule.
*
* <p>If the input path is already absolute, it will be returned unmodified.
*
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
index df0c8f98a2e..34532421bf5 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
@@ -1378,7 +1378,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
// Streaming updates can delay shutdown and cause big update reorderings (new streams can't be
// initiated, but existing streams carry on). This is why we check if the CC is shutdown.
- // See SOLR-8203 and loop HdfsChaosMonkeyNothingIsSafeTest (and check for inconsistent shards)
+ // See SOLR-8203 and loop ChaosMonkeyNothingIsSafeTest (and check for inconsistent shards)
// to test.
if (req.getCoreContainer().isShutDown()) {
throw new SolrException(
diff --git a/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java b/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java
index dbd69bca4d9..7a1a0c16fa8 100644
--- a/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java
+++ b/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java
@@ -142,10 +142,10 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {
String name = "test.eof";
createFile(name, fsDir, directory);
long fsLength = fsDir.fileLength(name);
- long hdfsLength = directory.fileLength(name);
- assertEquals(fsLength, hdfsLength);
+ long blockLength = directory.fileLength(name);
+ assertEquals(fsLength, blockLength);
testEof(name, fsDir, fsLength);
- testEof(name, directory, hdfsLength);
+ testEof(name, directory, blockLength);
fsDir.close();
}
@@ -189,11 +189,12 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {
testRandomAccessWrites();
}
- private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) throws IOException {
+ private void assertInputsEquals(String name, Directory fsDir, Directory blockDirectory)
+ throws IOException {
int reads = random.nextInt(MAX_NUMBER_OF_READS);
IndexInput fsInput = fsDir.openInput(name, IOContext.DEFAULT);
- IndexInput hdfsInput = hdfs.openInput(name, IOContext.DEFAULT);
- assertEquals(fsInput.length(), hdfsInput.length());
+ IndexInput blockInput = blockDirectory.openInput(name, IOContext.DEFAULT);
+ assertEquals(fsInput.length(), blockInput.length());
int fileLength = (int) fsInput.length();
for (int i = 0; i < reads; i++) {
int rnd;
@@ -204,7 +205,7 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {
}
byte[] fsBuf = new byte[rnd + MIN_BUFFER_SIZE];
- byte[] hdfsBuf = new byte[fsBuf.length];
+ byte[] blockBuf = new byte[fsBuf.length];
int offset = random.nextInt(fsBuf.length);
int length = random.nextInt(fsBuf.length - offset);
@@ -217,23 +218,24 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {
fsInput.seek(pos);
fsInput.readBytes(fsBuf, offset, length);
- hdfsInput.seek(pos);
- hdfsInput.readBytes(hdfsBuf, offset, length);
+ blockInput.seek(pos);
+ blockInput.readBytes(blockBuf, offset, length);
for (int f = offset; f < length; f++) {
- if (fsBuf[f] != hdfsBuf[f]) {
+ if (fsBuf[f] != blockBuf[f]) {
fail("read [" + i + "]");
}
}
}
fsInput.close();
- hdfsInput.close();
+ blockInput.close();
}
- private void createFile(String name, Directory fsDir, Directory hdfs) throws IOException {
+ private void createFile(String name, Directory fsDir, Directory blockDirectory)
+ throws IOException {
int writes = random.nextInt(MAX_NUMBER_OF_WRITES);
int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE;
IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT);
- IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT);
+ IndexOutput blockOutput = blockDirectory.createOutput(name, IOContext.DEFAULT);
for (int i = 0; i < writes; i++) {
byte[] buf =
new byte
@@ -243,10 +245,10 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {
int offset = random.nextInt(buf.length);
int length = random.nextInt(buf.length - offset);
fsOutput.writeBytes(buf, offset, length);
- hdfsOutput.writeBytes(buf, offset, length);
+ blockOutput.writeBytes(buf, offset, length);
}
fsOutput.close();
- hdfsOutput.close();
+ blockOutput.close();
}
private String getName() {
diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
index b99330645f1..d4ad2628c32 100644
--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
@@ -16,21 +16,474 @@
*/
package org.apache.solr.cloud;
+import static org.apache.solr.cloud.SolrCloudTestCase.configurePrsDefault;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.tests.mockfile.FilterPath;
+import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.handler.BackupStatusChecker;
+import org.apache.solr.handler.ReplicationHandler;
+import org.junit.BeforeClass;
import org.junit.Test;
/**
* This test simply does a bunch of basic things in solrcloud mode and asserts things work as
- * expected. Implementation moved to AbstractBasicDistributedZk2TestBase as it is used by HDFS
- * contrib module tests.
+ * expected.
*/
+// Backups do checksum validation against a footer value not present in 'SimpleText'
[email protected]({"SimpleText"})
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
-public class BasicDistributedZk2Test extends AbstractBasicDistributedZk2TestBase {
+public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
+ private static final String SHARD2 = "shard2";
+ private static final String SHARD1 = "shard1";
+ private static final String ONE_NODE_COLLECTION = "onenodecollection";
+ private final boolean onlyLeaderIndexes = random().nextBoolean();
+
+ public BasicDistributedZk2Test() {
+ super();
+ // we need DVs on point fields to compute stats & facets
+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP))
+ System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");
+
+ sliceCount = 2;
+ }
- @Test
@Override
+ protected boolean useTlogReplicas() {
+ return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug
+ // and it's TestInjection use
+ }
+
+ @BeforeClass
+ public static void _setPrsDefault() {
+ configurePrsDefault();
+ }
+
+ @Test
@ShardsFixed(num = 4)
public void test() throws Exception {
- super.test();
+ boolean testFinished = false;
+ try {
+ handle.clear();
+ handle.put("timestamp", SKIPVAL);
+
+ testNodeWithoutCollectionForwarding();
+
+ indexr(
+ id,
+ 1,
+ i1,
+ 100,
+ tlong,
+ 100,
+ t1,
+ "now is the time for all good men",
+ "foo_f",
+ 1.414f,
+ "foo_b",
+ "true",
+ "foo_d",
+ 1.414d);
+
+ commit();
+
+ // make sure we are in a steady state...
+ waitForRecoveriesToFinish(false);
+
+ assertDocCounts(false);
+
+ indexAbunchOfDocs();
+
+ // check again
+ waitForRecoveriesToFinish(false);
+
+ commit();
+
+ assertDocCounts(VERBOSE);
+ checkQueries();
+
+ assertDocCounts(VERBOSE);
+
+ query("q", "*:*", "sort", "n_tl1 desc");
+
+ bringDownShardIndexSomeDocsAndRecover();
+
+ query("q", "*:*", "sort", "n_tl1 desc");
+
+ // test adding another replica to a shard - it should do a
+ // recovery/replication to pick up the index from the leader
+ addNewReplica();
+
+ long docId = testUpdateAndDelete();
+
+ // index a bad doc...
+ expectThrows(SolrException.class, () -> indexr(t1, "a doc with no id"));
+
+ // TODO: bring this to its own method?
+ // try indexing to a leader that has no replicas up
+ ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
+ ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, SHARD2);
+
+ String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP);
+ chaosMonkey.stopShardExcept(SHARD2, nodeName);
+
+ SolrClient client = getClient(nodeName);
+
+ index_specific(client, "id", docId + 1, t1, "what happens here?");
+
+ // expire a session...
+ CloudJettyRunner cloudJetty = shardToJetty.get(SHARD1).get(0);
+ chaosMonkey.expireSession(cloudJetty.jetty);
+ // Wait until the jetty is reconnected, otherwise the following index command could fail
+ cloudJetty
+ .jetty
+ .getCoreContainer()
+ .getZkController()
+ .getZkClient()
+ .getCuratorFramework()
+ .blockUntilConnected(50, TimeUnit.MILLISECONDS);
+
+ indexr("id", docId + 1, t1, "slip this doc in");
+
+ waitForRecoveriesToFinish(false);
+
+ checkShardConsistency(SHARD1);
+ checkShardConsistency(SHARD2);
+
+ testFinished = true;
+ } finally {
+ if (!testFinished) {
+ printLayoutOnTearDown = true;
+ }
+ }
+ }
+
+ private void testNodeWithoutCollectionForwarding() throws Exception {
+ assertEquals(
+ 0,
+ CollectionAdminRequest.createCollection(ONE_NODE_COLLECTION, "conf1", 1, 1)
+ .setCreateNodeSet("")
+ .process(cloudClient)
+ .getStatus());
+ assertTrue(
+ CollectionAdminRequest.addReplicaToShard(ONE_NODE_COLLECTION, "shard1")
+ .setCoreName(ONE_NODE_COLLECTION + "core")
+ .process(cloudClient)
+ .isSuccess());
+
+ waitForCollection(ZkStateReader.from(cloudClient), ONE_NODE_COLLECTION, 1);
+ waitForRecoveriesToFinish(ONE_NODE_COLLECTION, ZkStateReader.from(cloudClient), false);
+
+ ZkStateReader.from(cloudClient).getLeaderRetry(ONE_NODE_COLLECTION, SHARD1, 30000);
+
+ int docs = 2;
+ for (JettySolrRunner jetty : jettys) {
+ final String clientUrl = getBaseUrl(jetty);
+ addAndQueryDocs(clientUrl, docs);
+ docs += 2;
+ }
+ }
+
+ // 2 docs added every call
+ private void addAndQueryDocs(final String baseUrl, int docs) throws Exception {
+
+ SolrQuery query = new SolrQuery("*:*");
+
+ try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollection")) {
+ // add a doc
+ client.add(sdoc("id", docs));
+ client.commit();
+
+ QueryResponse results = client.query(query);
+ assertEquals(docs - 1, results.getResults().getNumFound());
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", docs + 1);
+ client.add(doc);
+ client.commit();
+
+ query = new SolrQuery("*:*");
+ query.set("rows", 0);
+ results = client.query(query);
+ assertEquals(docs, results.getResults().getNumFound());
+ }
+ }
+
+ private long testUpdateAndDelete() throws Exception {
+ long docId = 99999999L;
+ indexr("id", docId, t1, "originalcontent");
+
+ commit();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("q", t1 + ":originalcontent");
+ QueryResponse results = clients.get(0).query(params);
+ assertEquals(1, results.getResults().getNumFound());
+
+ // update doc
+ indexr("id", docId, t1, "updatedcontent");
+
+ commit();
+
+ results = clients.get(0).query(params);
+ assertEquals(0, results.getResults().getNumFound());
+
+ params.set("q", t1 + ":updatedcontent");
+
+ results = clients.get(0).query(params);
+ assertEquals(1, results.getResults().getNumFound());
+
+ UpdateRequest uReq = new UpdateRequest();
+ // uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN);
+ uReq.deleteById(Long.toString(docId)).process(clients.get(0));
+
+ commit();
+
+ results = clients.get(0).query(params);
+ assertEquals(0, results.getResults().getNumFound());
+ return docId;
+ }
+
+ private void bringDownShardIndexSomeDocsAndRecover() throws Exception {
+ SolrQuery query = new SolrQuery("*:*");
+ query.set("distrib", false);
+
+ commit();
+
+ long deadShardCount =
+ shardToJetty.get(SHARD2).get(0).client.solrClient.query(query).getResults().getNumFound();
+
+ query("q", "*:*", "sort", "n_tl1 desc");
+
+ int oldLiveNodes =
+ ZkStateReader.from(cloudClient)
+ .getZkClient()
+ .getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true)
+ .size();
+
+ assertEquals(5, oldLiveNodes);
+
+ // kill a shard
+ CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0);
+
+ // ensure shard is dead
+ expectThrows(
+ SolrServerException.class,
+ "This server should be down and this update should have failed",
+ () -> index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1, "specific doc!"));
+
+ commit();
+
+ query("q", "*:*", "sort", "n_tl1 desc");
+
+ // long cloudClientDocs = cloudClient.query(new
+ // SolrQuery("*:*")).getResults().getNumFound();
+ // System.out.println("clouddocs:" + cloudClientDocs);
+
+ // try to index to a living shard at shard2
+
+ long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
+
+ ZkStateReader.from(cloudClient).getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 60000);
+
+ try {
+ index_specific(
+ shardToJetty.get(SHARD1).get(1).client.solrClient,
+ id,
+ 1000,
+ i1,
+ 108,
+ t1,
+ "specific doc!");
+ } catch (Exception e) {
+ // wait and try again
+ Thread.sleep(4000);
+ index_specific(
+ shardToJetty.get(SHARD1).get(1).client.solrClient,
+ id,
+ 1000,
+ i1,
+ 108,
+ t1,
+ "specific doc!");
+ }
+
+ commit();
+
+ checkShardConsistency(true, false);
+
+ query("q", "*:*", "sort", "n_tl1 desc");
+
+ long numFound2 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
+
+ assertEquals(numFound1 + 1, numFound2);
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", 1001);
+
+ controlClient.add(doc);
+
+ // try adding a doc with CloudSolrServer
+ UpdateRequest ureq = new UpdateRequest();
+ ureq.add(doc);
+ // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);
+
+ try {
+ ureq.process(cloudClient);
+ } catch (SolrServerException e) {
+ // try again
+ Thread.sleep(3500);
+ ureq.process(cloudClient);
+ }
+
+ commit();
+
+ query("q", "*:*", "sort", "n_tl1 desc");
+
+ long numFound3 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
+
+ // lets just check that the one doc since last commit made it in...
+ assertEquals(numFound2 + 1, numFound3);
+
+ // test debugging
+ testDebugQueries();
+
+ if (VERBOSE) {
+ System.err.println(controlClient.query(new SolrQuery("*:*")).getResults().getNumFound());
+
+ for (SolrClient client : clients) {
+ try {
+ SolrQuery q = new SolrQuery("*:*");
+ q.set("distrib", false);
+ System.err.println(client.query(q).getResults().getNumFound());
+ } catch (Exception e) {
+
+ }
+ }
+ }
+ // TODO: This test currently fails because debug info is obtained only
+ // on shards with matches.
+ // query("q","matchesnothing","fl","*,score", "debugQuery", "true");
+
+ // this should trigger a recovery phase on deadShard
+ deadShard.jetty.start();
+
+ // make sure we have published we are recovering
+ Thread.sleep(1500);
+
+ waitForRecoveriesToFinish(false);
+
+ deadShardCount =
+ shardToJetty.get(SHARD1).get(0).client.solrClient.query(query).getResults().getNumFound();
+ // if we properly recovered, we should now have the couple missing docs that
+ // came in while shard was down
+ checkShardConsistency(true, false);
+
+ // recover over 100 docs so we do more than just peer sync (replicate recovery)
+ chaosMonkey.stopJetty(deadShard);
+
+ for (int i = 0; i < 226; i++) {
+ doc = new SolrInputDocument();
+ doc.addField("id", 2000 + i);
+ controlClient.add(doc);
+ ureq = new UpdateRequest();
+ ureq.add(doc);
+ // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);
+ ureq.process(cloudClient);
+ }
+ commit();
+
+ Thread.sleep(1500);
+
+ deadShard.jetty.start();
+
+ // make sure we have published we are recovering
+ Thread.sleep(1500);
+
+ waitForThingsToLevelOut(1, TimeUnit.MINUTES);
+
+ Thread.sleep(500);
+
+ waitForRecoveriesToFinish(false);
+
+ checkShardConsistency(true, false);
+
+ // try a backup command
+ try (final SolrClient client =
+ getHttpSolrClient((String) shardToJetty.get(SHARD2).get(0).info.get("base_url"))) {
+ final String backupName = "the_backup";
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("qt", ReplicationHandler.PATH);
+ params.set("command", "backup");
+ params.set("name", backupName);
+ final Path location = FilterPath.unwrap(createTempDir()).toRealPath();
+ // Allow non-standard location outside SOLR_HOME
+ jettys.forEach(j -> j.getCoreContainer().getAllowPaths().add(location));
+ params.set("location", location.toString());
+
+ QueryRequest request = new QueryRequest(params);
+ client.request(request, DEFAULT_TEST_COLLECTION_NAME);
+
+ final BackupStatusChecker backupStatus =
+ new BackupStatusChecker(client, "/" + DEFAULT_TEST_COLLECTION_NAME + "/replication");
+ final String backupDirName = backupStatus.waitForBackupSuccess(backupName, 30);
+ assertTrue(
+ "Backup dir does not exist: " + backupDirName,
+ Files.exists(location.resolve(backupDirName)));
+ }
+ }
+
+ private void addNewReplica() throws Exception {
+
+ waitForRecoveriesToFinish(false);
+
+ // new server should be part of first shard
+ // how many docs are on the new shard?
+ for (CloudJettyRunner cjetty : shardToJetty.get(SHARD1)) {
+ if (VERBOSE)
+ System.err.println(
+ "shard1 total:"
+ + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());
+ }
+ for (CloudJettyRunner cjetty : shardToJetty.get(SHARD2)) {
+ if (VERBOSE)
+ System.err.println(
+ "shard2 total:"
+ + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());
+ }
+
+ checkShardConsistency(SHARD1);
+ checkShardConsistency(SHARD2);
+
+ assertDocCounts(VERBOSE);
+ }
+
+ private void testDebugQueries() throws Exception {
+ handle.put("explain", SKIPVAL);
+ handle.put("debug", UNORDERED);
+ handle.put("time", SKIPVAL);
+ handle.put("track", SKIP);
+ query("q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true");
+ query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true");
+ query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING);
+ query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS);
+ query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
index 7975cfb68d3..f9779cfb551 100644
--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
@@ -16,21 +16,1724 @@
*/
package org.apache.solr.cloud;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.lucene.util.IOUtils;
+import org.apache.solr.JSONTestUtil;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.apache.HttpSolrClient;
+import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
+import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.StreamingUpdateRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.client.solrj.response.Group;
+import org.apache.solr.client.solrj.response.GroupCommand;
+import org.apache.solr.client.solrj.response.GroupResponse;
+import org.apache.solr.client.solrj.response.InputStreamResponseParser;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.cloud.api.collections.CollectionHandlingUtils;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.util.TestInjection;
+import org.apache.solr.util.TestInjection.Hook;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This test simply does a bunch of basic things in solrcloud mode and asserts things work as
- * expected. Implementation moved to AbstractBasicDistributedZkTestBase as it is used by many HDFS
- * contrib tests.
+ * expected.
*/
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
-public class BasicDistributedZkTest extends AbstractBasicDistributedZkTestBase {
+public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final String DEFAULT_COLLECTION = "collection1";
+
+ private final boolean onlyLeaderIndexes = random().nextBoolean();
+
+ String tsort = "t_sortable";
+
+ private Map<String, List<SolrClient>> otherCollectionClients = new HashMap<>();
+
+ private String oneInstanceCollection = "oneInstanceCollection";
+ private String oneInstanceCollection2 = "oneInstanceCollection2";
+
+ private AtomicInteger nodeCounter = new AtomicInteger();
+
+ CompletionService<Object> completionService;
+ Set<Future<Object>> pending;
+
+ private static Hook newSearcherHook =
+ new Hook() {
+ volatile CountDownLatch latch;
+ AtomicReference<String> collection = new AtomicReference<>();
+
+ @Override
+ public void newSearcher(String collectionName) {
+ String c = collection.get();
+ if (c != null && c.equals(collectionName)) {
+ log.info("Hook detected newSearcher");
+ try {
+ latch.countDown();
+ } catch (NullPointerException e) {
+
+ }
+ }
+ }
+
+ @Override
+ public void waitForSearcher(
+ String collection, int cnt, int timeoutms, boolean failOnTimeout)
+ throws InterruptedException {
+ latch = new CountDownLatch(cnt);
+ this.collection.set(collection);
+ boolean timeout = !latch.await(timeoutms, TimeUnit.MILLISECONDS);
+ if (timeout && failOnTimeout) {
+ fail("timed out waiting for new searcher event " + latch.getCount());
+ }
+ }
+ };
+
+ public BasicDistributedZkTest() {
+ // we need DVs on point fields to compute stats & facets
+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP))
+ System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");
+
+ sliceCount = 2;
+ completionService = new ExecutorCompletionService<>(executor);
+ pending = new HashSet<>();
+ }
+
+ @BeforeClass
+ public static void beforeBDZKTClass() {
+ TestInjection.newSearcherHook(newSearcherHook);
+ }
+
+ @Override
+ protected boolean useTlogReplicas() {
+ // TODO: tlog replicas makes commits take way to long due to what is likely a bug and it's
+ // TestInjection use
+ return false;
+ }
- @Test
@Override
+ protected void setDistributedParams(ModifiableSolrParams params) {
+
+ if (r.nextBoolean()) {
+ // don't set shards, let that be figured out from the cloud state
+ } else {
+ // use shard ids rather than physical locations
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < getShardCount(); i++) {
+ if (i > 0) sb.append(',');
+ sb.append("shard" + (i + 3));
+ }
+ params.set("shards", sb.toString());
+ }
+ }
+
+ @Test
@ShardsFixed(num = 4)
public void test() throws Exception {
- super.test();
+ // setLoggingLevel(null);
+
+ ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
+ // make sure we have leaders for each shard
+ for (int j = 1; j < sliceCount; j++) {
+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
+ } // make sure we again have leaders for each shard
+
+ waitForRecoveriesToFinish(false);
+
+ handle.clear();
+ handle.put("timestamp", SKIPVAL);
+
+ del("*:*");
+ queryAndCompareShards(params("q", "*:*", "distrib", "false", "sanity_check", "is_empty"));
+
+ // ask every individual replica of every shard to update+commit the same doc id
+ // with an incrementing counter on each update+commit
+ int foo_i_counter = 0;
+ for (SolrClient client : clients) {
+ foo_i_counter++;
+ indexDoc(
+ client,
+ params("commit", "true"), // SOLR-4923
+ sdoc(id, 1, i1, 100, tlong, 100, "foo_i", foo_i_counter));
+ // after every update+commit, check all the shards consistency
+ queryAndCompareShards(
+ params("q", "id:1", "distrib", "false", "sanity_check", "non_distrib_id_1_lookup"));
+ queryAndCompareShards(
+ params(
+ "q", "id:1",
+ "sanity_check", "distrib_id_1_lookup"));
+ }
+
+ indexr(
+ id,
+ 1,
+ i1,
+ 100,
+ tlong,
+ 100,
+ t1,
+ "now is the time for all good men",
+ "foo_f",
+ 1.414f,
+ "foo_b",
+ "true",
+ "foo_d",
+ 1.414d,
+ tsort,
+ "now is the time for all good men");
+ indexr(
+ id,
+ 2,
+ i1,
+ 50,
+ tlong,
+ 50,
+ t1,
+ "to come to the aid of their country.",
+ tsort,
+ "to come to the aid of their country.");
+ indexr(id, 3, i1, 2, tlong, 2, t1, "how now brown cow", tsort, "how now brown cow");
+ indexr(
+ id,
+ 4,
+ i1,
+ -100,
+ tlong,
+ 101,
+ t1,
+ "the quick fox jumped over the lazy dog",
+ tsort,
+ "the quick fox jumped over the lazy dog");
+ indexr(
+ id,
+ 5,
+ i1,
+ 500,
+ tlong,
+ 500,
+ t1,
+ "the quick fox jumped way over the lazy dog",
+ tsort,
+ "the quick fox jumped over the lazy dog");
+ indexr(
+ id,
+ 6,
+ i1,
+ -600,
+ tlong,
+ 600,
+ t1,
+ "humpty dumpy sat on a wall",
+ tsort,
+ "the quick fox jumped over the lazy dog");
+ indexr(
+ id,
+ 7,
+ i1,
+ 123,
+ tlong,
+ 123,
+ t1,
+ "humpty dumpy had a great fall",
+ tsort,
+ "the quick fox jumped over the lazy dog");
+ indexr(
+ id,
+ 8,
+ i1,
+ 876,
+ tlong,
+ 876,
+ t1,
+ "all the kings horses and all the kings men",
+ tsort,
+ "all the kings horses and all the kings men");
+ indexr(
+ id,
+ 9,
+ i1,
+ 7,
+ tlong,
+ 7,
+ t1,
+ "couldn't put humpty together again",
+ tsort,
+ "the quick fox jumped over the lazy dog");
+ indexr(id, 10, i1, 4321, tlong, 4321, t1, "this too shall pass", tsort, "this too shall pass");
+ indexr(
+ id,
+ 11,
+ i1,
+ -987,
+ tlong,
+ 987,
+ t1,
+ "An eye for eye only ends up making the whole world blind.",
+ tsort,
+ "An eye for eye only ends up making the whole world blind.");
+ indexr(
+ id,
+ 12,
+ i1,
+ 379,
+ tlong,
+ 379,
+ t1,
+ "Great works are performed, not by strength, but by perseverance.",
+ tsort,
+ "Great works are performed, not by strength, but by perseverance.");
+ indexr(
+ id,
+ 13,
+ i1,
+ 232,
+ tlong,
+ 232,
+ t1,
+ "no eggs on wall, lesson learned",
+ oddField,
+ "odd man out",
+ tsort,
+ "no eggs on wall, lesson learned");
+
+ indexr(
+ id,
+ 14,
+ "SubjectTerms_mfacet",
+ new String[] {"mathematical models", "mathematical analysis"});
+ indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});
+ indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});
+ String[] vals = new String[100];
+ for (int i = 0; i < 100; i++) {
+ vals[i] = "test " + i;
+ }
+ indexr(id, 17, "SubjectTerms_mfacet", vals);
+
+ for (int i = 100; i < 150; i++) {
+ indexr(id, i);
+ }
+
+ commit();
+
+ testTokenizedGrouping();
+ testSortableTextFaceting();
+ testSortableTextSorting();
+ testSortableTextGrouping();
+
+ queryAndCompareShards(
+ params(
+ "q", "*:*",
+ "sort", "id desc",
+ "distrib", "false",
+ "sanity_check", "is_empty"));
+
+ // random value sort
+ for (String f : fieldNames) {
+ query(false, new String[] {"q", "*:*", "sort", f + " desc"});
+ query(false, new String[] {"q", "*:*", "sort", f + " asc"});
+ }
+
+ // these queries should be exactly ordered and scores should exactly match
+ query(false, new String[] {"q", "*:*", "sort", i1 + " desc"});
+ query(false, new String[] {"q", "*:*", "sort", i1 + " asc"});
+ query(false, new String[] {"q", "*:*", "sort", i1 + " desc", "fl", "*,score"});
+ query(false, new String[] {"q", "*:*", "sort", "n_tl1 asc", "fl", "*,score"});
+ query(false, new String[] {"q", "*:*", "sort", "n_tl1 desc"});
+ handle.put("maxScore", SKIPVAL);
+ query(
+ false,
+ new String[] {"q", "{!func}" + i1}); // does not expect maxScore. So if it comes ,ignore it.
+ // JavaBinCodec.writeSolrDocumentList()
+ // is agnostic of request params.
+ handle.remove("maxScore");
+ query(
+ false,
+ new String[] {
+ "q", "{!func}" + i1, "fl", "*,score"
+ }); // even scores should match exactly here
+
+ handle.put("highlighting", UNORDERED);
+ handle.put("response", UNORDERED);
+
+ handle.put("maxScore", SKIPVAL);
+ query(false, new String[] {"q", "quick"});
+ query(false, new String[] {"q", "all", "fl", "id", "start", "0"});
+ query(
+ false,
+ new String[] {"q", "all", "fl", "foofoofoo", "start", "0"}); // no fields in returned docs
+ query(false, new String[] {"q", "all", "fl", "id", "start", "100"});
+
+ handle.put("score", SKIPVAL);
+ query(false, new String[] {"q", "quick", "fl", "*,score"});
+ query(false, new String[] {"q", "all", "fl", "*,score", "start", "1"});
+ query(false, new String[] {"q", "all", "fl", "*,score", "start", "100"});
+
+ query(
+ false,
+ new String[] {
+ "q", "now their fox sat had put", "fl", "*,score", "hl", "true", "hl.fl", t1
+ });
+
+ query(
+ false,
+ new String[] {
+ "q", "now their fox sat had put", "fl", "foofoofoo", "hl", "true", "hl.fl", t1
+ });
+
+ query(false, new String[] {"q", "matchesnothing", "fl", "*,score"});
+
+ query(false, new Object[] {"q", "*:*", "rows", 100, "facet", "true", "facet.field", t1});
+ query(
+ false,
+ new Object[] {
+ "q",
+ "*:*",
+ "rows",
+ 100,
+ "facet",
+ "true",
+ "facet.field",
+ t1,
+ "facet.limit",
+ -1,
+ "facet.sort",
+ "count"
+ });
+ query(
+ false,
+ new Object[] {
+ "q",
+ "*:*",
+ "rows",
+ 100,
+ "facet",
+ "true",
+ "facet.field",
+ t1,
+ "facet.limit",
+ -1,
+ "facet.sort",
+ "count",
+ "facet.mincount",
+ 2
+ });
+ query(
+ false,
+ new Object[] {
+ "q",
+ "*:*",
+ "rows",
+ 100,
+ "facet",
+ "true",
+ "facet.field",
+ t1,
+ "facet.limit",
+ -1,
+ "facet.sort",
+ "index"
+ });
+ query(
+ false,
+ new Object[] {
+ "q",
+ "*:*",
+ "rows",
+ 100,
+ "facet",
+ "true",
+ "facet.field",
+ t1,
+ "facet.limit",
+ -1,
+ "facet.sort",
+ "index",
+ "facet.mincount",
+ 2
+ });
+ query(
+ false,
+ new Object[] {
+ "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.limit", 1
+ });
+ query(
+ false,
+ new Object[] {
+ "q",
+ "*:*",
+ "rows",
+ 100,
+ "facet",
+ "true",
+ "facet.query",
+ "quick",
+ "facet.query",
+ "all",
+ "facet.query",
+ "*:*"
+ });
+ query(
+ false,
+ new Object[] {
+ "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.offset", 1
+ });
+ query(
+ false,
+ new Object[] {
+ "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.mincount", 2
+ });
+
+ // test faceting multiple things at once
+ query(
+ false,
+ new Object[] {
+ "q",
+ "*:*",
+ "rows",
+ 100,
+ "facet",
+ "true",
+ "facet.query",
+ "quick",
+ "facet.query",
+ "all",
+ "facet.query",
+ "*:*",
+ "facet.field",
+ t1
+ });
+
+ // test filter tagging, facet exclusion, and naming (multi-select facet support)
+ query(
+ false,
+ new Object[] {
+ "q",
+ "*:*",
+ "rows",
+ 100,
+ "facet",
+ "true",
+ "facet.query",
+ "{!key=myquick}quick",
+ "facet.query",
+ "{!key=myall ex=a}all",
+ "facet.query",
+ "*:*",
+ "facet.field",
+ "{!key=mykey ex=a}" + t1,
+ "facet.field",
+ "{!key=other ex=b}" + t1,
+ "facet.field",
+ "{!key=again ex=a,b}" + t1,
+ "facet.field",
+ t1,
+ "fq",
+ "{!tag=a}id_i1:[1 TO 7]",
+ "fq",
+ "{!tag=b}id_i1:[3 TO 9]"
+ });
+ query(
+ false,
+ new Object[] {
+ "q",
+ "*:*",
+ "facet",
+ "true",
+ "facet.field",
+ "{!ex=t1}SubjectTerms_mfacet",
+ "fq",
+ "{!tag=t1}SubjectTerms_mfacet:(test 1)",
+ "facet.limit",
+ "10",
+ "facet.mincount",
+ "1"
+ });
+
+ // test field that is valid in schema but missing in all shards
+ query(
+ false,
+ new Object[] {
+ "q", "*:*", "rows", 100, "facet", "true", "facet.field", missingField, "facet.mincount", 2
+ });
+ // test field that is valid in schema and missing in some shards
+ query(
+ false,
+ new Object[] {
+ "q", "*:*", "rows", 100, "facet", "true", "facet.field", oddField, "facet.mincount", 2
+ });
+
+ query(
+ false, new Object[] {"q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", i1});
+
+ /* TODO: the failure may come back in "exception"
+ try {
+ // test error produced for field that is invalid for schema
+ query("q","*:*", "rows",100, "facet","true", "facet.field",invalidField, "facet.mincount",2);
+ TestCase.fail("SolrServerException expected for invalid field that is not in schema");
+ } catch (SolrServerException ex) {
+ // expected
+ }
+ */
+
+ // Try to get better coverage for refinement queries by turning off over requesting.
+ // This makes it much more likely that we may not get the top facet values and hence
+ // we turn of that checking.
+ handle.put("facet_fields", SKIPVAL);
+ query(
+ false,
+ new Object[] {
+ "q",
+ "*:*",
+ "rows",
+ 0,
+ "facet",
+ "true",
+ "facet.field",
+ t1,
+ "facet.limit",
+ 5,
+ "facet.shard.limit",
+ 5
+ });
+ // check a complex key name
+ query(
+ false,
+ new Object[] {
+ "q",
+ "*:*",
+ "rows",
+ 0,
+ "facet",
+ "true",
+ "facet.field",
+ "{!key='a b/c \\' \\} foo'}" + t1,
+ "facet.limit",
+ 5,
+ "facet.shard.limit",
+ 5
+ });
+ handle.remove("facet_fields");
+
+ // index the same document to two servers and make sure things
+ // don't blow up.
+ if (clients.size() >= 2) {
+ index(id, 100, i1, 107, t1, "oh no, a duplicate!");
+ for (int i = 0; i < clients.size(); i++) {
+ index_specific(i, id, 100, i1, 107, t1, "oh no, a duplicate!");
+ }
+ commit();
+ query(false, new Object[] {"q", "duplicate", "hl", "true", "hl.fl", t1});
+ query(false, new Object[] {"q", "fox duplicate horses", "hl", "true", "hl.fl", t1});
+ query(false, new Object[] {"q", "*:*", "rows", 100});
+ }
+
+ // test debugging
+ handle.put("explain", SKIPVAL);
+ handle.put("debug", UNORDERED);
+ handle.put("time", SKIPVAL);
+ handle.put("track", SKIP);
+ query(
+ false,
+ new Object[] {
+ "q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true"
+ });
+ query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"});
+ query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING});
+ query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS});
+ query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY});
+
+ // try add commitWithin
+ long before = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
+ for (SolrClient client : clients) {
+ assertEquals(
+ "unexpected pre-commitWithin document count on node: "
+ + ((HttpSolrClient) client).getBaseURL(),
+ before,
+ client.query(new SolrQuery("*:*")).getResults().getNumFound());
+ }
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("commitWithin", 10);
+ add(cloudClient, params, List.of(getDoc("id", 300), getDoc("id", 301)));
+
+ newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);
+
+ ClusterState clusterState = getCommonCloudSolrClient().getClusterState();
+ DocCollection dColl = clusterState.getCollection(DEFAULT_COLLECTION);
+
+ assertSliceCounts("should have found 2 docs, 300 and 301", before + 2, dColl);
+
+ // try deleteById commitWithin
+ UpdateRequest deleteByIdReq = new UpdateRequest();
+ deleteByIdReq.deleteById("300");
+ deleteByIdReq.setCommitWithin(10);
+ deleteByIdReq.process(cloudClient);
+
+ newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);
+
+ assertSliceCounts("deleteById commitWithin did not work", before + 1, dColl);
+
+ // try deleteByQuery commitWithin
+ UpdateRequest deleteByQueryReq = new UpdateRequest();
+ deleteByQueryReq.deleteByQuery("id:301");
+ deleteByQueryReq.setCommitWithin(10);
+ deleteByQueryReq.process(cloudClient);
+
+ newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);
+
+ assertSliceCounts("deleteByQuery commitWithin did not work", before, dColl);
+
+ // TODO: This test currently fails because debug info is obtained only
+ // on shards with matches.
+ // query("q","matchesnothing","fl","*,score", "debugQuery", "true");
+
+ // would be better if these where all separate tests - but much, much
+ // slower
+ doOptimisticLockingAndUpdating();
+ testShardParamVariations();
+ testMultipleCollections();
+ testANewCollectionInOneInstance();
+ testSearchByCollectionName();
+ testUpdateByCollectionName();
+ testANewCollectionInOneInstanceWithManualShardAssignement();
+ testNumberOfCommitsWithCommitAfterAdd();
+
+ testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit");
+ testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit");
+
+ testStopAndStartCoresInOneInstance();
+ }
+
+ private void testSortableTextFaceting() throws Exception {
+ SolrQuery query = new SolrQuery("*:*");
+ query.addFacetField(tsort);
+ query.setFacetMissing(false);
+ QueryResponse resp = queryRandomShard(query);
+ List<FacetField> ffs = resp.getFacetFields();
+ for (FacetField ff : ffs) {
+ if (ff.getName().equals(tsort) == false) continue;
+ for (FacetField.Count count : ff.getValues()) {
+ long num = count.getCount();
+ switch (count.getName()) {
+ case "all the kings horses and all the kings men":
+ case "An eye for eye only ends up making the whole world blind.":
+ case "Great works are performed, not by strength, but by perseverance.":
+ case "how now brown cow":
+ case "no eggs on wall, lesson learned":
+ case "now is the time for all good men":
+ case "this too shall pass":
+ case "to come to the aid of their country.":
+ assertEquals("Should have exactly one facet count for field " + ff.getName(), 1, num);
+ break;
+ case "the quick fox jumped over the lazy dog":
+ assertEquals("Should have 5 docs for the lazy dog", 5, num);
+ break;
+ default:
+ fail("No case for facet '" + ff.getName() + "'");
+ }
+ }
+ }
+ }
+
+ private void testSortableTextSorting() throws Exception {
+ SolrQuery query = new SolrQuery("*:*");
+ query.addSort(tsort, SolrQuery.ORDER.desc);
+ query.addField("*");
+ query.addField("eoe_sortable");
+ query.addField(tsort);
+ QueryResponse resp = queryRandomShard(query);
+
+ SolrDocumentList docs = resp.getResults();
+
+ String title = docs.get(0).getFieldValue(tsort).toString();
+ for (SolrDocument doc : docs) {
+ assertTrue(
+ "Docs should be back in sorted order, descending",
+ title.compareTo(doc.getFieldValue(tsort).toString()) >= 0);
+ title = doc.getFieldValue(tsort).toString();
+ }
+ }
+
+ private void testSortableTextGrouping() throws Exception {
+ SolrQuery query = new SolrQuery("*:*");
+ query.add("group", "true");
+ query.add("group.field", tsort);
+ QueryResponse resp = queryRandomShard(query);
+ GroupResponse groupResp = resp.getGroupResponse();
+ List<GroupCommand> grpCmds = groupResp.getValues();
+ for (GroupCommand grpCmd : grpCmds) {
+ if (grpCmd.getName().equals(tsort) == false) continue;
+ for (Group grp : grpCmd.getValues()) {
+ long count = grp.getResult().getNumFound();
+ if (grp.getGroupValue() == null)
+ continue; // Don't count the groups without an entry as the numnber is variable
+ switch (grp.getGroupValue()) {
+ case "all the kings horses and all the kings men":
+ case "An eye for eye only ends up making the whole world blind.":
+ case "Great works are performed, not by strength, but by perseverance.":
+ case "how now brown cow":
+ case "no eggs on wall, lesson learned":
+ case "now is the time for all good men":
+ case "this too shall pass":
+ case "to come to the aid of their country.":
+ assertEquals(
+ "Should have exactly one facet count for field " + grpCmd.getName(), 1, count);
+ break;
+ case "the quick fox jumped over the lazy dog":
+ assertEquals("Should have 5 docs for the lazy dog", 5, count);
+ break;
+ default:
+ fail("No case for facet '" + grpCmd.getName() + "'");
+ }
+ }
+ }
+ }
+
+ private void testTokenizedGrouping() throws Exception {
+ SolrException ex =
+ expectThrows(
+ SolrException.class,
+ () -> {
+ query(false, new String[] {"q", "*:*", "group", "true", "group.field", t1});
+ });
+ assertTrue(
+ "Expected error from server that SortableTextFields are required",
+ ex.getMessage()
+ .contains(
+ "Sorting on a tokenized field that is not a SortableTextField is not supported in cloud mode"));
+ }
+
+ private void assertSliceCounts(String msg, long expected, DocCollection dColl) throws Exception {
+ long found = checkSlicesSameCounts(dColl);
+
+ if (found != expected) {
+ // we get one do over in a bad race
+ Thread.sleep(1000);
+ found = checkSlicesSameCounts(dColl);
+ }
+
+ assertEquals(msg, expected, checkSlicesSameCounts(dColl));
+ }
+
+ // Ensure that total docs found is the expected number.
+ private void waitForDocCount(long expectedNumFound, long waitMillis, String failureMessage)
+ throws Exception {
+ AtomicLong total = new AtomicLong(-1);
+ try {
+ ZkStateReader.from(getCommonCloudSolrClient())
+ .waitForState(
+ DEFAULT_COLLECTION,
+ waitMillis,
+ TimeUnit.MILLISECONDS,
+ (n, c) -> {
+ long docTotal;
+ try {
+ docTotal = checkSlicesSameCounts(c);
+ } catch (SolrServerException | IOException e) {
+ throw new RuntimeException(e);
+ }
+ total.set(docTotal);
+ if (docTotal == expectedNumFound) {
+ return true;
+ }
+ return false;
+ });
+ } catch (TimeoutException | InterruptedException e) {
+
+ }
+ // We could fail here if we broke out of the above because we exceeded the time allowed.
+ assertEquals(failureMessage, expectedNumFound, total.get());
+
+ // This should be redundant, but it caught a test error after all.
+ for (SolrClient client : clients) {
+ assertEquals(
+ failureMessage,
+ expectedNumFound,
+ client.query(new SolrQuery("*:*")).getResults().getNumFound());
+ }
+ }
+
+ // Insure that counts are the same for all replicas in each shard
+ // Return the total doc count for the query.
+ private long checkSlicesSameCounts(DocCollection dColl) throws SolrServerException, IOException {
+ long docTotal = 0; // total number of documents found counting only one replica per slice.
+ for (Slice slice : dColl.getActiveSlices()) {
+ long sliceDocCount = -1;
+ for (Replica rep : slice.getReplicas()) {
+ try (SolrClient one = getHttpSolrClient(rep)) {
+ SolrQuery query = new SolrQuery("*:*");
+ query.setDistrib(false);
+ QueryResponse resp = one.query(query);
+ long hits = resp.getResults().getNumFound();
+ if (sliceDocCount == -1) {
+ sliceDocCount = hits;
+ docTotal += hits;
+ } else {
+ if (hits != sliceDocCount) {
+ return -1;
+ }
+ }
+ }
+ }
+ }
+ return docTotal;
+ }
+
+ private void testShardParamVariations() throws Exception {
+ SolrQuery query = new SolrQuery("*:*");
+ Map<String, Long> shardCounts = new HashMap<>();
+
+ for (String shard : shardToJetty.keySet()) {
+ // every client should give the same numDocs for this shard
+ // shffle the clients in a diff order for each shard
+ List<SolrClient> solrclients = new ArrayList<>(this.clients);
+ Collections.shuffle(solrclients, random());
+ for (SolrClient client : solrclients) {
+ query.set("shards", shard);
+ long numDocs = client.query(query).getResults().getNumFound();
+ assertTrue("numDocs < 0 for shard " + shard + " via " + client, 0 <= numDocs);
+ if (!shardCounts.containsKey(shard)) {
+ shardCounts.put(shard, numDocs);
+ }
+ assertEquals(
+ "inconsitent numDocs for shard " + shard + " via " + client,
+ shardCounts.get(shard).longValue(),
+ numDocs);
+
+ List<CloudJettyRunner> replicaJetties = new ArrayList<>(shardToJetty.get(shard));
+ Collections.shuffle(replicaJetties, random());
+
+ // each replica should also give the same numDocs
+ ArrayList<String> replicaAlts = new ArrayList<>(replicaJetties.size() * 2);
+ for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) {
+ String replica = replicaJetty.url;
+ query.set("shards", replica);
+
+ // replicas already shuffled, use this in the alternative check below
+ if (0 == random().nextInt(3) || replicaAlts.size() < 2) {
+ replicaAlts.add(replica);
+ }
+
+ numDocs = client.query(query).getResults().getNumFound();
+ assertTrue("numDocs < 0 for replica " + replica + " via " + client, 0 <= numDocs);
+ assertEquals(
+ "inconsitent numDocs for shard "
+ + shard
+ + " in replica "
+ + replica
+ + " via "
+ + client,
+ shardCounts.get(shard).longValue(),
+ numDocs);
+ }
+
+ // any combination of replica alternatives should give same numDocs
+ String replicas = String.join("|", replicaAlts);
+ query.set("shards", replicas);
+ numDocs = client.query(query).getResults().getNumFound();
+ assertTrue("numDocs < 0 for replicas " + replicas + " via " + client, 0 <= numDocs);
+ assertEquals(
+ "inconsitent numDocs for replicas " + replicas + " via " + client,
+ shardCounts.get(shard).longValue(),
+ numDocs);
+ }
+ }
+
+ // sums of multiple shards should add up regardless of how we
+ // query those shards or which client we use
+ long randomShardCountsExpected = 0;
+ ArrayList<String> randomShards = new ArrayList<>(shardCounts.size());
+ for (Map.Entry<String, Long> shardData : shardCounts.entrySet()) {
+ if (random().nextBoolean() || randomShards.size() < 2) {
+ String shard = shardData.getKey();
+ randomShardCountsExpected += shardData.getValue();
+ if (random().nextBoolean()) {
+ // use shard id
+ randomShards.add(shard);
+ } else {
+ // use some set explicit replicas
+ ArrayList<String> replicas = new ArrayList<>(7);
+ for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) {
+ if (0 == random().nextInt(3) || 0 == replicas.size()) {
+ replicas.add(replicaJetty.url);
+ }
+ }
+ Collections.shuffle(replicas, random());
+ randomShards.add(String.join("|", replicas));
+ }
+ }
+ }
+ String randShards = String.join(",", randomShards);
+ query.set("shards", randShards);
+ for (SolrClient client : this.clients) {
+ assertEquals(
+ "numDocs for " + randShards + " via " + client,
+ randomShardCountsExpected,
+ client.query(query).getResults().getNumFound());
+ }
+
+ // total num docs must match sum of every shard's numDocs
+ query = new SolrQuery("*:*");
+ long totalShardNumDocs = 0;
+ for (Long c : shardCounts.values()) {
+ totalShardNumDocs += c;
+ }
+ for (SolrClient client : clients) {
+ assertEquals(
+ "sum of shard numDocs on client: " + client,
+ totalShardNumDocs,
+ client.query(query).getResults().getNumFound());
+ }
+ assertTrue("total numDocs <= 0, WTF? Test is useless", 0 < totalShardNumDocs);
+ }
+
+ private void testStopAndStartCoresInOneInstance() throws Exception {
+ JettySolrRunner jetty = jettys.get(0);
+ try (final SolrClient httpSolrClient = (HttpSolrClient) jetty.newClient(15000, 60000)) {
+ ThreadPoolExecutor executor = null;
+ try {
+ executor =
+ new ExecutorUtil.MDCAwareThreadPoolExecutor(
+ 0,
+ Integer.MAX_VALUE,
+ 5,
+ TimeUnit.SECONDS,
+ new SynchronousQueue<Runnable>(),
+ new SolrNamedThreadFactory("testExecutor"));
+ int cnt = 3;
+
+ // create the cores
+ createCollectionInOneInstance(
+ httpSolrClient, jetty.getNodeName(), executor, "multiunload2", 1, cnt);
+ } finally {
+ if (executor != null) {
+ ExecutorUtil.shutdownAndAwaitTermination(executor);
+ }
+ }
+ }
+
+ cloudJettys.get(0).jetty.stop();
+ printLayout();
+
+ cloudJettys.get(0).jetty.start();
+ ZkStateReader.from(cloudClient).forceUpdateCollection("multiunload2");
+ try {
+ ZkStateReader.from(cloudClient).getLeaderRetry("multiunload2", "shard1", 30000);
+ } catch (SolrException e) {
+ printLayout();
+ throw e;
+ }
+
+ printLayout();
+ }
+
+ /** Create a collection in single node */
+ public static void createCollectionInOneInstance(
+ final SolrClient client,
+ String nodeName,
+ ThreadPoolExecutor executor,
+ final String collection,
+ final int numShards,
+ int numReplicas) {
+ assertNotNull(nodeName);
+ try {
+ assertEquals(
+ 0,
+ CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1)
+ .setCreateNodeSet("")
+ .process(client)
+ .getStatus());
+ } catch (SolrServerException | IOException e) {
+ throw new RuntimeException(e);
+ }
+ for (int i = 0; i < numReplicas; i++) {
+ final int freezeI = i;
+ executor.execute(
+ () -> {
+ try {
+ assertTrue(
+ CollectionAdminRequest.addReplicaToShard(
+ collection, "shard" + ((freezeI % numShards) + 1))
+ .setCoreName(collection + freezeI)
+ .setNode(nodeName)
+ .process(client)
+ .isSuccess());
+ } catch (SolrServerException | IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+ }
+
+ @Override
+ protected CollectionAdminResponse createCollection(
+ Map<String, List<Integer>> collectionInfos,
+ String collectionName,
+ String configSetName,
+ int numShards,
+ int numReplicas,
+ SolrClient client,
+ String createNodeSetStr)
+ throws SolrServerException, IOException {
+ // TODO: Use CollectionAdminRequest for this test
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionAction.CREATE.toString());
+
+ params.set(CollectionHandlingUtils.NUM_SLICES, numShards);
+ params.set(ZkStateReader.REPLICATION_FACTOR, numReplicas);
+ if (createNodeSetStr != null)
+ params.set(CollectionHandlingUtils.CREATE_NODE_SET, createNodeSetStr);
+
+ int clientIndex = clients.size() > 1 ? random().nextInt(2) : 0;
+ List<Integer> list = new ArrayList<>();
+ list.add(numShards);
+ list.add(numReplicas);
+ if (collectionInfos != null) {
+ collectionInfos.put(collectionName, list);
+ }
+ params.set("name", collectionName);
+ params.set("collection.configName", configSetName);
+ QueryRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ CollectionAdminResponse res = new CollectionAdminResponse();
+ if (client == null) {
+ final String baseUrl = ((HttpSolrClient) clients.get(clientIndex)).getBaseURL();
+
+ try (SolrClient aClient = createNewSolrClient("", baseUrl)) {
+ res.setResponse(aClient.request(request));
+ }
+ } else {
+ res.setResponse(client.request(request));
+ }
+ return res;
+ }
+
+ @Override
+ protected Replica getLeaderFromZk(String collection, String slice) {
+ ClusterState clusterState = getCommonCloudSolrClient().getClusterState();
+ Replica leader = clusterState.getCollection(collection).getLeader(slice);
+ if (leader == null) {
+ throw new RuntimeException("Could not find leader:" + collection + " " + slice);
+ }
+ return leader;
+ }
+
+ /**
+ * Expects a RegexReplaceProcessorFactories in the chain which will "double up" the values in two
+ * (stored) string fields.
+ *
+ * <p>If the values are "double-doubled" or "not-doubled" then we know the processor was not run
+ * the appropriate number of times
+ */
+ private void testUpdateProcessorsRunOnlyOnce(final String chain) throws Exception {
+
+ final String fieldA = "regex_dup_A_s";
+ final String fieldB = "regex_dup_B_s";
+ final String val = "x";
+ final String expected = "x_x";
+ final ModifiableSolrParams updateParams = new ModifiableSolrParams();
+ updateParams.add(UpdateParams.UPDATE_CHAIN, chain);
+
+ final int numLoops = atLeast(50);
+
+ for (int i = 1; i < numLoops; i++) {
+ // add doc to random client
+ SolrClient updateClient = clients.get(random().nextInt(clients.size()));
+ SolrInputDocument doc = new SolrInputDocument();
+ addFields(doc, id, i, fieldA, val, fieldB, val);
+ UpdateResponse ures = add(updateClient, updateParams, doc);
+ assertEquals(chain + ": update failed", 0, ures.getStatus());
+ ures = updateClient.commit();
+ assertEquals(chain + ": commit failed", 0, ures.getStatus());
+ }
+
+ // query for each doc, and check both fields to ensure the value is correct
+ for (int i = 1; i < numLoops; i++) {
+ final String query = id + ":" + i;
+ QueryResponse qres = queryRandomShard(new SolrQuery(query));
+ assertEquals(chain + ": query failed: " + query, 0, qres.getStatus());
+ assertEquals(
+ chain + ": didn't find correct # docs with query: " + query,
+ 1,
+ qres.getResults().getNumFound());
+ SolrDocument doc = qres.getResults().get(0);
+
+ for (String field : new String[] {fieldA, fieldB}) {
+ assertEquals(
+ chain + ": doc#" + i + " has wrong value for " + field,
+ expected,
+ doc.getFirstValue(field));
+ }
+ }
+ }
+
+ // cloud level test mainly needed just to make sure that versions and errors are propagated
+ // correctly
+ private void doOptimisticLockingAndUpdating() throws Exception {
+ log.info("### STARTING doOptimisticLockingAndUpdating");
+ printLayout();
+
+ final SolrInputDocument sd = sdoc("id", 1000, "_version_", -1);
+ indexDoc(sd);
+
+ ignoreException("version conflict");
+ for (SolrClient client : clients) {
+ SolrException e = expectThrows(SolrException.class, () -> client.add(sd));
+ assertEquals(409, e.code());
+ }
+ unIgnoreException("version conflict");
+
+ // TODO: test deletes. SolrJ needs a good way to pass version for delete...
+
+ final SolrInputDocument sd2 = sdoc("id", 1000, "foo_i", 5);
+ clients.get(0).add(sd2);
+
+ List<Integer> expected = new ArrayList<>();
+ int val = 0;
+ for (SolrClient client : clients) {
+ val += 10;
+ client.add(sdoc("id", 1000, "val_i", map("add", val), "foo_i", val));
+ expected.add(val);
+ }
+
+ QueryRequest qr = new QueryRequest(params("qt", "/get", "id", "1000"));
+ for (SolrClient client : clients) {
+ val += 10;
+ NamedList<?> rsp = client.request(qr);
+ String match = JSONTestUtil.matchObj("/val_i", rsp.get("doc"), expected);
+ if (match != null) throw new RuntimeException(match);
+ }
+ }
+
+ private void testNumberOfCommitsWithCommitAfterAdd() throws SolrServerException, IOException {
+ log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd");
+ long startCommits = getNumCommits((HttpSolrClient) clients.get(0));
+
+ NamedList<Object> result =
+ clients
+ .get(0)
+ .request(
+ new StreamingUpdateRequest(
+ "/update", getFile("books_numeric_ids.csv"), "application/csv")
+ .setCommitWithin(900000)
+ .setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true));
+
+ long endCommits = getNumCommits((HttpSolrClient) clients.get(0));
+ assertEquals(startCommits + 1L, endCommits);
+ }
+
+ private Long getNumCommits(HttpSolrClient sourceClient) throws SolrServerException, IOException {
+ String collection = sourceClient.getDefaultCollection();
+ try (SolrClient client =
+ new HttpSolrClient.Builder(sourceClient.getBaseURL())
+ .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)
+ .withSocketTimeout(60000, TimeUnit.MILLISECONDS)
+ .build()) {
+ var req =
+ new GenericSolrRequest(
+ SolrRequest.METHOD.GET,
+ "/admin/metrics",
+ SolrRequest.SolrRequestType.ADMIN,
+ SolrParams.of("wt", "prometheus"));
+ req.setResponseParser(new InputStreamResponseParser("prometheus"));
+
+ NamedList<Object> resp = client.request(req);
+ try (InputStream in = (InputStream) resp.get("stream")) {
+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ String metricName = "solr_core_update_commit_ops";
+
+ return (long)
+ output
+ .lines()
+ .filter(
+ l ->
+ l.startsWith(metricName)
+ && l.contains("collection=\"" + collection + "\"")
+ && l.contains("ops=\"commits\""))
+ .mapToDouble(s -> Double.parseDouble(s.substring(s.lastIndexOf(" "))))
+ .sum();
+ }
+ }
+ }
+
+ private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception {
+ log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement");
+ assertEquals(
+ 0,
+ CollectionAdminRequest.createCollection(oneInstanceCollection2, "conf1", 2, 2)
+ .setCreateNodeSet("")
+ .process(cloudClient)
+ .getStatus());
+
+ List<SolrClient> collectionClients = new ArrayList<>();
+ for (int i = 0; i < 4; i++) {
+ CollectionAdminResponse resp =
+ CollectionAdminRequest.addReplicaToShard(oneInstanceCollection2, "shard" + ((i % 2) + 1))
+ .setNode(jettys.get(0).getNodeName())
+ .process(cloudClient);
+ for (String coreName : resp.getCollectionCoresStatus().keySet()) {
+ collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString()));
+ }
+ }
+
+ SolrClient client1 = collectionClients.get(0);
+ SolrClient client2 = collectionClients.get(1);
+ SolrClient client3 = collectionClients.get(2);
+ SolrClient client4 = collectionClients.get(3);
+
+ // no one should be recovering
+ waitForRecoveriesToFinish(
+ oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()), false, true);
+
+ assertAllActive(oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()));
+
+ // printLayout();
+
+ // TODO: enable when we don't falsely get slice1...
+ // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000);
+ // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000);
+ client2.add(getDoc(id, "1"));
+ client3.add(getDoc(id, "2"));
+ client4.add(getDoc(id, "3"));
+
+ client1.commit();
+ SolrQuery query = new SolrQuery("*:*");
+ query.set("distrib", false);
+ long oneDocs = client1.query(query).getResults().getNumFound();
+ long twoDocs = client2.query(query).getResults().getNumFound();
+ long threeDocs = client3.query(query).getResults().getNumFound();
+ long fourDocs = client4.query(query).getResults().getNumFound();
+
+ query.set("collection", oneInstanceCollection2);
+ query.set("distrib", true);
+ long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound();
+
+ // System.out.println("1:" + oneDocs);
+ // System.out.println("2:" + twoDocs);
+ // System.out.println("3:" + threeDocs);
+ // System.out.println("4:" + fourDocs);
+ // System.out.println("All Docs:" + allDocs);
+
+ // assertEquals(oneDocs, threeDocs);
+ // assertEquals(twoDocs, fourDocs);
+ // assertNotSame(oneDocs, twoDocs);
+ assertEquals(3, allDocs);
+
+ // we added a role of none on these creates - check for it
+ ZkStateReader zkStateReader = ZkStateReader.from(getCommonCloudSolrClient());
+ zkStateReader.forceUpdateCollection(oneInstanceCollection2);
+ Map<String, Slice> slices =
+ zkStateReader.getClusterState().getCollection(oneInstanceCollection2).getSlicesMap();
+ assertNotNull(slices);
+
+ Replica leader =
+ getCommonCloudSolrClient()
+ .getClusterState()
+ .getCollection(oneInstanceCollection2)
+ .getLeader("shard1");
+
+ // now test that unloading a core gets us a new leader
+ try (SolrClient unloadClient =
+ new HttpSolrClient.Builder(jettys.get(0).getBaseUrl().toString())
+ .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)
+ .withSocketTimeout(60000, TimeUnit.MILLISECONDS)
+ .build()) {
+ Unload unloadCmd = new Unload(true);
+ unloadCmd.setCoreName(leader.getCoreName());
+
+ String leaderUrl = leader.getCoreUrl();
+
+ testExecutor.execute(
+ new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ unloadClient.request(unloadCmd);
+ } catch (SolrServerException e) {
+ throw new RuntimeException(e);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+
+ try {
+ ZkStateReader.from(getCommonCloudSolrClient())
+ .waitForState(
+ oneInstanceCollection2,
+ 20000,
+ TimeUnit.MILLISECONDS,
+ (n, c) -> {
+ try {
+ if (leaderUrl.equals(
+ zkStateReader.getLeaderUrl(oneInstanceCollection2, "shard1", 10000))) {
+ return false;
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ return true;
+ });
+ } catch (TimeoutException | InterruptedException e) {
+ fail("Leader never changed");
+ }
+ }
+
+ IOUtils.close(collectionClients);
+ }
+
+ private void testSearchByCollectionName() throws SolrServerException, IOException {
+ log.info("### STARTING testSearchByCollectionName");
+ SolrClient client = clients.get(0);
+ final String baseUrl = ((HttpSolrClient) client).getBaseURL();
+
+ // the cores each have different names, but if we add the collection name to the url
+ // we should get mapped to the right core
+ try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) {
+ SolrQuery query = new SolrQuery("*:*");
+ long oneDocs = client1.query(query).getResults().getNumFound();
+ assertEquals(3, oneDocs);
+ }
+ }
+
+ private void testUpdateByCollectionName() throws SolrServerException, IOException {
+ log.info("### STARTING testUpdateByCollectionName");
+ SolrClient client = clients.get(0);
+ final String baseUrl = ((HttpSolrClient) client).getBaseURL();
+
+ // the cores each have different names, but if we add the collection name to the url
+ // we should get mapped to the right core
+ // test hitting an update url
+ try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) {
+ client1.commit();
+ }
+ }
+
+ private void testANewCollectionInOneInstance() throws Exception {
+ log.info("### STARTING testANewCollectionInOneInstance");
+ CollectionAdminResponse response =
+ CollectionAdminRequest.createCollection(oneInstanceCollection, "conf1", 2, 2)
+ .setCreateNodeSet(jettys.get(0).getNodeName())
+ .process(cloudClient);
+ assertEquals(0, response.getStatus());
+ List<SolrClient> collectionClients = new ArrayList<>();
+ for (String coreName : response.getCollectionCoresStatus().keySet()) {
+ collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString()));
+ }
+
+ SolrClient client1 = collectionClients.get(0);
+ SolrClient client2 = collectionClients.get(1);
+ SolrClient client3 = collectionClients.get(2);
+ SolrClient client4 = collectionClients.get(3);
+
+ waitForRecoveriesToFinish(
+ oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()), false);
+ assertAllActive(oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()));
+
+ client2.add(getDoc(id, "1"));
+ client3.add(getDoc(id, "2"));
+ client4.add(getDoc(id, "3"));
+
+ client1.commit();
+ SolrQuery query = new SolrQuery("*:*");
+ query.set("distrib", false);
+ long oneDocs = client1.query(query).getResults().getNumFound();
+ long twoDocs = client2.query(query).getResults().getNumFound();
+ long threeDocs = client3.query(query).getResults().getNumFound();
+ long fourDocs = client4.query(query).getResults().getNumFound();
+
+ query.set("collection", oneInstanceCollection);
+ query.set("distrib", true);
+ long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound();
+
+ // System.out.println("1:" + oneDocs);
+ // System.out.println("2:" + twoDocs);
+ // System.out.println("3:" + threeDocs);
+ // System.out.println("4:" + fourDocs);
+ // System.out.println("All Docs:" + allDocs);
+
+ assertEquals(3, allDocs);
+ IOUtils.close(collectionClients);
+ }
+
+ private void createSolrCore(
+ final String collection,
+ List<SolrClient> collectionClients,
+ final String baseUrl,
+ final int num,
+ final String shardId) {
+ Callable<Object> call =
+ () -> {
+ try (SolrClient client = getHttpSolrClient(baseUrl)) {
+ // client.setConnectionTimeout(15000);
+ Create createCmd = new Create();
+ createCmd.setCoreName(collection + num);
+ createCmd.setCollection(collection);
+
+ if (random().nextBoolean()) {
+ // sometimes we use an explicit core node name
+ createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet());
+ }
+
+ if (shardId == null) {
+ createCmd.setNumShards(2);
+ }
+ createCmd.setDataDir(getDataDir(createTempDir(collection).toString()));
+ if (shardId != null) {
+ createCmd.setShardId(shardId);
+ }
+ client.request(createCmd);
+ } catch (Exception e) {
+ log.error("error creating core", e);
+ // fail
+ }
+ return null;
+ };
+
+ pending.add(completionService.submit(call));
+
+ collectionClients.add(createNewSolrClient(collection + num, baseUrl));
+ }
+
+ private void testMultipleCollections() throws Exception {
+ log.info("### STARTING testMultipleCollections");
+ // create another 2 collections and search across them
+ createNewCollection("collection2");
+ createNewCollection("collection3");
+
+ while (pending != null && pending.size() > 0) {
+
+ Future<Object> future = completionService.take();
+ if (future == null) return;
+ pending.remove(future);
+ }
+
+ indexDoc("collection2", getDoc(id, "10000000"));
+ indexDoc("collection2", getDoc(id, "10000001"));
+ indexDoc("collection2", getDoc(id, "10000003"));
+
+ getSolrClient("collection2").add(getDoc(id, "10000004"));
+
+ indexDoc("collection3", getDoc(id, "20000000"));
+ indexDoc("collection3", getDoc(id, "20000001"));
+
+ getSolrClient("collection3").add(getDoc(id, "10000005"));
+
+ otherCollectionClients.get("collection2").get(0).commit();
+ otherCollectionClients.get("collection3").get(0).commit();
+
+ SolrClient clientForCollection1 = getSolrClient("collection1");
+ long collection1Docs =
+ clientForCollection1.query(new SolrQuery("*:*")).getResults().getNumFound();
+
+ long collection2Docs =
+ otherCollectionClients
+ .get("collection2")
+ .get(0)
+ .query(new SolrQuery("*:*"))
+ .getResults()
+ .getNumFound();
+
+ long collection3Docs =
+ otherCollectionClients
+ .get("collection3")
+ .get(0)
+ .query(new SolrQuery("*:*"))
+ .getResults()
+ .getNumFound();
+
+ SolrQuery query = new SolrQuery("*:*");
+ query.set("collection", "collection2,collection3");
+ long found = clients.get(0).query(query).getResults().getNumFound();
+ assertEquals(collection2Docs + collection3Docs, found);
+
+ query = new SolrQuery("*:*");
+ query.set("collection", "collection1,collection2,collection3");
+ found = clients.get(0).query(query).getResults().getNumFound();
+ assertEquals(collection1Docs + collection2Docs + collection3Docs, found);
+
+ // try to search multiple with cloud client
+ found = getCommonCloudSolrClient().query(query).getResults().getNumFound();
+ assertEquals(collection1Docs + collection2Docs + collection3Docs, found);
+
+ query.set("collection", "collection2,collection3");
+ found = getCommonCloudSolrClient().query(query).getResults().getNumFound();
+ assertEquals(collection2Docs + collection3Docs, found);
+
+ query.set("collection", "collection3");
+ found = getCommonCloudSolrClient().query(query).getResults().getNumFound();
+ assertEquals(collection3Docs, found);
+
+ query.remove("collection");
+ found = getCommonCloudSolrClient().query(query).getResults().getNumFound();
+ assertEquals(collection1Docs, found);
+
+ assertEquals(collection3Docs, collection2Docs - 1);
+ }
+
+ protected void indexDoc(String collection, SolrInputDocument doc)
+ throws IOException, SolrServerException {
+ List<SolrClient> clients = otherCollectionClients.get(collection);
+ int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size();
+ SolrClient client = clients.get(which);
+ client.add(doc);
+ }
+
+ @SuppressWarnings({"unchecked"})
+ private void createNewCollection(final String collection) throws InterruptedException {
+ try {
+ assertEquals(
+ 0,
+ CollectionAdminRequest.createCollection(collection, "conf1", 2, 1)
+ .setCreateNodeSet("")
+ .process(cloudClient)
+ .getStatus());
+ } catch (Exception e) {
+ log.error("error creating collection", e);
+ // fails
+ }
+ final List<SolrClient> collectionClients = new ArrayList<>();
+ otherCollectionClients.put(collection, collectionClients);
+ int unique = 0;
+ for (final JettySolrRunner runner : jettys) {
+ unique++;
+ final int frozeUnique = unique;
+ Callable<Object> call =
+ () -> {
+ try {
+ assertTrue(
+ CollectionAdminRequest.addReplicaToShard(
+ collection, "shard" + ((frozeUnique % 2) + 1))
+ .setNode(runner.getNodeName())
+ .process(cloudClient)
+ .isSuccess());
+ } catch (Exception e) {
+ log.error("error adding replica", e);
+ // fails
+ }
+ return null;
+ };
+
+ collectionClients.add(createNewSolrClient(collection, runner.getBaseUrl().toString()));
+ pending.add(completionService.submit(call));
+ while (pending != null && pending.size() > 0) {
+
+ Future<Object> future = completionService.take();
+ if (future == null) return;
+ pending.remove(future);
+ }
+ }
+ }
+
+ @Override
+ protected SolrClient createNewSolrClient(String collection, String baseUrl) {
+
+ SolrClient client = getHttpSolrClient(baseUrl, collection);
+
+ return client;
+ }
+
+ /**
+ * @param collection the name of a collection or core to set as the "default" on the created
+ * client.
+ * @param baseUrl the "base" URL of a Solr node. Should <em>not</em> contain a collection or core
+ * name.
+ * @param connectionTimeoutMillis the HTTP connection timeout in milliseconds
+ * @param socketTimeoutMillis the HTTP socket-read timeout in milliseconds
+ */
+ protected SolrClient createNewSolrClient(
+ String collection, String baseUrl, int connectionTimeoutMillis, int socketTimeoutMillis) {
+
+ SolrClient client =
+ new HttpSolrClient.Builder(baseUrl)
+ .withDefaultCollection(collection)
+ .withConnectionTimeout(connectionTimeoutMillis, TimeUnit.MILLISECONDS)
+ .withSocketTimeout(socketTimeoutMillis, TimeUnit.MILLISECONDS)
+ .build();
+
+ return client;
+ }
+
+ @Override
+ protected QueryResponse queryRandomShard(ModifiableSolrParams params)
+ throws SolrServerException, IOException {
+
+ if (r.nextBoolean()) return super.queryRandomShard(params);
+
+ if (r.nextBoolean()) params.set("collection", DEFAULT_COLLECTION);
+
+ QueryResponse rsp = getCommonCloudSolrClient().query(params);
+ return rsp;
+ }
+
+ @Override
+ public void distribTearDown() throws Exception {
+ super.distribTearDown();
+ if (otherCollectionClients != null) {
+ for (List<SolrClient> clientList : otherCollectionClients.values()) {
+ IOUtils.close(clientList);
+ }
+ }
+ otherCollectionClients = null;
+ List<Runnable> tasks = executor.shutdownNow();
+ assertTrue(tasks.isEmpty());
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
index 3fb1bbabc6e..3f4ab451c85 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
@@ -16,11 +16,324 @@
*/
package org.apache.solr.cloud;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
+import org.apache.solr.client.solrj.apache.CloudLegacySolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
-/*
- * Implementation moved to AbstractChaosMonkeyNothingIsSafeTestBase.java as it is also
- * used by the HDFS contrib tests.
- */
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
-public class ChaosMonkeyNothingIsSafeTest extends AbstractChaosMonkeyNothingIsSafeTestBase {}
+public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase {
+ private static final int FAIL_TOLERANCE = 100;
+
+ private static final Integer RUN_LENGTH =
+ Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
+
+ private final boolean onlyLeaderIndexes = random().nextBoolean();
+
+ @BeforeClass
+ public static void beforeSuperClass() {
+ schemaString = "schema15.xml"; // we need a string id
+ System.setProperty("solr.autoCommit.maxTime", "15000");
+ System.clearProperty("solr.httpclient.retries");
+ System.clearProperty("solr.retries.on.forward");
+ System.clearProperty("solr.retries.to.followers");
+ setErrorHook();
+ }
+
+ @AfterClass
+ public static void afterSuperClass() {
+ System.clearProperty("solr.autoCommit.maxTime");
+ clearErrorHook();
+ }
+
+ @Override
+ protected void destroyServers() throws Exception {
+
+ super.destroyServers();
+ }
+
+ protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};
+ protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};
+
+ private int clientSoTimeout = 60000;
+
+ private volatile FullThrottleStoppableIndexingThread ftIndexThread;
+
+ private final boolean runFullThrottle;
+
+ @Override
+ public String[] getFieldNames() {
+ return fieldNames;
+ }
+
+ @Override
+ public RandVal[] getRandValues() {
+ return randVals;
+ }
+
+ @Override
+ public void distribSetUp() throws Exception {
+ super.distribSetUp();
+ // can help to hide this when testing and looking at logs
+ // ignoreException("shard update error");
+ useFactory("solr.StandardDirectoryFactory");
+ }
+
+ @Override
+ public void distribTearDown() throws Exception {
+ try {
+ ftIndexThread.safeStop();
+ } catch (NullPointerException e) {
+ // okay
+ }
+ super.distribTearDown();
+ }
+
+ public ChaosMonkeyNothingIsSafeTest() {
+ super();
+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1"));
+ if (sliceCount == -1) {
+ sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1;
+ }
+
+ int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1"));
+ if (numShards == -1) {
+ // we make sure that there's at least one shard with more than one replica
+ // so that the ChaosMonkey has something to kill
+ numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1;
+ }
+ fixShardCount(numShards);
+
+ // TODO: we only do this sometimes so that we can sometimes compare against control,
+ // it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer
+ runFullThrottle = random().nextBoolean();
+ }
+
+ @Override
+ protected boolean useTlogReplicas() {
+ return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug
+ // and it's TestInjection use
+ }
+
+ @Override
+ protected CloudSolrClient createCloudClient(String defaultCollection) {
+ return this.createCloudClient(defaultCollection, this.clientSoTimeout);
+ }
+
+ protected CloudSolrClient createCloudClient(String defaultCollection, int socketTimeout) {
+
+ return getCloudSolrClient(
+ zkServer.getZkAddress(), defaultCollection, random().nextBoolean(), 30000, socketTimeout);
+ }
+
+ @Test
+ @SuppressWarnings({"try"})
+ public void test() throws Exception {
+ // None of the operations used here are particularly costly, so this should work.
+ // Using this low timeout will also help us catch index stalling.
+ clientSoTimeout = 5000;
+
+ boolean testSuccessful = false;
+ try (CloudSolrClient ourCloudClient = createCloudClient(DEFAULT_COLLECTION)) {
+ handle.clear();
+ handle.put("timestamp", SKIPVAL);
+ ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
+ // make sure we have leaders for each shard
+ for (int j = 1; j < sliceCount; j++) {
+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
+ } // make sure we again have leaders for each shard
+
+ waitForRecoveriesToFinish(false);
+
+ // we cannot do delete by query
+ // as it's not supported for recovery
+ del("*:*");
+
+ List<StoppableThread> threads = new ArrayList<>();
+ List<StoppableIndexingThread> indexTreads = new ArrayList<>();
+ int threadCount = TEST_NIGHTLY ? 3 : 1;
+ int i = 0;
+ for (i = 0; i < threadCount; i++) {
+ StoppableIndexingThread indexThread =
+ new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true);
+ threads.add(indexThread);
+ indexTreads.add(indexThread);
+ indexThread.start();
+ }
+
+ threadCount = 1;
+ i = 0;
+ for (i = 0; i < threadCount; i++) {
+ StoppableSearchThread searchThread = new StoppableSearchThread(cloudClient);
+ threads.add(searchThread);
+ searchThread.start();
+ }
+
+ if (runFullThrottle) {
+ ftIndexThread =
+ new FullThrottleStoppableIndexingThread(
+ ((CloudLegacySolrClient) cloudClient).getHttpClient(),
+ controlClient,
+ cloudClient,
+ clients,
+ "ft1",
+ true,
+ this.clientSoTimeout);
+ ftIndexThread.start();
+ }
+
+ chaosMonkey.startTheMonkey(true, 10000);
+ try {
+ long runLength;
+ if (RUN_LENGTH != -1) {
+ runLength = RUN_LENGTH;
+ } else {
+ int[] runTimes;
+ if (TEST_NIGHTLY) {
+ runTimes =
+ new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000};
+ } else {
+ runTimes = new int[] {5000, 7000, 15000};
+ }
+ runLength = runTimes[random().nextInt(runTimes.length - 1)];
+ }
+
+ Thread.sleep(runLength);
+ } finally {
+ chaosMonkey.stopTheMonkey();
+ }
+
+ // ideally this should go into chaosMonkey
+ restartZk(1000 * (5 + random().nextInt(4)));
+
+ if (runFullThrottle) {
+ ftIndexThread.safeStop();
+ }
+
+ for (StoppableThread indexThread : threads) {
+ indexThread.safeStop();
+ }
+
+ // start any downed jetties to be sure we still will end up with a leader per shard...
+
+ // wait for stop...
+ for (StoppableThread indexThread : threads) {
+ indexThread.join();
+ }
+
+ // try and wait for any replications and what not to finish...
+
+ Thread.sleep(2000);
+
+ // wait until there are no recoveries...
+ waitForThingsToLevelOut();
+
+ // make sure we again have leaders for each shard
+ for (int j = 1; j < sliceCount; j++) {
+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000);
+ }
+
+ commit();
+
+ // TODO: assert we didn't kill everyone
+
+ zkStateReader.updateLiveNodes();
+ assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0);
+
+ // we expect full throttle fails, but cloud client should not easily fail
+ for (StoppableThread indexThread : threads) {
+ if (indexThread instanceof StoppableIndexingThread
+ && !(indexThread instanceof FullThrottleStoppableIndexingThread)) {
+ int failCount = ((StoppableIndexingThread) indexThread).getFailCount();
+ assertFalse(
+ "There were too many update fails ("
+ + failCount
+ + " > "
+ + FAIL_TOLERANCE
+ + ") - we expect it can happen, but shouldn't easily",
+ failCount > FAIL_TOLERANCE);
+ }
+ }
+
+ waitForThingsToLevelOut(20, TimeUnit.SECONDS);
+
+ commit();
+
+ Set<String> addFails = getAddFails(indexTreads);
+ Set<String> deleteFails = getDeleteFails(indexTreads);
+ // full throttle thread can
+ // have request fails
+ checkShardConsistency(!runFullThrottle, true, addFails, deleteFails);
+
+ long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();
+
+ // ensure we have added more than 0 docs
+ long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
+
+ assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0);
+
+ if (VERBOSE)
+ System.out.println(
+ "control docs:"
+ + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()
+ + "\n\n");
+
+ // try and make a collection to make sure the overseer has survived the expiration and session
+ // loss
+
+ // sometimes we restart zookeeper as well
+ if (random().nextBoolean()) {
+ // restartZk(1000 * (5 + random().nextInt(4)));
+ }
+
+ try (CloudSolrClient client = createCloudClient("collection1", 30000)) {
+ createCollection(null, "testcollection", 1, 1, client, null, "conf1");
+ }
+ List<Integer> numShardsNumReplicas = new ArrayList<>(2);
+ numShardsNumReplicas.add(1);
+ numShardsNumReplicas.add(1);
+ checkForCollection("testcollection", numShardsNumReplicas);
+
+ testSuccessful = true;
+ } finally {
+ if (!testSuccessful) {
+ printLayout();
+ }
+ }
+ }
+
+ private Set<String> getAddFails(List<StoppableIndexingThread> threads) {
+ Set<String> addFails = new HashSet<>();
+ for (StoppableIndexingThread thread : threads) {
+ addFails.addAll(thread.getAddFails());
+ }
+ return addFails;
+ }
+
+ private Set<String> getDeleteFails(List<StoppableIndexingThread> threads) {
+ Set<String> deleteFails = new HashSet<>();
+ for (StoppableIndexingThread thread : threads) {
+ deleteFails.addAll(thread.getDeleteFails());
+ }
+ return deleteFails;
+ }
+
+ // skip the randoms - they can deadlock...
+ @Override
+ protected void indexr(Object... fields) throws Exception {
+ SolrInputDocument doc = getDoc(fields);
+ indexDoc(doc);
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
index a4c1ab267d1..a75b637af22 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
@@ -14,14 +14,220 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.solr.cloud;
-/*
- * Implementation moved to AbstractChaosMonkeySafeLeaderTestBase as is used by HDFS contrib module test
- */
-public class ChaosMonkeySafeLeaderTest extends AbstractChaosMonkeySafeLeaderTestBase {
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.common.SolrInputDocument;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final Integer RUN_LENGTH =
+ Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
+
+ @BeforeClass
+ public static void beforeSuperClass() {
+ schemaString = "schema15.xml"; // we need a string id
+ System.setProperty("solr.autoCommit.maxTime", "15000");
+ System.clearProperty("solr.httpclient.retries");
+ System.clearProperty("solr.retries.on.forward");
+ System.clearProperty("solr.retries.to.followers");
+ setErrorHook();
+ }
+
+ @AfterClass
+ public static void afterSuperClass() {
+ System.clearProperty("solr.autoCommit.maxTime");
+ clearErrorHook();
+ }
+
+ protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};
+ protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};
+
+ @Override
+ public String[] getFieldNames() {
+ return fieldNames;
+ }
+
@Override
+ public RandVal[] getRandValues() {
+ return randVals;
+ }
+
protected String getDirectoryFactory() {
return "solr.StandardDirectoryFactory";
}
+
+ @Override
+ public void distribSetUp() throws Exception {
+ useFactory(getDirectoryFactory());
+ super.distribSetUp();
+ }
+
+ public ChaosMonkeySafeLeaderTest() {
+ super();
+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1"));
+ if (sliceCount == -1) {
+ sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1;
+ }
+
+ int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1"));
+ if (numShards == -1) {
+ // we make sure that there's at least one shard with more than one replica
+ // so that the ChaosMonkey has something to kill
+ numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1;
+ }
+ fixShardCount(numShards);
+ }
+
+ @Test
+ public void test() throws Exception {
+
+ handle.clear();
+ handle.put("timestamp", SKIPVAL);
+
+ // randomly turn on 1 seconds 'soft' commit
+ randomlyEnableAutoSoftCommit();
+
+ tryDelete();
+
+ List<StoppableIndexingThread> threads = new ArrayList<>();
+ int threadCount = 2;
+ int batchSize = 1;
+ if (random().nextBoolean()) {
+ batchSize = random().nextInt(98) + 2;
+ }
+
+ boolean pauseBetweenUpdates = TEST_NIGHTLY ? random().nextBoolean() : true;
+ int maxUpdates = -1;
+ if (!pauseBetweenUpdates) {
+ maxUpdates = 1000 + random().nextInt(1000);
+ } else {
+ maxUpdates = 15000;
+ }
+
+ for (int i = 0; i < threadCount; i++) {
+ StoppableIndexingThread indexThread =
+ new StoppableIndexingThread(
+ controlClient,
+ cloudClient,
+ Integer.toString(i),
+ true,
+ maxUpdates,
+ batchSize,
+ pauseBetweenUpdates); // random().nextInt(999) + 1
+ threads.add(indexThread);
+ indexThread.start();
+ }
+
+ chaosMonkey.startTheMonkey(false, 500);
+ try {
+ long runLength;
+ if (RUN_LENGTH != -1) {
+ runLength = RUN_LENGTH;
+ } else {
+ int[] runTimes;
+ if (TEST_NIGHTLY) {
+ runTimes =
+ new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000};
+ } else {
+ runTimes = new int[] {5000, 7000, 15000};
+ }
+ runLength = runTimes[random().nextInt(runTimes.length - 1)];
+ }
+
+ Thread.sleep(runLength);
+ } finally {
+ chaosMonkey.stopTheMonkey();
+ }
+
+ for (StoppableIndexingThread indexThread : threads) {
+ indexThread.safeStop();
+ }
+
+ // wait for stop...
+ for (StoppableIndexingThread indexThread : threads) {
+ indexThread.join();
+ }
+
+ for (StoppableIndexingThread indexThread : threads) {
+ assertEquals(0, indexThread.getFailCount());
+ }
+
+ // try and wait for any replications and what not to finish...
+
+ Thread.sleep(2000);
+
+ waitForThingsToLevelOut(3, TimeUnit.MINUTES);
+
+ // even if things were leveled out, a jetty may have just been stopped or something
+ // we wait again and wait to level out again to make sure the system is not still in flux
+
+ Thread.sleep(3000);
+
+ waitForThingsToLevelOut(3, TimeUnit.MINUTES);
+
+ checkShardConsistency(batchSize == 1, true);
+
+ if (VERBOSE)
+ System.out.println(
+ "control docs:"
+ + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()
+ + "\n\n");
+
+ // try and make a collection to make sure the overseer has survived the expiration and session
+ // loss
+
+ // sometimes we restart zookeeper as well
+ if (random().nextBoolean()) {
+ zkServer.shutdown();
+ zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());
+ zkServer.run(false);
+ }
+
+ try (CloudSolrClient client = createCloudClient("collection1")) {
+ createCollection(null, "testcollection", 1, 1, client, null, "conf1");
+ }
+ List<Integer> numShardsNumReplicas = new ArrayList<>(2);
+ numShardsNumReplicas.add(1);
+ numShardsNumReplicas.add(1);
+ checkForCollection("testcollection", numShardsNumReplicas);
+ }
+
+ private void tryDelete() throws Exception {
+ long start = System.nanoTime();
+ long timeout = start + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS);
+ while (System.nanoTime() < timeout) {
+ try {
+ del("*:*");
+ break;
+ } catch (SolrServerException e) {
+ log.error("cluster may not be up yet", e);
+ }
+ Thread.sleep(100);
+ }
+ }
+
+ // skip the randoms - they can deadlock...
+ @Override
+ protected void indexr(Object... fields) throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ addFields(doc, fields);
+ addFields(doc, "rnd_b", true);
+ indexDoc(doc);
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java
index 265cbd4d1ea..d6e00a202ee 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java
@@ -17,23 +17,392 @@
package org.apache.solr.cloud;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.solr.client.api.model.CoreStatusResponse;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.response.CoreAdminResponse;
+import org.apache.solr.client.solrj.response.RequestStatusState;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.util.IdUtils;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-/**
- * Implementation moved to AbstractMoveReplicaTestBase as it is referenced by HDFS contrib module
- * tests.
- */
-public class MoveReplicaTest extends AbstractMoveReplicaTestBase {
+public class MoveReplicaTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected boolean inPlaceMove = random().nextBoolean();
+ protected boolean isCollectionApiDistributed = false;
+
+ protected String getConfigSet() {
+ return "cloud-dynamic";
+ }
+
+ @Before
+ public void beforeTest() throws Exception {
+
+ configureCluster(4)
+ .addConfig("conf1", configset(getConfigSet()))
+ .addConfig("conf2", configset(getConfigSet()))
+ .withSolrXml(TEST_PATH().resolve("solr.xml"))
+ .configure();
+
+ // If Collection API is distributed let's not wait for Overseer.
+ isCollectionApiDistributed =
+ new CollectionAdminRequest.RequestApiDistributedProcessing()
+ .process(cluster.getSolrClient())
+ .getIsCollectionApiDistributed();
+ if (isCollectionApiDistributed) {
+ return;
+ }
+
+ NamedList<Object> overSeerStatus =
+ cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ JettySolrRunner overseerJetty = null;
+ String overseerLeader = (String) overSeerStatus.get("leader");
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ if (jetty.getNodeName().equals(overseerLeader)) {
+ overseerJetty = jetty;
+ break;
+ }
+ }
+ if (overseerJetty == null) {
+ fail("no overseer leader!");
+ }
+ }
+
+ @After
+ public void afterTest() throws Exception {
+ try {
+ shutdownCluster();
+ } finally {
+ super.tearDown();
+ }
+ }
- @Override
@Test
public void test() throws Exception {
- super.test();
+ String coll = getTestClass().getSimpleName() + "_coll_" + inPlaceMove;
+ if (log.isInfoEnabled()) {
+ log.info("total_jettys: {}", cluster.getJettySolrRunners().size());
+ }
+ int REPLICATION = 2;
+
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+
+ // random create tlog or pull type replicas with nrt
+ boolean isTlog = random().nextBoolean();
+ CollectionAdminRequest.Create create =
+ CollectionAdminRequest.createCollection(
+ coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0);
+ cloudClient.request(create);
+
+ addDocs(coll, 100);
+
+ Replica replica = getRandomReplica(coll, cloudClient);
+ Set<String> liveNodes = cloudClient.getClusterState().getLiveNodes();
+ ArrayList<String> l = new ArrayList<>(liveNodes);
+ Collections.shuffle(l, random());
+ String targetNode = null;
+ for (String node : liveNodes) {
+ if (!replica.getNodeName().equals(node)) {
+ targetNode = node;
+ break;
+ }
+ }
+ assertNotNull(targetNode);
+ String shardId = null;
+ for (Slice slice : cloudClient.getClusterState().getCollection(coll).getSlices()) {
+ if (slice.getReplicas().contains(replica)) {
+ shardId = slice.getName();
+ }
+ }
+
+ int sourceNumCores =
+ getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name());
+ int targetNumCores = getNumOfCores(cloudClient, targetNode, coll, replica.getType().name());
+
+ CollectionAdminRequest.MoveReplica moveReplica =
+ createMoveReplicaRequest(coll, replica, targetNode);
+ moveReplica.setInPlaceMove(inPlaceMove);
+ String asyncId = IdUtils.randomId();
+ moveReplica.processAsync(asyncId, cloudClient);
+ CollectionAdminRequest.RequestStatus requestStatus =
+ CollectionAdminRequest.requestStatus(asyncId);
+ // wait for async request success
+ boolean success = false;
+ for (int i = 0; i < 200; i++) {
+ CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
+ if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) {
+ success = true;
+ break;
+ }
+ assertNotSame(rsp.getRequestStatus(), RequestStatusState.FAILED);
+ Thread.sleep(500);
+ }
+ assertTrue(success);
+ assertEquals(
+ "should be one less core on the source node!",
+ sourceNumCores - 1,
+ getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name()));
+ assertEquals(
+ "should be one more core on target node!",
+ targetNumCores + 1,
+ getNumOfCores(cloudClient, targetNode, coll, replica.getType().name()));
+ // wait for recovery
+ boolean recovered = false;
+ for (int i = 0; i < 300; i++) {
+ DocCollection collState = getCollectionState(coll);
+ log.debug("###### {}", collState);
+ Collection<Replica> replicas = collState.getSlice(shardId).getReplicas();
+ boolean allActive = true;
+ boolean hasLeaders = true;
+ if (replicas != null && !replicas.isEmpty()) {
+ for (Replica r : replicas) {
+ if (!r.getNodeName().equals(targetNode)) {
+ continue;
+ }
+ if (!r.isActive(Collections.singleton(targetNode))) {
+ log.info("Not active: {}", r);
+ allActive = false;
+ }
+ }
+ } else {
+ allActive = false;
+ }
+ for (Slice slice : collState.getSlices()) {
+ if (slice.getLeader() == null) {
+ hasLeaders = false;
+ }
+ }
+ if (allActive && hasLeaders) {
+ // check the number of active replicas
+ assertEquals("total number of replicas", REPLICATION, replicas.size());
+ recovered = true;
+ break;
+ } else {
+ log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders);
+ Thread.sleep(1000);
+ }
+ }
+ assertTrue("replica never fully recovered", recovered);
+
+ assertEquals(
+ 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
+
+ moveReplica = createMoveReplicaRequest(coll, replica, targetNode, shardId);
+ moveReplica.setInPlaceMove(inPlaceMove);
+ moveReplica.process(cloudClient);
+ checkNumOfCores(cloudClient, replica.getNodeName(), coll, sourceNumCores);
+ // wait for recovery
+ recovered = false;
+ for (int i = 0; i < 300; i++) {
+ DocCollection collState = getCollectionState(coll);
+ log.debug("###### {}", collState);
+ Collection<Replica> replicas = collState.getSlice(shardId).getReplicas();
+ boolean allActive = true;
+ boolean hasLeaders = true;
+ if (replicas != null && !replicas.isEmpty()) {
+ for (Replica r : replicas) {
+ if (!r.getNodeName().equals(replica.getNodeName())) {
+ continue;
+ }
+ if (!r.isActive(Collections.singleton(replica.getNodeName()))) {
+ log.info("Not active yet: {}", r);
+ allActive = false;
+ }
+ }
+ } else {
+ allActive = false;
+ }
+ for (Slice slice : collState.getSlices()) {
+ if (slice.getLeader() == null) {
+ hasLeaders = false;
+ }
+ }
+ if (allActive && hasLeaders) {
+ assertEquals("total number of replicas", REPLICATION, replicas.size());
+ recovered = true;
+ break;
+ } else {
+ Thread.sleep(1000);
+ }
+ }
+ assertTrue("replica never fully recovered", recovered);
+
+ assertEquals(
+ 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
}
- @Override
@Test
public void testFailedMove() throws Exception {
- super.testFailedMove();
+ String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove;
+ int REPLICATION = 2;
+
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+
+ // random create tlog or pull type replicas with nrt
+ boolean isTlog = random().nextBoolean();
+ CollectionAdminRequest.Create create =
+ CollectionAdminRequest.createCollection(
+ coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0);
+ cloudClient.request(create);
+
+ addDocs(coll, 100);
+
+ NamedList<Object> overSeerStatus =
+ cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ String overseerLeader = (String) overSeerStatus.get("leader");
+
+ // don't kill overseer in this test
+ Replica replica;
+ int count = 10;
+ do {
+ replica = getRandomReplica(coll, cloudClient);
+ } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0);
+ assertNotNull("could not find non-overseer replica???", replica);
+ Set<String> liveNodes = cloudClient.getClusterState().getLiveNodes();
+ ArrayList<String> l = new ArrayList<>(liveNodes);
+ Collections.shuffle(l, random());
+ String targetNode = null;
+ for (String node : liveNodes) {
+ if (!replica.getNodeName().equals(node)
+ && (isCollectionApiDistributed || !overseerLeader.equals(node))) {
+ targetNode = node;
+ break;
+ }
+ }
+ assertNotNull(targetNode);
+ CollectionAdminRequest.MoveReplica moveReplica =
+ createMoveReplicaRequest(coll, replica, targetNode);
+ moveReplica.setInPlaceMove(inPlaceMove);
+ // start moving
+ String asyncId = IdUtils.randomId();
+ moveReplica.processAsync(asyncId, cloudClient);
+ // shut down target node
+ for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+ if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) {
+ JettySolrRunner j = cluster.stopJettySolrRunner(i);
+ cluster.waitForJettyToStop(j);
+ break;
+ }
+ }
+ CollectionAdminRequest.RequestStatus requestStatus =
+ CollectionAdminRequest.requestStatus(asyncId);
+ // wait for async request success
+ boolean success = true;
+ for (int i = 0; i < 200; i++) {
+ CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
+ assertNotSame(
+ rsp.getRequestStatus().toString(), rsp.getRequestStatus(), RequestStatusState.COMPLETED);
+ if (rsp.getRequestStatus() == RequestStatusState.FAILED) {
+ success = false;
+ break;
+ }
+ Thread.sleep(500);
+ }
+ assertFalse(success);
+
+ if (log.isInfoEnabled()) {
+ log.info(
+ "--- current collection state: {}", cloudClient.getClusterState().getCollection(coll));
+ }
+ assertEquals(
+ 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
+ }
+
+ private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(
+ String coll, Replica replica, String targetNode, String shardId) {
+ return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName());
+ }
+
+ private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(
+ String coll, Replica replica, String targetNode) {
+ return new CollectionAdminRequest.MoveReplica(coll, replica.getName(), targetNode);
+ }
+
+ private Replica getRandomReplica(String coll, CloudSolrClient cloudClient) throws IOException {
+ List<Replica> replicas = cloudClient.getClusterState().getCollection(coll).getReplicas();
+ Collections.shuffle(replicas, random());
+ return replicas.get(0);
+ }
+
+ private void checkNumOfCores(
+ CloudSolrClient cloudClient, String nodeName, String collectionName, int expectedCores)
+ throws IOException, SolrServerException {
+ assertEquals(
+ nodeName + " does not have expected number of cores",
+ expectedCores,
+ getNumOfCores(cloudClient, nodeName, collectionName));
+ }
+
+ private int getNumOfCores(CloudSolrClient cloudClient, String nodeName, String collectionName)
+ throws IOException, SolrServerException {
+ return getNumOfCores(cloudClient, nodeName, collectionName, null);
+ }
+
+ private int getNumOfCores(
+ CloudSolrClient cloudClient, String nodeName, String collectionName, String replicaType)
+ throws IOException, SolrServerException {
+ try (SolrClient coreclient =
+ getHttpSolrClient(ZkStateReader.from(cloudClient).getBaseUrlForNodeName(nodeName))) {
+ CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient);
+ if (status.getCoreStatus().size() == 0) {
+ return 0;
+ }
+ if (collectionName == null && replicaType == null) {
+ return status.getCoreStatus().size();
+ }
+ // filter size by collection name
+ int size = 0;
+ for (Map.Entry<String, CoreStatusResponse.SingleCoreData> coreStatusEntry :
+ status.getCoreStatus().entrySet()) {
+ if (collectionName != null) {
+ String coll = coreStatusEntry.getValue().cloud.collection;
+ if (!collectionName.equals(coll)) {
+ continue;
+ }
+ }
+ if (replicaType != null) {
+ String type = coreStatusEntry.getValue().cloud.replicaType;
+ if (!replicaType.equals(type)) {
+ continue;
+ }
+ }
+ size++;
+ }
+ return size;
+ }
+ }
+
+ protected void addDocs(String collection, int numDocs) throws Exception {
+ SolrClient solrClient = cluster.getSolrClient();
+ for (int docId = 1; docId <= numDocs; docId++) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", docId);
+ solrClient.add(collection, doc);
+ }
+ solrClient.commit(collection);
+ Thread.sleep(5000);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java
index b0188069562..4c0ad7af750 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java
@@ -122,7 +122,7 @@ public class OutOfBoxZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
.withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
.build();
try {
- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(
+ VMParamsZkACLAndCredentialsProvidersTest.doTest(
zkClient, true, true, true, true, true, true, true, true, true, true);
} finally {
zkClient.close();
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java
index 5810c50fe17..2f3f2201ede 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java
@@ -153,7 +153,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
try (SolrZkClient zkClient =
new SolrZkClientFactoryUsingCompletelyNewProviders(null, null, null, null)
.getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) {
- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(
+ VMParamsZkACLAndCredentialsProvidersTest.doTest(
zkClient, false, false, false, false, false, false, false, false, false, false);
}
}
@@ -165,7 +165,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
new SolrZkClientFactoryUsingCompletelyNewProviders(
"connectAndAllACLUsername", "connectAndAllACLPasswordWrong", null, null)
.getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) {
- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(
+ VMParamsZkACLAndCredentialsProvidersTest.doTest(
zkClient, false, false, false, false, false, false, false, false, false, false);
}
}
@@ -176,7 +176,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
new SolrZkClientFactoryUsingCompletelyNewProviders(
"connectAndAllACLUsername", "connectAndAllACLPassword", null, null)
.getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) {
- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(
+ VMParamsZkACLAndCredentialsProvidersTest.doTest(
zkClient, true, true, true, true, true, true, true, true, true, true);
}
}
@@ -188,7 +188,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
new SolrZkClientFactoryUsingCompletelyNewProviders(
"readonlyACLUsername", "readonlyACLPassword", null, null)
.getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) {
- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(
+ VMParamsZkACLAndCredentialsProvidersTest.doTest(
zkClient, true, true, false, false, false, false, false, false, false, false);
}
}
@@ -204,7 +204,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
new SolrZkClient.Builder()
.withUrl(zkServer.getZkAddress())
.withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) {
- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(
+ VMParamsZkACLAndCredentialsProvidersTest.doTest(
zkClient, false, false, false, false, false, false, false, false, false, false);
}
}
@@ -220,7 +220,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
new SolrZkClient.Builder()
.withUrl(zkServer.getZkAddress())
.withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) {
- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(
+ VMParamsZkACLAndCredentialsProvidersTest.doTest(
zkClient, false, false, false, false, false, false, false, false, false, false);
}
}
@@ -236,7 +236,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
new SolrZkClient.Builder()
.withUrl(zkServer.getZkAddress())
.withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) {
- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(
+ VMParamsZkACLAndCredentialsProvidersTest.doTest(
zkClient, true, true, true, true, true, true, true, true, true, true);
}
}
@@ -252,7 +252,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
new SolrZkClient.Builder()
.withUrl(zkServer.getZkAddress())
.withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) {
- AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest(
+ VMParamsZkACLAndCredentialsProvidersTest.doTest(
zkClient, true, true, false, false, false, false, false, false, false, false);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
index 6394eb566e0..5ea4794ff4b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
@@ -16,16 +16,143 @@
*/
package org.apache.solr.cloud;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.apache.CloudLegacySolrClient;
+import org.apache.solr.client.solrj.apache.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.junit.After;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-/*
- * Implementation moved to AbstractRecoveryZkTestBase as it is used by HDFS contrib tests.
- */
-public class RecoveryZkTest extends AbstractRecoveryZkTestBase {
+public class RecoveryZkTest extends SolrCloudTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ cluster = configureCluster(2).addConfig("conf", configset("cloud-minimal")).configure();
+ }
+
+ private final List<StoppableIndexingThread> threads = new ArrayList<>();
+
+ @After
+ public void stopThreads() throws InterruptedException {
+ for (StoppableIndexingThread t : threads) {
+ t.safeStop();
+ }
+ for (StoppableIndexingThread t : threads) {
+ t.join();
+ }
+ threads.clear();
+ }
@Test
- @Override
public void test() throws Exception {
- super.test();
+
+ final String collection = "recoverytest";
+
+ CollectionAdminRequest.createCollection(collection, "conf", 1, 2)
+ .process(cluster.getSolrClient());
+ waitForState(
+ "Expected a collection with one shard and two replicas", collection, clusterShape(1, 2));
+
+ // start a couple indexing threads
+
+ int[] maxDocList = new int[] {300, 700, 1200, 1350, 3000};
+ int[] maxDocNightlyList = new int[] {3000, 7000, 12000, 30000, 45000, 60000};
+
+ int maxDoc;
+ if (!TEST_NIGHTLY) {
+ maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];
+ } else {
+ maxDoc = maxDocNightlyList[random().nextInt(maxDocList.length - 1)];
+ }
+ log.info("Indexing {} documents", maxDoc);
+
+ try (SolrClient solrClient =
+ cluster.basicSolrClientBuilder().withDefaultCollection(collection).build(); ) {
+ final StoppableIndexingThread indexThread =
+ new StoppableIndexingThread(null, solrClient, "1", true, maxDoc, 1, true);
+ threads.add(indexThread);
+ indexThread.start();
+
+ final StoppableIndexingThread indexThread2 =
+ new StoppableIndexingThread(null, solrClient, "2", true, maxDoc, 1, true);
+ threads.add(indexThread2);
+ indexThread2.start();
+
+ // give some time to index...
+ int[] waitTimes = new int[] {200, 2000, 3000};
+ Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
+
+ // bring shard replica down
+ DocCollection state = getCollectionState(collection);
+ Replica leader = state.getLeader("shard1");
+ Replica replica = getRandomReplica(state.getSlice("shard1"), (r) -> !leader.equals(r));
+
+ JettySolrRunner jetty = cluster.getReplicaJetty(replica);
+ jetty.stop();
+
+ // wait a moment - lets allow some docs to be indexed so replication time is non 0
+ Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
+
+ // bring shard replica up
+ jetty.start();
+
+ // make sure replication can start
+ Thread.sleep(3000);
+
+ // stop indexing threads
+ indexThread.safeStop();
+ indexThread2.safeStop();
+
+ indexThread.join();
+ indexThread2.join();
+
+ new UpdateRequest().commit(solrClient, collection);
+
+ cluster
+ .getZkStateReader()
+ .waitForState(collection, 120, TimeUnit.SECONDS, clusterShape(1, 2));
+
+ // test that leader and replica have same doc count
+ state = getCollectionState(collection);
+ assertShardConsistency(state.getSlice("shard1"), true);
+ }
+ }
+
+ private void assertShardConsistency(Slice shard, boolean expectDocs) throws Exception {
+ List<Replica> replicas = shard.getReplicas(r -> r.getState() == Replica.State.ACTIVE);
+ long[] numCounts = new long[replicas.size()];
+ int i = 0;
+ for (Replica replica : replicas) {
+ try (var client =
+ new HttpSolrClient.Builder(replica.getBaseUrl())
+ .withDefaultCollection(replica.getCoreName())
+ .withHttpClient(((CloudLegacySolrClient) cluster.getSolrClient()).getHttpClient())
+ .build()) {
+ numCounts[i] =
+ client.query(new SolrQuery("*:*").add("distrib", "false")).getResults().getNumFound();
+ i++;
+ }
+ }
+ for (int j = 1; j < replicas.size(); j++) {
+ if (numCounts[j] != numCounts[j - 1])
+ fail("Mismatch in counts between replicas"); // TODO improve this!
+ if (numCounts[j] == 0 && expectDocs)
+ fail("Expected docs on shard " + shard.getName() + " but found none");
+ }
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java
index 1bc74fff254..22b0084e989 100644
--- a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java
@@ -16,23 +16,186 @@
*/
package org.apache.solr.cloud;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
-/**
- * Implementation moved to AbstractRestartWhileUpdatingTestBase because it is used by HDFS contrib
- * module tests
- */
@Nightly
-public class RestartWhileUpdatingTest extends AbstractRestartWhileUpdatingTestBase {
+public class RestartWhileUpdatingTest extends AbstractFullDistribZkTestBase {
+
+ // private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain";
+ private List<StoppableIndexingThread> threads;
+
+ private volatile boolean stopExpire = false;
public RestartWhileUpdatingTest() throws Exception {
super();
+ sliceCount = 1;
+ fixShardCount(3);
+ schemaString = "schema15.xml"; // we need a string id
+ useFactory("solr.StandardDirectoryFactory");
}
- @Test
+ public static String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};
+ public static RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};
+
@Override
+ protected String[] getFieldNames() {
+ return fieldNames;
+ }
+
+ @Override
+ protected RandVal[] getRandValues() {
+ return randVals;
+ }
+
+ @BeforeClass
+ public static void beforeRestartWhileUpdatingTest() {
+ System.setProperty("leaderVoteWait", "300000");
+ System.setProperty("solr.autoCommit.maxTime", "30000");
+ System.setProperty("solr.autoSoftCommit.maxTime", "3000");
+ // SOLR-13212 // TestInjection.nonGracefullClose = "true:60";
+ // SOLR-13189 // TestInjection.failReplicaRequests = "true:03";
+ }
+
+ @AfterClass
+ public static void afterRestartWhileUpdatingTest() {
+ System.clearProperty("leaderVoteWait");
+ System.clearProperty("solr.autoCommit.maxTime");
+ System.clearProperty("solr.autoSoftCommit.maxTime");
+ }
+
+ @Test
public void test() throws Exception {
- super.test();
+ handle.clear();
+ handle.put("timestamp", SKIPVAL);
+
+ // start a couple indexing threads
+
+ int[] maxDocList = new int[] {5000, 10000};
+
+ int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];
+
+ int numThreads = random().nextInt(4) + 1;
+
+ threads = new ArrayList<>(numThreads);
+
+ Thread expireThread =
+ new Thread("expireThread") {
+ @Override
+ public void run() {
+ while (!stopExpire) {
+ try {
+ Thread.sleep(random().nextInt(15000));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ // try {
+ // chaosMonkey.expireRandomSession();
+ // } catch (KeeperException e) {
+ // throw new RuntimeException(e);
+ // } catch (InterruptedException e) {
+ // throw new RuntimeException(e);
+ // }
+ }
+ }
+ };
+
+ // Currently unused
+ // expireThread.start();
+
+ StoppableIndexingThread indexThread;
+ for (int i = 0; i < numThreads; i++) {
+ indexThread =
+ new StoppableIndexingThread(
+ controlClient, cloudClient, Integer.toString(i), true, maxDoc, 1, true);
+ threads.add(indexThread);
+ indexThread.start();
+ }
+
+ Thread.sleep(2000);
+
+ int restartTimes = 1; // random().nextInt(4) + 1;;
+ for (int i = 0; i < restartTimes; i++) {
+ Thread.sleep(random().nextInt(30000));
+ stopAndStartAllReplicas();
+ Thread.sleep(random().nextInt(30000));
+ }
+
+ Thread.sleep(2000);
+
+ // stop indexing threads
+ for (StoppableIndexingThread thread : threads) {
+ thread.safeStop();
+ }
+ stopExpire = true;
+ expireThread.join();
+
+ Thread.sleep(1000);
+
+ waitForThingsToLevelOut(320, TimeUnit.SECONDS);
+
+ Thread.sleep(2000);
+
+ waitForThingsToLevelOut(30, TimeUnit.SECONDS);
+
+ Thread.sleep(5000);
+
+ waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true);
+
+ for (StoppableIndexingThread thread : threads) {
+ thread.join();
+ }
+
+ checkShardConsistency(false, false);
+ }
+
+ public void stopAndStartAllReplicas() throws Exception, InterruptedException {
+ chaosMonkey.stopAll(random().nextInt(1));
+
+ if (random().nextBoolean()) {
+ for (StoppableIndexingThread thread : threads) {
+ thread.safeStop();
+ }
+ }
+ Thread.sleep(1000);
+
+ chaosMonkey.startAll();
+ }
+
+ @Override
+ protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException {
+ cloudClient.add(doc);
+ }
+
+ @Override
+ public void distribTearDown() throws Exception {
+ // make sure threads have been stopped...
+ if (threads != null) {
+ for (StoppableIndexingThread thread : threads) {
+ thread.safeStop();
+ thread.safeStop();
+ }
+ }
+
+ super.distribTearDown();
+ }
+
+ // skip the randoms - they can deadlock...
+ @Override
+ protected void indexr(Object... fields) throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ addFields(doc, fields);
+ addFields(doc, "rnd_b", true);
+ indexDoc(doc);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
index bdff2a6a477..8f49afba970 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
@@ -16,8 +16,285 @@
*/
package org.apache.solr.cloud;
-/**
- * Test sync phase that occurs when Leader goes down and a new Leader is elected. Implementation
- * moved to AbstractSyncSliceTestBase.java as it is also used by the HDFS contrib tests.
- */
-public class SyncSliceTest extends AbstractSyncSliceTestBase {}
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.apache.HttpSolrClient;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.util.LogLevel;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Test sync phase that occurs when Leader goes down and a new Leader is elected. */
+@LogLevel("org.apache.solr.update.processor.DistributedZkUpdateProcessor=WARN")
+public class SyncSliceTest extends AbstractFullDistribZkTestBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private boolean success = false;
+
+ @Override
+ public void distribTearDown() throws Exception {
+ if (!success) {
+ printLayoutOnTearDown = true;
+ }
+ super.distribTearDown();
+ }
+
+ public SyncSliceTest() {
+ super();
+ sliceCount = 1;
+ fixShardCount(TEST_NIGHTLY ? 7 : 4);
+ }
+
+ @Test
+ public void test() throws Exception {
+
+ handle.clear();
+ handle.put("timestamp", SKIPVAL);
+
+ waitForThingsToLevelOut(30, TimeUnit.SECONDS);
+
+ del("*:*");
+ List<CloudJettyRunner> skipServers = new ArrayList<>();
+ int docId = 0;
+ indexDoc(
+ skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");
+
+ indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "old haven was blue.");
+
+ skipServers.add(shardToJetty.get("shard1").get(1));
+
+ indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "but the song was fancy.");
+
+ skipServers.add(shardToJetty.get("shard1").get(2));
+
+ indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "under the moon and over the lake");
+
+ commit();
+
+ waitForRecoveriesToFinish(false);
+
+ // shard should be inconsistent
+ String shardFailMessage = checkShardConsistency("shard1", true, false);
+ assertNotNull(shardFailMessage);
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionAction.SYNCSHARD.toString());
+ params.set("collection", "collection1");
+ params.set("shard", "shard1");
+ final var request =
+ new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/collections", params);
+
+ String baseUrl = shardToJetty.get(SHARD1).get(2).jetty.getBaseUrl().toString();
+
+ // we only set the connect timeout, not so timeout
+ try (SolrClient baseClient =
+ new HttpSolrClient.Builder(baseUrl)
+ .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)
+ .build()) {
+ baseClient.request(request);
+ }
+
+ waitForThingsToLevelOut(15, TimeUnit.SECONDS);
+
+ checkShardConsistency(false, true);
+
+ long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
+ assertEquals(4, cloudClientDocs);
+
+ // kill the leader - new leader could have all the docs or be missing one
+ CloudJettyRunner leaderJetty = shardToLeaderJetty.get("shard1");
+
+ skipServers = getRandomOtherJetty(leaderJetty, null); // but not the leader
+
+ // this doc won't be on one node
+ indexDoc(
+ skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");
+ commit();
+
+ Set<CloudJettyRunner> jetties = new HashSet<>();
+ jetties.addAll(shardToJetty.get("shard1"));
+ jetties.remove(leaderJetty);
+ assertEquals(getShardCount() - 1, jetties.size());
+
+ leaderJetty.jetty.stop();
+
+ Thread.sleep(3000);
+
+ waitForNoShardInconsistency();
+
+ Thread.sleep(1000);
+
+ checkShardConsistency(false, true);
+
+ cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
+ assertEquals(5, cloudClientDocs);
+
+ CloudJettyRunner deadJetty = leaderJetty;
+
+ // let's get the latest leader
+ while (deadJetty.equals(leaderJetty)) {
+ updateMappingsFromZk(this.jettys, this.clients);
+ leaderJetty = shardToLeaderJetty.get("shard1");
+ }
+
+ // bring back dead node
+ deadJetty.jetty.start(); // he is not the leader anymore
+
+ waitTillAllNodesActive();
+
+ skipServers = getRandomOtherJetty(leaderJetty, deadJetty);
+ skipServers.addAll(getRandomOtherJetty(leaderJetty, deadJetty));
+ // skip list should be
+
+ // System.out.println("leader:" + leaderJetty.url);
+ // System.out.println("dead:" + deadJetty.url);
+ // System.out.println("skip list:" + skipServers);
+
+ // we are skipping 2 nodes
+ assertEquals(2, skipServers.size());
+
+ // more docs than can peer sync
+ for (int i = 0; i < 300; i++) {
+ indexDoc(
+ skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");
+ }
+
+ commit();
+ waitForRecoveriesToFinish(false);
+
+ // shard should be inconsistent
+ shardFailMessage = waitTillInconsistent();
+ assertNotNull(
+ "Test Setup Failure: shard1 should have just been set up to be inconsistent - but it's still consistent. Leader:"
+ + leaderJetty.url
+ + " Dead Guy:"
+ + deadJetty.url
+ + "skip list:"
+ + skipServers,
+ shardFailMessage);
+
+ // good place to test compareResults
+ boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient);
+ assertTrue("A test that compareResults is working correctly failed", shouldFail);
+
+ jetties = new HashSet<>();
+ jetties.addAll(shardToJetty.get("shard1"));
+ jetties.remove(leaderJetty);
+ assertEquals(getShardCount() - 1, jetties.size());
+
+ // kill the current leader
+ leaderJetty.jetty.stop();
+
+ waitForNoShardInconsistency();
+
+ checkShardConsistency(true, true);
+
+ success = true;
+ }
+
+ private void waitTillAllNodesActive() throws InterruptedException, TimeoutException {
+ ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
+
+ zkStateReader.waitForState(
+ "collection1",
+ 3,
+ TimeUnit.MINUTES,
+ (n, c) -> {
+ Collection<Replica> replicas = c.getSlice("shard1").getReplicas();
+ Set<String> nodes =
+ replicas.stream().map(Replica::getNodeName).collect(Collectors.toSet());
+ return replicas.stream().map(Replica::getState).allMatch(Replica.State.ACTIVE::equals)
+ && n.containsAll(nodes);
+ });
+ }
+
+ private String waitTillInconsistent() throws Exception, InterruptedException {
+ String shardFailMessage = null;
+
+ shardFailMessage = pollConsistency(shardFailMessage, 0);
+ shardFailMessage = pollConsistency(shardFailMessage, 3000);
+ shardFailMessage = pollConsistency(shardFailMessage, 5000);
+ shardFailMessage = pollConsistency(shardFailMessage, 15000);
+
+ return shardFailMessage;
+ }
+
+ private String pollConsistency(String shardFailMessage, int sleep)
+ throws InterruptedException, Exception {
+ try {
+ commit();
+ } catch (Throwable t) {
+ log.error("commit error", t);
+ }
+ if (shardFailMessage == null) {
+ // try again
+ Thread.sleep(sleep);
+ shardFailMessage = checkShardConsistency("shard1", true, false);
+ }
+ return shardFailMessage;
+ }
+
+ private List<CloudJettyRunner> getRandomOtherJetty(
+ CloudJettyRunner leader, CloudJettyRunner down) {
+ List<CloudJettyRunner> skipServers = new ArrayList<>();
+ List<CloudJettyRunner> candidates = new ArrayList<>();
+ candidates.addAll(shardToJetty.get("shard1"));
+
+ if (leader != null) {
+ candidates.remove(leader);
+ }
+
+ if (down != null) {
+ candidates.remove(down);
+ }
+
+ CloudJettyRunner cjetty = candidates.get(random().nextInt(candidates.size()));
+ skipServers.add(cjetty);
+ return skipServers;
+ }
+
+ protected void indexDoc(List<CloudJettyRunner> skipServers, Object... fields)
+ throws IOException, SolrServerException {
+ SolrInputDocument doc = new SolrInputDocument();
+
+ addFields(doc, fields);
+ addFields(doc, "rnd_b", true);
+
+ controlClient.add(doc);
+
+ UpdateRequest ureq = new UpdateRequest();
+ ureq.add(doc);
+ for (CloudJettyRunner skip : skipServers) {
+ ureq.getParams().add("test.distrib.skip.servers", skip.url + "/");
+ }
+ ureq.process(cloudClient);
+ }
+
+ // skip the randoms - they can deadlock...
+ @Override
+ protected void indexr(Object... fields) throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ addFields(doc, fields);
+ addFields(doc, "rnd_b", true);
+ indexDoc(doc);
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java b/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java
index e056c40a893..2fbd5128177 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java
@@ -16,26 +16,134 @@
*/
package org.apache.solr.cloud;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.util.TestInjection;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
@Nightly
@SuppressSSL
-/*
- * Implementation moved to AbstractTlogReplayBufferedWhileIndexingTestBase as it is also used by HDFS contrib
- * module tests
- */
-public class TlogReplayBufferedWhileIndexingTest
- extends AbstractTlogReplayBufferedWhileIndexingTestBase {
+public class TlogReplayBufferedWhileIndexingTest extends AbstractFullDistribZkTestBase {
+
+ private List<StoppableIndexingThread> threads;
public TlogReplayBufferedWhileIndexingTest() throws Exception {
super();
+ sliceCount = 1;
+ fixShardCount(2);
+ schemaString = "schema15.xml"; // we need a string id
+ }
+
+ @BeforeClass
+ public static void beforeRestartWhileUpdatingTest() throws Exception {
+ System.setProperty("leaderVoteWait", "300000");
+ System.setProperty("solr.autoCommit.maxTime", "10000");
+ System.setProperty("solr.autoSoftCommit.maxTime", "3000");
+ TestInjection.updateLogReplayRandomPause = "true:10";
+ TestInjection.updateRandomPause = "true:10";
+ useFactory("solr.StandardDirectoryFactory");
+ }
+
+ @AfterClass
+ public static void afterRestartWhileUpdatingTest() {
+ System.clearProperty("leaderVoteWait");
+ System.clearProperty("solr.autoCommit.maxTime");
+ System.clearProperty("solr.autoSoftCommit.maxTime");
}
@Test
- @Override
public void test() throws Exception {
- super.test();
+ handle.clear();
+ handle.put("timestamp", SKIPVAL);
+
+ waitForRecoveriesToFinish(false);
+
+ int numThreads = 3;
+
+ threads = new ArrayList<>(numThreads);
+
+ ArrayList<JettySolrRunner> allJetty = new ArrayList<>();
+ allJetty.addAll(jettys);
+ allJetty.remove(shardToLeaderJetty.get("shard1").jetty);
+ assert allJetty.size() == 1 : allJetty.size();
+ allJetty.get(0).stop();
+
+ StoppableIndexingThread indexThread;
+ for (int i = 0; i < numThreads; i++) {
+ boolean pauseBetweenUpdates = random().nextBoolean();
+ int batchSize = random().nextInt(4) + 1;
+ indexThread =
+ new StoppableIndexingThread(
+ controlClient,
+ cloudClient,
+ Integer.toString(i),
+ true,
+ 900,
+ batchSize,
+ pauseBetweenUpdates);
+ threads.add(indexThread);
+ indexThread.start();
+ }
+
+ Thread.sleep(2000);
+
+ allJetty.get(0).start();
+
+ Thread.sleep(45000);
+
+ waitForThingsToLevelOut(); // we can insert random update delays, so this can take a while,
+ // especially when beasting this test
+
+ Thread.sleep(2000);
+
+ waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true);
+
+ for (StoppableIndexingThread thread : threads) {
+ thread.safeStop();
+ }
+
+ waitForThingsToLevelOut(30, TimeUnit.SECONDS);
+
+ checkShardConsistency(false, false);
+ }
+
+ @Override
+ protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException {
+ cloudClient.add(doc);
+ }
+
+ @Override
+ public void distribTearDown() throws Exception {
+ // make sure threads have been stopped...
+ if (threads != null) {
+ for (StoppableIndexingThread thread : threads) {
+ thread.safeStop();
+ }
+
+ for (StoppableIndexingThread thread : threads) {
+ thread.join();
+ }
+ }
+
+ super.distribTearDown();
+ }
+
+ // skip the randoms - they can deadlock...
+ @Override
+ protected void indexr(Object... fields) throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ addFields(doc, fields);
+ addFields(doc, "rnd_b", true);
+ indexDoc(doc);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
index 30d3f884b97..7e66787a4a5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
@@ -16,12 +16,450 @@
*/
package org.apache.solr.cloud;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.apache.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrPaths;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.util.TestInjection;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This test simply does a bunch of basic things in solrcloud mode and asserts things work as
- * expected. Implementation moved to AbstractUnloadDistributedZkTestBase as it is used by HDFS
- * contrib module tests.
+ * expected.
*/
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
-public class UnloadDistributedZkTest extends AbstractUnloadDistributedZkTestBase {}
+public class UnloadDistributedZkTest extends AbstractFullDistribZkTestBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public UnloadDistributedZkTest() {
+ super();
+ fixShardCount(4); // needs at least 4 servers
+ }
+
+ @Override
+ protected String getSolrXml() {
+ return "solr.xml";
+ }
+
+ @Test
+ public void test() throws Exception {
+ jettys.forEach(
+ j -> {
+ Set<Path> allowPath = j.getCoreContainer().getAllowPaths();
+ allowPath.clear();
+ allowPath.add(SolrPaths.ALL_PATH); // Allow non-standard core instance path
+ });
+ log.info("###Starting testCoreUnloadAndLeaders");
+ testCoreUnloadAndLeaders(); // long
+ log.info("###Starting testUnloadOfCores");
+ testUnloadLotsOfCores(); // long
+ log.info("###Starting testUnloadShardAndCollection");
+ testUnloadShardAndCollection();
+ }
+
+ /**
+ * @param url a Solr node base URL. Should <em>not</em> contain a core or collection name.
+ */
+ private SolrClient newSolrClient(String url) {
+ return new HttpSolrClient.Builder(url)
+ .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)
+ .withSocketTimeout(30000, TimeUnit.MILLISECONDS)
+ .build();
+ }
+
+ private void checkCoreNamePresenceAndSliceCount(
+ String collectionName, String coreName, boolean shouldBePresent, int expectedSliceCount)
+ throws Exception {
+ ZkStateReader reader = ZkStateReader.from(cloudClient);
+ try {
+ reader.waitForState(
+ collectionName,
+ 45,
+ TimeUnit.SECONDS,
+ c -> {
+ final Collection<Slice> slices = (c != null) ? c.getSlices() : Collections.emptyList();
+ if (expectedSliceCount == slices.size()) {
+ for (Slice slice : slices) {
+ for (Replica replica : slice.getReplicas()) {
+ if (coreName.equals(replica.get("core"))) {
+ return shouldBePresent;
+ }
+ }
+ }
+ return !shouldBePresent;
+ } else {
+ return false;
+ }
+ });
+ } catch (TimeoutException e) {
+ printLayout();
+ fail(
+ "checkCoreNamePresenceAndSliceCount failed:"
+ + " collection="
+ + collectionName
+ + " CoreName="
+ + coreName
+ + " shouldBePresent="
+ + shouldBePresent);
+ }
+ }
+
+ private void testUnloadShardAndCollection() throws Exception {
+ final int numShards = 2;
+
+ final String collection = "test_unload_shard_and_collection";
+
+ final String coreName1 = collection + "_1";
+ final String coreName2 = collection + "_2";
+
+ assertEquals(
+ 0,
+ CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1)
+ .setCreateNodeSet("")
+ .process(cloudClient)
+ .getStatus());
+ assertTrue(
+ CollectionAdminRequest.addReplicaToShard(collection, "shard1")
+ .setCoreName(coreName1)
+ .setNode(jettys.get(0).getNodeName())
+ .process(cloudClient)
+ .isSuccess());
+
+ assertTrue(
+ CollectionAdminRequest.addReplicaToShard(collection, "shard2")
+ .setCoreName(coreName2)
+ .setNode(jettys.get(0).getNodeName())
+ .process(cloudClient)
+ .isSuccess());
+
+ // does not mean they are active and up yet :*
+ waitForRecoveriesToFinish(collection, false);
+
+ final boolean unloadInOrder = random().nextBoolean();
+ final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2);
+ final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1);
+
+ try (SolrClient adminClient = getHttpSolrClient(buildUrl(jettys.get(0).getLocalPort()))) {
+ // now unload one of the two
+ Unload unloadCmd = new Unload(false);
+ unloadCmd.setCoreName(unloadCmdCoreName1);
+ adminClient.request(unloadCmd);
+
+ // there should still be two shards (as of SOLR-5209)
+ checkCoreNamePresenceAndSliceCount(
+ collection,
+ unloadCmdCoreName1,
+ false /* shouldBePresent */,
+ numShards /* expectedSliceCount */);
+
+ // now unload one of the other
+ unloadCmd = new Unload(false);
+ unloadCmd.setCoreName(unloadCmdCoreName2);
+ adminClient.request(unloadCmd);
+ checkCoreNamePresenceAndSliceCount(
+ collection,
+ unloadCmdCoreName2,
+ false /* shouldBePresent */,
+ numShards /* expectedSliceCount */);
+ }
+
+ // printLayout();
+ // the collection should still be present (as of SOLR-5209 replica removal does not cascade to
+ // remove the slice and collection)
+ getCommonCloudSolrClient();
+ assertTrue(
+ "No longer found collection " + collection,
+ cloudClient.getClusterState().hasCollection(collection));
+ }
+
+ protected SolrCore getFirstCore(String collection, JettySolrRunner jetty) {
+ SolrCore solrCore = null;
+ for (SolrCore core : jetty.getCoreContainer().getCores()) {
+ if (core.getName().startsWith(collection)) {
+ solrCore = core;
+ }
+ }
+ return solrCore;
+ }
+
+ /**
+ * @throws Exception on any problem
+ */
+ private void testCoreUnloadAndLeaders() throws Exception {
+ JettySolrRunner jetty1 = jettys.get(0);
+
+ assertEquals(
+ 0,
+ CollectionAdminRequest.createCollection("unloadcollection", "conf1", 1, 1)
+ .setCreateNodeSet(jetty1.getNodeName())
+ .process(cloudClient)
+ .getStatus());
+ getCommonCloudSolrClient();
+ ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
+
+ zkStateReader.forceUpdateCollection("unloadcollection");
+
+ int slices =
+ zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size();
+ assertEquals(1, slices);
+ SolrCore solrCore = getFirstCore("unloadcollection", jetty1);
+ String core1DataDir = solrCore.getDataDir();
+
+ assertTrue(
+ CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")
+ .setCoreName("unloadcollection_shard1_replica2")
+ .setNode(jettys.get(1).getNodeName())
+ .process(cloudClient)
+ .isSuccess());
+ zkStateReader.forceUpdateCollection("unloadcollection");
+ slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size();
+ assertEquals(1, slices);
+
+ waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
+
+ Replica leader = getLeaderFromZk("unloadcollection", "shard1");
+
+ Random random = random();
+ if (random.nextBoolean()) {
+ try (SolrClient collectionClient =
+ getHttpSolrClient(leader.getBaseUrl(), leader.getCoreName())) {
+ // lets try and use the solrj client to index and retrieve a couple
+ // documents
+ SolrInputDocument doc1 =
+ getDoc(id, 6, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");
+ SolrInputDocument doc2 =
+ getDoc(id, 7, i1, -600, tlong, 600, t1, "humpty dumpy3 sat on a walls");
+ SolrInputDocument doc3 =
+ getDoc(id, 8, i1, -600, tlong, 600, t1, "humpty dumpy2 sat on a walled");
+ collectionClient.add(doc1);
+ collectionClient.add(doc2);
+ collectionClient.add(doc3);
+ collectionClient.commit();
+ }
+ }
+
+ assertTrue(
+ CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")
+ .setCoreName("unloadcollection_shard1_replica3")
+ .setNode(jettys.get(2).getNodeName())
+ .process(cloudClient)
+ .isSuccess());
+
+ waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
+
+ // so that we start with some versions when we reload...
+ TestInjection.skipIndexWriterCommitOnClose = true;
+
+ try (SolrClient addClient =
+ new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString())
+ .withDefaultCollection("unloadcollection_shard1_replica3")
+ .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)
+ .build()) {
+
+ // add a few docs
+ for (int x = 20; x < 100; x++) {
+ SolrInputDocument doc1 =
+ getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");
+ addClient.add(doc1);
+ }
+ }
+ // don't commit so they remain in the tran log
+ // collectionClient.commit();
+
+ // unload the leader
+ try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) {
+
+ Unload unloadCmd = new Unload(false);
+ unloadCmd.setCoreName(leader.getCoreName());
+ ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams();
+
+ collectionClient.request(unloadCmd);
+ }
+ // Thread.currentThread().sleep(500);
+ // printLayout();
+
+ int tries = 50;
+ while (leader
+ .getCoreUrl()
+ .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
+ Thread.sleep(100);
+ if (tries-- == 0) {
+ fail("Leader never changed");
+ }
+ }
+
+ // ensure there is a leader
+ zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
+
+ try (SolrClient addClient =
+ new HttpSolrClient.Builder(jettys.get(1).getBaseUrl().toString())
+ .withDefaultCollection("unloadcollection_shard1_replica2")
+ .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)
+ .withSocketTimeout(90000, TimeUnit.MILLISECONDS)
+ .build()) {
+
+ // add a few docs while the leader is down
+ for (int x = 101; x < 200; x++) {
+ SolrInputDocument doc1 =
+ getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");
+ addClient.add(doc1);
+ }
+ }
+
+ assertTrue(
+ CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")
+ .setCoreName("unloadcollection_shard1_replica4")
+ .setNode(jettys.get(3).getNodeName())
+ .process(cloudClient)
+ .isSuccess());
+
+ waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
+
+ // unload the leader again
+ leader = getLeaderFromZk("unloadcollection", "shard1");
+ try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) {
+
+ Unload unloadCmd = new Unload(false);
+ unloadCmd.setCoreName(leader.getCoreName());
+ collectionClient.request(unloadCmd);
+ }
+ tries = 50;
+ while (leader
+ .getCoreUrl()
+ .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
+ Thread.sleep(100);
+ if (tries-- == 0) {
+ fail("Leader never changed");
+ }
+ }
+
+ zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
+
+ TestInjection.skipIndexWriterCommitOnClose = false; // set this back
+ assertTrue(
+ CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")
+ .setCoreName(leader.getCoreName())
+ .setDataDir(core1DataDir)
+ .setNode(leader.getNodeName())
+ .process(cloudClient)
+ .isSuccess());
+
+ waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
+
+ long found1, found3;
+
+ try (SolrClient adminClient =
+ newSolrClient((jettys.get(1).getBaseUrl() + "/unloadcollection_shard1_replica2"))) {
+ adminClient.commit();
+ SolrQuery q = new SolrQuery("*:*");
+ q.set("distrib", false);
+ found1 = adminClient.query(q).getResults().getNumFound();
+ }
+
+ try (SolrClient adminClient =
+ new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString())
+ .withDefaultCollection("unloadcollection_shard1_replica3")
+ .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)
+ .withSocketTimeout(30000, TimeUnit.MILLISECONDS)
+ .build()) {
+ adminClient.commit();
+ SolrQuery q = new SolrQuery("*:*");
+ q.set("distrib", false);
+ found3 = adminClient.query(q).getResults().getNumFound();
+ }
+
+ try (SolrClient adminClient =
+ newSolrClient((jettys.get(3).getBaseUrl() + "/unloadcollection_shard1_replica4"))) {
+ adminClient.commit();
+ SolrQuery q = new SolrQuery("*:*");
+ q.set("distrib", false);
+ long found4 = adminClient.query(q).getResults().getNumFound();
+
+ // all 3 shards should now have the same number of docs
+ assertEquals(found1, found3);
+ assertEquals(found3, found4);
+ }
+ }
+
+ private void testUnloadLotsOfCores() throws Exception {
+ JettySolrRunner jetty = jettys.get(0);
+ int shards = TEST_NIGHTLY ? 2 : 1;
+ try (final SolrClient adminClient = jetty.newClient(15000, 60000)) {
+ int numReplicas = atLeast(3);
+ ThreadPoolExecutor executor =
+ new ExecutorUtil.MDCAwareThreadPoolExecutor(
+ 0,
+ Integer.MAX_VALUE,
+ 5,
+ TimeUnit.SECONDS,
+ new SynchronousQueue<>(),
+ new SolrNamedThreadFactory("testExecutor"));
+ try {
+ // create the cores
+ BasicDistributedZkTest.createCollectionInOneInstance(
+ adminClient, jetty.getNodeName(), executor, "multiunload", shards, numReplicas);
+ } finally {
+ ExecutorUtil.shutdownAndAwaitTermination(executor);
+ }
+
+ if (TEST_NIGHTLY == false) {
+ // with nightly tests, we can try doing the unloads before the creates are done
+ // it still works, but takes much longer since we end up waiting for a timeout
+ waitForRecoveriesToFinish("multiunload", false);
+ }
+
+ executor =
+ new ExecutorUtil.MDCAwareThreadPoolExecutor(
+ 0,
+ Integer.MAX_VALUE,
+ 5,
+ TimeUnit.SECONDS,
+ new SynchronousQueue<>(),
+ new SolrNamedThreadFactory("testExecutor"));
+ try {
+ for (int j = 0; j < numReplicas; j++) {
+ final int freezeJ = j;
+ executor.execute(
+ () -> {
+ Unload unloadCmd = new Unload(true);
+ unloadCmd.setCoreName("multiunload" + freezeJ);
+ try {
+ adminClient.request(unloadCmd);
+ } catch (SolrServerException | IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ Thread.sleep(random().nextInt(50));
+ }
+ } finally {
+ ExecutorUtil.shutdownAndAwaitTermination(executor);
+ }
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java
index 82b60068d83..aca50b7bd3a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java
@@ -16,5 +16,552 @@
*/
package org.apache.solr.cloud;
-public class VMParamsZkACLAndCredentialsProvidersTest
- extends AbstractDigestZkACLAndCredentialsProvidersTestBase {}
+import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_FILE_VM_PARAM_NAME;
+import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME;
+import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME;
+import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.cloud.DigestZkACLProvider;
+import org.apache.solr.common.cloud.DigestZkCredentialsProvider;
+import org.apache.solr.common.cloud.SecurityAwareZkACLProvider;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.VMParamsZkCredentialsInjector;
+import org.apache.solr.common.cloud.ZkCredentialsInjector;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException.NoAuthException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final Charset DATA_ENCODING = StandardCharsets.UTF_8;
+
+ private static final String ALL_USERNAME = "connectAndAllACLUsername";
+ private static final String ALL_PASSWORD = "connectAndAllACLPassword";
+ private static final String READONLY_USERNAME = "readonlyACLUsername";
+ private static final String READONLY_PASSWORD = "readonlyACLPassword";
+
+ protected ZkTestServer zkServer;
+
+ protected Path zkDir;
+
+ @BeforeClass
+ public static void beforeClass() {
+ System.setProperty("solrcloud.skip.autorecovery", "true");
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ System.clearProperty("solrcloud.skip.autorecovery");
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ // TODO: Does all of this setup need to happen for each test case, or can it be done once for
+ // the class? (i.e. @BeforeClass) and maybe some minor reset logic in setup, instead of full
+ // startup and teardown of a new ZkTestServer in each cycle?
+ super.setUp();
+ if (log.isInfoEnabled()) {
+ log.info("####SETUP_START {}", getTestName());
+ }
+ createTempDir();
+
+ zkDir = createTempDir().resolve("zookeeper/server1/data");
+ log.info("ZooKeeper dataDir:{}", zkDir);
+ setSecuritySystemProperties();
+ zkServer = new ZkTestServer(zkDir);
+ zkServer.run(false);
+
+ System.setProperty("zkHost", zkServer.getZkAddress());
+
+ setDigestZkSystemProps();
+ System.setProperty(
+ SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,
+ AllAndReadonlyCredentialZkCredentialsInjector.class.getName());
+
+ SolrZkClient zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(zkServer.getZkHost())
+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
+ .withConnTimeOut(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
+ .build();
+
+ zkClient.makePath("/solr", false, true);
+ zkClient.close();
+
+ zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(zkServer.getZkAddress())
+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
+ .build();
+ zkClient.create(
+ "/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
+ zkClient.makePath(
+ "/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
+
+ zkClient.create(
+ SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH,
+ "content".getBytes(DATA_ENCODING),
+ CreateMode.PERSISTENT,
+ false);
+ zkClient.close();
+
+ clearSecuritySystemProperties();
+
+ zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(zkServer.getZkAddress())
+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
+ .build();
+ // Currently, no credentials on ZK connection, because those same VM-params are used for adding
+ // ACLs, and here we want
+ // no (or completely open) ACLs added. Therefore, hack your way into being authorized for
+ // creating anyway
+ zkClient
+ .getCuratorFramework()
+ .getZookeeperClient()
+ .getZooKeeper()
+ .addAuthInfo(
+ "digest", (ALL_USERNAME + ":" + ALL_PASSWORD).getBytes(StandardCharsets.UTF_8));
+ zkClient.create(
+ "/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
+ zkClient.makePath(
+ "/unprotectedMakePathNode",
+ "content".getBytes(DATA_ENCODING),
+ CreateMode.PERSISTENT,
+ false);
+ zkClient.close();
+
+ setDigestZkSystemProps();
+ if (log.isInfoEnabled()) {
+ log.info("####SETUP_END {}", getTestName());
+ }
+ }
+
+ private void setDigestZkSystemProps() {
+ System.setProperty(
+ SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME,
+ DigestZkCredentialsProvider.class.getName());
+ System.setProperty(
+ SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName());
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ zkServer.shutdown();
+
+ clearSecuritySystemProperties();
+
+ super.tearDown();
+ }
+
+ @Test
+ public void testNoCredentials() throws Exception {
+ List<TestZkCredentialsInjector> testZkCredentialsInjectors =
+ List.of(
+ new TestZkCredentialsInjector(NoCredentialZkCredentialsInjector.class),
+ new TestZkCredentialsInjector(VMParamsZkCredentialsInjector.class));
+
+ testInjectors(
+ testZkCredentialsInjectors,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false);
+ }
+
+ @Test
+ public void testWrongCredentials() throws Exception {
+ List<TestZkCredentialsInjector> testZkCredentialsInjectors =
+ List.of(
+ new TestZkCredentialsInjector(WrongAllCredentialZkCredentialsInjector.class),
+ new TestZkCredentialsInjector(
+ VMParamsZkCredentialsInjector.class,
+ List.of(
+ DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),
+ List.of(ALL_USERNAME, ALL_PASSWORD + "Wrong")));
+
+ testInjectors(
+ testZkCredentialsInjectors,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false);
+ }
+
+ @Test
+ public void testAllCredentials() throws Exception {
+ List<TestZkCredentialsInjector> testZkCredentialsInjectors =
+ List.of(
+ new TestZkCredentialsInjector(AllCredentialZkCredentialsInjector.class),
+ new TestZkCredentialsInjector(
+ VMParamsZkCredentialsInjector.class,
+ List.of(
+ DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),
+ List.of(ALL_USERNAME, ALL_PASSWORD)));
+
+ testInjectors(
+ testZkCredentialsInjectors, true, true, true, true, true, true, true, true, true, true);
+ }
+
+ @Test
+ public void testReadonlyCredentials() throws Exception {
+ List<TestZkCredentialsInjector> testZkCredentialsInjectors =
+ List.of(
+ new TestZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class),
+ new TestZkCredentialsInjector(
+ VMParamsZkCredentialsInjector.class,
+ List.of(
+ DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),
+ List.of(READONLY_USERNAME, READONLY_PASSWORD)));
+ testInjectors(
+ testZkCredentialsInjectors,
+ true,
+ true,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false);
+ }
+
+ protected void testInjectors(
+ List<TestZkCredentialsInjector> testZkCredentialsInjectors,
+ boolean getData,
+ boolean list,
+ boolean create,
+ boolean setData,
+ boolean delete,
+ boolean secureGet,
+ boolean secureList,
+ boolean secureCreate,
+ boolean secureSet,
+ boolean secureDelete)
+ throws Exception {
+ for (TestZkCredentialsInjector testZkCredentialsInjector : testZkCredentialsInjectors) {
+ tearDown();
+ setUp();
+ testZkCredentialsInjector.setSystemProps();
+ try (SolrZkClient zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(zkServer.getZkAddress())
+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
+ .build()) {
+ doTest(
+ zkClient,
+ getData,
+ list,
+ create,
+ setData,
+ delete,
+ secureGet,
+ secureList,
+ secureCreate,
+ secureSet,
+ secureDelete);
+ }
+ }
+ }
+
+ @Test
+ public void testRepairACL() throws Exception {
+ clearSecuritySystemProperties();
+ try (SolrZkClient zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(zkServer.getZkAddress())
+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
+ .build()) {
+ // Currently, no credentials on ZK connection, because those same VM-params are used for
+ // adding ACLs, and here we want
+ // no (or completely open) ACLs added. Therefore, hack your way into being authorized for
+ // creating anyway
+ zkClient
+ .getCuratorFramework()
+ .getZookeeperClient()
+ .getZooKeeper()
+ .addAuthInfo(
+ "digest",
+ ("connectAndAllACLUsername:connectAndAllACLPassword")
+ .getBytes(StandardCharsets.UTF_8));
+
+ zkClient.create(
+ "/security.json", "{}".getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, false);
+ assertEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false));
+ }
+
+ setSecuritySystemProperties();
+ try (SolrZkClient zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(zkServer.getZkAddress())
+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
+ .build()) {
+ ZkController.createClusterZkNodes(zkClient);
+ assertNotEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false));
+ }
+
+ useZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class);
+ // useReadonlyCredentials();
+ try (SolrZkClient zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(zkServer.getZkAddress())
+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
+ .build()) {
+ NoAuthException e =
+ assertThrows(
+ NoAuthException.class, () -> zkClient.getData("/security.json", null, null, false));
+ assertEquals("/solr/security.json", e.getPath());
+ }
+ }
+
+ private void useZkCredentialsInjector(Class<?> zkCredentialsInjectorClass) {
+ clearSecuritySystemProperties();
+ setDigestZkSystemProps();
+ System.setProperty(
+ SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,
+ zkCredentialsInjectorClass.getName());
+ }
+
+ private void setSecuritySystemProperties() {
+ System.setProperty(
+ SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME,
+ DigestZkCredentialsProvider.class.getName());
+ System.setProperty(
+ SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName());
+ System.setProperty(
+ SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,
+ AllAndReadonlyCredentialZkCredentialsInjector.class.getName());
+ }
+
+ private void clearSecuritySystemProperties() {
+ System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME);
+ System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME);
+ System.clearProperty(SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME);
+ }
+
+ public static void doTest(
+ SolrZkClient zkClient,
+ boolean getData,
+ boolean list,
+ boolean create,
+ boolean setData,
+ boolean delete,
+ boolean secureGet,
+ boolean secureList,
+ boolean secureCreate,
+ boolean secureSet,
+ boolean secureDelete)
+ throws Exception {
+ doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete);
+ doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete);
+ doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete);
+ doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete);
+ doTest(
+ zkClient,
+ SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH,
+ secureGet,
+ secureList,
+ secureCreate,
+ secureSet,
+ secureDelete);
+ }
+
+ protected static void doTest(
+ SolrZkClient zkClient,
+ String path,
+ boolean getData,
+ boolean list,
+ boolean create,
+ boolean setData,
+ boolean delete)
+ throws Exception {
+ doTest(getData, () -> zkClient.getData(path, null, null, false));
+ doTest(list, () -> zkClient.getChildren(path, null, false));
+
+ doTest(
+ create,
+ () -> {
+ zkClient.create(path + "/subnode", null, CreateMode.PERSISTENT, false);
+ zkClient.delete(path + "/subnode", -1, false);
+ });
+ doTest(
+ create,
+ () -> {
+ zkClient.makePath(path + "/subnode/subsubnode", false);
+ zkClient.delete(path + "/subnode/subsubnode", -1, false);
+ zkClient.delete(path + "/subnode", -1, false);
+ });
+
+ doTest(setData, () -> zkClient.setData(path, (byte[]) null, false));
+
+ // Actually about the ACLs on /solr, but that is protected
+ doTest(delete, () -> zkClient.delete(path, -1, false));
+ }
+
+ interface ExceptingRunnable {
+ void run() throws Exception;
+ }
+
+ private static void doTest(boolean shouldSucceed, ExceptingRunnable action) throws Exception {
+ if (shouldSucceed) {
+ action.run();
+ } else {
+ expectThrows(NoAuthException.class, action::run);
+ }
+ }
+
+ @Test
+ public void testVMParamsAllCredentialsFromFile() throws Exception {
+ useVMParamsAllCredentialsFromFile();
+
+ try (SolrZkClient zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(zkServer.getZkAddress())
+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
+ .build()) {
+ doTest(zkClient, true, true, true, true, true, true, true, true, true, true);
+ }
+ }
+
+ @Test
+ public void testVMParamsReadonlyCredentialsFromFile() throws Exception {
+ useVMParamsReadonlyCredentialsFromFile();
+
+ try (SolrZkClient zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(zkServer.getZkAddress())
+ .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
+ .build()) {
+ doTest(zkClient, true, true, false, false, false, false, false, false, false, false);
+ }
+ }
+
+ private void useVMParamsAllCredentialsFromFile() throws IOException {
+ useVMParamsCredentialsFromFile("connectAndAllACLUsername", "connectAndAllACLPassword");
+ }
+
+ private void useVMParamsReadonlyCredentialsFromFile() throws IOException {
+ useVMParamsCredentialsFromFile("readonlyACLUsername", "readonlyACLPassword");
+ }
+
+ private void useVMParamsCredentialsFromFile(String username, String password) throws IOException {
+ Properties props = new Properties();
+ props.setProperty(
+ VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, username);
+ props.setProperty(
+ VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, password);
+ String credsFile = saveCredentialsFile(props);
+
+ useZkCredentialsInjector(VMParamsZkCredentialsInjector.class);
+ System.setProperty(DEFAULT_DIGEST_FILE_VM_PARAM_NAME, credsFile);
+ }
+
+ private String saveCredentialsFile(Properties props) throws IOException {
+ Path tmp = createTempFile("zk-creds", "properties");
+ try (FileWriter w = new FileWriter(tmp.toFile(), StandardCharsets.UTF_8)) {
+ props.store(w, "test");
+ }
+ return tmp.toAbsolutePath().toString();
+ }
+
+ public static class NoCredentialZkCredentialsInjector implements ZkCredentialsInjector {
+ @Override
+ public List<ZkCredential> getZkCredentials() {
+ return Collections.emptyList();
+ }
+ }
+
+ public static class AllAndReadonlyCredentialZkCredentialsInjector
+ implements ZkCredentialsInjector {
+ @Override
+ public List<ZkCredential> getZkCredentials() {
+ return List.of(
+ new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL),
+ new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.READ));
+ }
+ }
+
+ public static class ConnectWithReadonlyCredsInjector implements ZkCredentialsInjector {
+ @Override
+ public List<ZkCredential> getZkCredentials() {
+ return List.of(
+ new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.ALL));
+ }
+ }
+
+ public static class AllCredentialZkCredentialsInjector implements ZkCredentialsInjector {
+ @Override
+ public List<ZkCredential> getZkCredentials() {
+ return List.of(new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL));
+ }
+ }
+
+ public static class WrongAllCredentialZkCredentialsInjector implements ZkCredentialsInjector {
+ @Override
+ public List<ZkCredential> getZkCredentials() {
+ return List.of(
+ new ZkCredential(ALL_USERNAME, ALL_PASSWORD + "Wrong", ZkCredential.Perms.ALL));
+ }
+ }
+
+ class TestZkCredentialsInjector {
+ private final Class<?> zkCredentialsInjectorClass;
+ private final List<String> systemPropsKeys;
+ private final List<String> systemPropsValues;
+
+ public TestZkCredentialsInjector(Class<?> zkCredentialsInjectorClass) {
+ this(zkCredentialsInjectorClass, Collections.emptyList(), Collections.emptyList());
+ }
+
+ public TestZkCredentialsInjector(
+ Class<?> zkCredentialsInjectorClass,
+ List<String> systemPropsKeys,
+ List<String> systemPropsValues) {
+ this.zkCredentialsInjectorClass = zkCredentialsInjectorClass;
+ this.systemPropsKeys = systemPropsKeys;
+ this.systemPropsValues = systemPropsValues;
+ }
+
+ private void setSystemProps() {
+ clearSecuritySystemProperties();
+ setDigestZkSystemProps();
+ System.setProperty(
+ SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,
+ zkCredentialsInjectorClass.getName());
+ int i = 0;
+ for (String key : systemPropsKeys) {
+ System.setProperty(key, systemPropsValues.get(i++));
+ }
+ }
+ }
+}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java
deleted file mode 100644
index e758dbf26a7..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java
+++ /dev/null
@@ -1,487 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import static org.apache.solr.cloud.SolrCloudTestCase.configurePrsDefault;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.concurrent.TimeUnit;
-import org.apache.lucene.tests.mockfile.FilterPath;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.request.SolrQuery;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.handler.BackupStatusChecker;
-import org.apache.solr.handler.ReplicationHandler;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * This test simply does a bunch of basic things in solrcloud mode and asserts things work as
- * expected.
- */
-// Backups do checksum validation against a footer value not present in 'SimpleText'
[email protected]({"SimpleText"})
-public abstract class AbstractBasicDistributedZk2TestBase extends AbstractFullDistribZkTestBase {
- private static final String SHARD2 = "shard2";
- private static final String SHARD1 = "shard1";
- private static final String ONE_NODE_COLLECTION = "onenodecollection";
- private final boolean onlyLeaderIndexes = random().nextBoolean();
-
- public AbstractBasicDistributedZk2TestBase() {
- super();
- // we need DVs on point fields to compute stats & facets
- if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP))
- System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");
-
- sliceCount = 2;
- }
-
- @Override
- protected boolean useTlogReplicas() {
- return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug
- // and it's TestInjection use
- }
-
- @BeforeClass
- public static void _setPrsDefault() {
- configurePrsDefault();
- }
-
- @Test
- @ShardsFixed(num = 4)
- public void test() throws Exception {
- boolean testFinished = false;
- try {
- handle.clear();
- handle.put("timestamp", SKIPVAL);
-
- testNodeWithoutCollectionForwarding();
-
- indexr(
- id,
- 1,
- i1,
- 100,
- tlong,
- 100,
- t1,
- "now is the time for all good men",
- "foo_f",
- 1.414f,
- "foo_b",
- "true",
- "foo_d",
- 1.414d);
-
- commit();
-
- // make sure we are in a steady state...
- waitForRecoveriesToFinish(false);
-
- assertDocCounts(false);
-
- indexAbunchOfDocs();
-
- // check again
- waitForRecoveriesToFinish(false);
-
- commit();
-
- assertDocCounts(VERBOSE);
- checkQueries();
-
- assertDocCounts(VERBOSE);
-
- query("q", "*:*", "sort", "n_tl1 desc");
-
- bringDownShardIndexSomeDocsAndRecover();
-
- query("q", "*:*", "sort", "n_tl1 desc");
-
- // test adding another replica to a shard - it should do a
- // recovery/replication to pick up the index from the leader
- addNewReplica();
-
- long docId = testUpdateAndDelete();
-
- // index a bad doc...
- expectThrows(SolrException.class, () -> indexr(t1, "a doc with no id"));
-
- // TODO: bring this to its own method?
- // try indexing to a leader that has no replicas up
- ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
- ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, SHARD2);
-
- String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP);
- chaosMonkey.stopShardExcept(SHARD2, nodeName);
-
- SolrClient client = getClient(nodeName);
-
- index_specific(client, "id", docId + 1, t1, "what happens here?");
-
- // expire a session...
- CloudJettyRunner cloudJetty = shardToJetty.get(SHARD1).get(0);
- chaosMonkey.expireSession(cloudJetty.jetty);
- // Wait until the jetty is reconnected, otherwise the following index command could fail
- cloudJetty
- .jetty
- .getCoreContainer()
- .getZkController()
- .getZkClient()
- .getCuratorFramework()
- .blockUntilConnected(50, TimeUnit.MILLISECONDS);
-
- indexr("id", docId + 1, t1, "slip this doc in");
-
- waitForRecoveriesToFinish(false);
-
- checkShardConsistency(SHARD1);
- checkShardConsistency(SHARD2);
-
- testFinished = true;
- } finally {
- if (!testFinished) {
- printLayoutOnTearDown = true;
- }
- }
- }
-
- private void testNodeWithoutCollectionForwarding() throws Exception {
- assertEquals(
- 0,
- CollectionAdminRequest.createCollection(ONE_NODE_COLLECTION, "conf1", 1, 1)
- .setCreateNodeSet("")
- .process(cloudClient)
- .getStatus());
- assertTrue(
- CollectionAdminRequest.addReplicaToShard(ONE_NODE_COLLECTION, "shard1")
- .setCoreName(ONE_NODE_COLLECTION + "core")
- .process(cloudClient)
- .isSuccess());
-
- waitForCollection(ZkStateReader.from(cloudClient), ONE_NODE_COLLECTION, 1);
- waitForRecoveriesToFinish(ONE_NODE_COLLECTION, ZkStateReader.from(cloudClient), false);
-
- ZkStateReader.from(cloudClient).getLeaderRetry(ONE_NODE_COLLECTION, SHARD1, 30000);
-
- int docs = 2;
- for (JettySolrRunner jetty : jettys) {
- final String clientUrl = getBaseUrl(jetty);
- addAndQueryDocs(clientUrl, docs);
- docs += 2;
- }
- }
-
- // 2 docs added every call
- private void addAndQueryDocs(final String baseUrl, int docs) throws Exception {
-
- SolrQuery query = new SolrQuery("*:*");
-
- try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollection")) {
- // add a doc
- client.add(sdoc("id", docs));
- client.commit();
-
- QueryResponse results = client.query(query);
- assertEquals(docs - 1, results.getResults().getNumFound());
-
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField("id", docs + 1);
- client.add(doc);
- client.commit();
-
- query = new SolrQuery("*:*");
- query.set("rows", 0);
- results = client.query(query);
- assertEquals(docs, results.getResults().getNumFound());
- }
- }
-
- private long testUpdateAndDelete() throws Exception {
- long docId = 99999999L;
- indexr("id", docId, t1, "originalcontent");
-
- commit();
-
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.add("q", t1 + ":originalcontent");
- QueryResponse results = clients.get(0).query(params);
- assertEquals(1, results.getResults().getNumFound());
-
- // update doc
- indexr("id", docId, t1, "updatedcontent");
-
- commit();
-
- results = clients.get(0).query(params);
- assertEquals(0, results.getResults().getNumFound());
-
- params.set("q", t1 + ":updatedcontent");
-
- results = clients.get(0).query(params);
- assertEquals(1, results.getResults().getNumFound());
-
- UpdateRequest uReq = new UpdateRequest();
- // uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN);
- uReq.deleteById(Long.toString(docId)).process(clients.get(0));
-
- commit();
-
- results = clients.get(0).query(params);
- assertEquals(0, results.getResults().getNumFound());
- return docId;
- }
-
- private void bringDownShardIndexSomeDocsAndRecover() throws Exception {
- SolrQuery query = new SolrQuery("*:*");
- query.set("distrib", false);
-
- commit();
-
- long deadShardCount =
- shardToJetty.get(SHARD2).get(0).client.solrClient.query(query).getResults().getNumFound();
-
- query("q", "*:*", "sort", "n_tl1 desc");
-
- int oldLiveNodes =
- ZkStateReader.from(cloudClient)
- .getZkClient()
- .getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true)
- .size();
-
- assertEquals(5, oldLiveNodes);
-
- // kill a shard
- CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0);
-
- // ensure shard is dead
- expectThrows(
- SolrServerException.class,
- "This server should be down and this update should have failed",
- () -> index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1, "specific doc!"));
-
- commit();
-
- query("q", "*:*", "sort", "n_tl1 desc");
-
- // long cloudClientDocs = cloudClient.query(new
- // SolrQuery("*:*")).getResults().getNumFound();
- // System.out.println("clouddocs:" + cloudClientDocs);
-
- // try to index to a living shard at shard2
-
- long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
-
- ZkStateReader.from(cloudClient).getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 60000);
-
- try {
- index_specific(
- shardToJetty.get(SHARD1).get(1).client.solrClient,
- id,
- 1000,
- i1,
- 108,
- t1,
- "specific doc!");
- } catch (Exception e) {
- // wait and try again
- Thread.sleep(4000);
- index_specific(
- shardToJetty.get(SHARD1).get(1).client.solrClient,
- id,
- 1000,
- i1,
- 108,
- t1,
- "specific doc!");
- }
-
- commit();
-
- checkShardConsistency(true, false);
-
- query("q", "*:*", "sort", "n_tl1 desc");
-
- long numFound2 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
-
- assertEquals(numFound1 + 1, numFound2);
-
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField("id", 1001);
-
- controlClient.add(doc);
-
- // try adding a doc with CloudSolrServer
- UpdateRequest ureq = new UpdateRequest();
- ureq.add(doc);
- // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);
-
- try {
- ureq.process(cloudClient);
- } catch (SolrServerException e) {
- // try again
- Thread.sleep(3500);
- ureq.process(cloudClient);
- }
-
- commit();
-
- query("q", "*:*", "sort", "n_tl1 desc");
-
- long numFound3 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
-
- // lets just check that the one doc since last commit made it in...
- assertEquals(numFound2 + 1, numFound3);
-
- // test debugging
- testDebugQueries();
-
- if (VERBOSE) {
- System.err.println(controlClient.query(new SolrQuery("*:*")).getResults().getNumFound());
-
- for (SolrClient client : clients) {
- try {
- SolrQuery q = new SolrQuery("*:*");
- q.set("distrib", false);
- System.err.println(client.query(q).getResults().getNumFound());
- } catch (Exception e) {
-
- }
- }
- }
- // TODO: This test currently fails because debug info is obtained only
- // on shards with matches.
- // query("q","matchesnothing","fl","*,score", "debugQuery", "true");
-
- // this should trigger a recovery phase on deadShard
- deadShard.jetty.start();
-
- // make sure we have published we are recovering
- Thread.sleep(1500);
-
- waitForRecoveriesToFinish(false);
-
- deadShardCount =
- shardToJetty.get(SHARD1).get(0).client.solrClient.query(query).getResults().getNumFound();
- // if we properly recovered, we should now have the couple missing docs that
- // came in while shard was down
- checkShardConsistency(true, false);
-
- // recover over 100 docs so we do more than just peer sync (replicate recovery)
- chaosMonkey.stopJetty(deadShard);
-
- for (int i = 0; i < 226; i++) {
- doc = new SolrInputDocument();
- doc.addField("id", 2000 + i);
- controlClient.add(doc);
- ureq = new UpdateRequest();
- ureq.add(doc);
- // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);
- ureq.process(cloudClient);
- }
- commit();
-
- Thread.sleep(1500);
-
- deadShard.jetty.start();
-
- // make sure we have published we are recovering
- Thread.sleep(1500);
-
- waitForThingsToLevelOut(1, TimeUnit.MINUTES);
-
- Thread.sleep(500);
-
- waitForRecoveriesToFinish(false);
-
- checkShardConsistency(true, false);
-
- // try a backup command
- try (final SolrClient client =
- getHttpSolrClient((String) shardToJetty.get(SHARD2).get(0).info.get("base_url"))) {
- final String backupName = "the_backup";
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("qt", ReplicationHandler.PATH);
- params.set("command", "backup");
- params.set("name", backupName);
- final Path location = FilterPath.unwrap(createTempDir()).toRealPath();
- // Allow non-standard location outside SOLR_HOME
- jettys.forEach(j -> j.getCoreContainer().getAllowPaths().add(location));
- params.set("location", location.toString());
-
- QueryRequest request = new QueryRequest(params);
- client.request(request, DEFAULT_TEST_COLLECTION_NAME);
-
- final BackupStatusChecker backupStatus =
- new BackupStatusChecker(client, "/" + DEFAULT_TEST_COLLECTION_NAME + "/replication");
- final String backupDirName = backupStatus.waitForBackupSuccess(backupName, 30);
- assertTrue(
- "Backup dir does not exist: " + backupDirName,
- Files.exists(location.resolve(backupDirName)));
- }
- }
-
- private void addNewReplica() throws Exception {
-
- waitForRecoveriesToFinish(false);
-
- // new server should be part of first shard
- // how many docs are on the new shard?
- for (CloudJettyRunner cjetty : shardToJetty.get(SHARD1)) {
- if (VERBOSE)
- System.err.println(
- "shard1 total:"
- + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());
- }
- for (CloudJettyRunner cjetty : shardToJetty.get(SHARD2)) {
- if (VERBOSE)
- System.err.println(
- "shard2 total:"
- + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());
- }
-
- checkShardConsistency(SHARD1);
- checkShardConsistency(SHARD2);
-
- assertDocCounts(VERBOSE);
- }
-
- private void testDebugQueries() throws Exception {
- handle.put("explain", SKIPVAL);
- handle.put("debug", UNORDERED);
- handle.put("time", SKIPVAL);
- handle.put("track", SKIP);
- query("q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true");
- query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true");
- query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING);
- query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS);
- query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY);
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java
deleted file mode 100644
index 24d0689c5a6..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java
+++ /dev/null
@@ -1,1737 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.lucene.util.IOUtils;
-import org.apache.solr.JSONTestUtil;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.apache.HttpSolrClient;
-import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
-import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.request.SolrQuery;
-import org.apache.solr.client.solrj.request.StreamingUpdateRequest;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.client.solrj.response.FacetField;
-import org.apache.solr.client.solrj.response.Group;
-import org.apache.solr.client.solrj.response.GroupCommand;
-import org.apache.solr.client.solrj.response.GroupResponse;
-import org.apache.solr.client.solrj.response.InputStreamResponseParser;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.cloud.api.collections.CollectionHandlingUtils;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CollectionParams.CollectionAction;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.params.UpdateParams;
-import org.apache.solr.common.util.ExecutorUtil;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SolrNamedThreadFactory;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.util.TestInjection;
-import org.apache.solr.util.TestInjection.Hook;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This test simply does a bunch of basic things in solrcloud mode and asserts things work as
- * expected.
- */
-public abstract class AbstractBasicDistributedZkTestBase extends AbstractFullDistribZkTestBase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final String DEFAULT_COLLECTION = "collection1";
-
- private final boolean onlyLeaderIndexes = random().nextBoolean();
-
- String tsort = "t_sortable";
-
- private Map<String, List<SolrClient>> otherCollectionClients = new HashMap<>();
-
- private String oneInstanceCollection = "oneInstanceCollection";
- private String oneInstanceCollection2 = "oneInstanceCollection2";
-
- private AtomicInteger nodeCounter = new AtomicInteger();
-
- CompletionService<Object> completionService;
- Set<Future<Object>> pending;
-
- private static Hook newSearcherHook =
- new Hook() {
- volatile CountDownLatch latch;
- AtomicReference<String> collection = new AtomicReference<>();
-
- @Override
- public void newSearcher(String collectionName) {
- String c = collection.get();
- if (c != null && c.equals(collectionName)) {
- log.info("Hook detected newSearcher");
- try {
- latch.countDown();
- } catch (NullPointerException e) {
-
- }
- }
- }
-
- @Override
- public void waitForSearcher(
- String collection, int cnt, int timeoutms, boolean failOnTimeout)
- throws InterruptedException {
- latch = new CountDownLatch(cnt);
- this.collection.set(collection);
- boolean timeout = !latch.await(timeoutms, TimeUnit.MILLISECONDS);
- if (timeout && failOnTimeout) {
- fail("timed out waiting for new searcher event " + latch.getCount());
- }
- }
- };
-
- public AbstractBasicDistributedZkTestBase() {
- // we need DVs on point fields to compute stats & facets
- if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP))
- System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");
-
- sliceCount = 2;
- completionService = new ExecutorCompletionService<>(executor);
- pending = new HashSet<>();
- }
-
- @BeforeClass
- public static void beforeBDZKTClass() {
- TestInjection.newSearcherHook(newSearcherHook);
- }
-
- @Override
- protected boolean useTlogReplicas() {
- // TODO: tlog replicas makes commits take way to long due to what is likely a bug and it's
- // TestInjection use
- return false;
- }
-
- @Override
- protected void setDistributedParams(ModifiableSolrParams params) {
-
- if (r.nextBoolean()) {
- // don't set shards, let that be figured out from the cloud state
- } else {
- // use shard ids rather than physical locations
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < getShardCount(); i++) {
- if (i > 0) sb.append(',');
- sb.append("shard" + (i + 3));
- }
- params.set("shards", sb.toString());
- }
- }
-
- @Test
- @ShardsFixed(num = 4)
- protected void test() throws Exception {
- // setLoggingLevel(null);
-
- ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
- // make sure we have leaders for each shard
- for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
- } // make sure we again have leaders for each shard
-
- waitForRecoveriesToFinish(false);
-
- handle.clear();
- handle.put("timestamp", SKIPVAL);
-
- del("*:*");
- queryAndCompareShards(params("q", "*:*", "distrib", "false", "sanity_check", "is_empty"));
-
- // ask every individual replica of every shard to update+commit the same doc id
- // with an incrementing counter on each update+commit
- int foo_i_counter = 0;
- for (SolrClient client : clients) {
- foo_i_counter++;
- indexDoc(
- client,
- params("commit", "true"), // SOLR-4923
- sdoc(id, 1, i1, 100, tlong, 100, "foo_i", foo_i_counter));
- // after every update+commit, check all the shards consistency
- queryAndCompareShards(
- params("q", "id:1", "distrib", "false", "sanity_check", "non_distrib_id_1_lookup"));
- queryAndCompareShards(
- params(
- "q", "id:1",
- "sanity_check", "distrib_id_1_lookup"));
- }
-
- indexr(
- id,
- 1,
- i1,
- 100,
- tlong,
- 100,
- t1,
- "now is the time for all good men",
- "foo_f",
- 1.414f,
- "foo_b",
- "true",
- "foo_d",
- 1.414d,
- tsort,
- "now is the time for all good men");
- indexr(
- id,
- 2,
- i1,
- 50,
- tlong,
- 50,
- t1,
- "to come to the aid of their country.",
- tsort,
- "to come to the aid of their country.");
- indexr(id, 3, i1, 2, tlong, 2, t1, "how now brown cow", tsort, "how now brown cow");
- indexr(
- id,
- 4,
- i1,
- -100,
- tlong,
- 101,
- t1,
- "the quick fox jumped over the lazy dog",
- tsort,
- "the quick fox jumped over the lazy dog");
- indexr(
- id,
- 5,
- i1,
- 500,
- tlong,
- 500,
- t1,
- "the quick fox jumped way over the lazy dog",
- tsort,
- "the quick fox jumped over the lazy dog");
- indexr(
- id,
- 6,
- i1,
- -600,
- tlong,
- 600,
- t1,
- "humpty dumpy sat on a wall",
- tsort,
- "the quick fox jumped over the lazy dog");
- indexr(
- id,
- 7,
- i1,
- 123,
- tlong,
- 123,
- t1,
- "humpty dumpy had a great fall",
- tsort,
- "the quick fox jumped over the lazy dog");
- indexr(
- id,
- 8,
- i1,
- 876,
- tlong,
- 876,
- t1,
- "all the kings horses and all the kings men",
- tsort,
- "all the kings horses and all the kings men");
- indexr(
- id,
- 9,
- i1,
- 7,
- tlong,
- 7,
- t1,
- "couldn't put humpty together again",
- tsort,
- "the quick fox jumped over the lazy dog");
- indexr(id, 10, i1, 4321, tlong, 4321, t1, "this too shall pass", tsort, "this too shall pass");
- indexr(
- id,
- 11,
- i1,
- -987,
- tlong,
- 987,
- t1,
- "An eye for eye only ends up making the whole world blind.",
- tsort,
- "An eye for eye only ends up making the whole world blind.");
- indexr(
- id,
- 12,
- i1,
- 379,
- tlong,
- 379,
- t1,
- "Great works are performed, not by strength, but by perseverance.",
- tsort,
- "Great works are performed, not by strength, but by perseverance.");
- indexr(
- id,
- 13,
- i1,
- 232,
- tlong,
- 232,
- t1,
- "no eggs on wall, lesson learned",
- oddField,
- "odd man out",
- tsort,
- "no eggs on wall, lesson learned");
-
- indexr(
- id,
- 14,
- "SubjectTerms_mfacet",
- new String[] {"mathematical models", "mathematical analysis"});
- indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});
- indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});
- String[] vals = new String[100];
- for (int i = 0; i < 100; i++) {
- vals[i] = "test " + i;
- }
- indexr(id, 17, "SubjectTerms_mfacet", vals);
-
- for (int i = 100; i < 150; i++) {
- indexr(id, i);
- }
-
- commit();
-
- testTokenizedGrouping();
- testSortableTextFaceting();
- testSortableTextSorting();
- testSortableTextGrouping();
-
- queryAndCompareShards(
- params(
- "q", "*:*",
- "sort", "id desc",
- "distrib", "false",
- "sanity_check", "is_empty"));
-
- // random value sort
- for (String f : fieldNames) {
- query(false, new String[] {"q", "*:*", "sort", f + " desc"});
- query(false, new String[] {"q", "*:*", "sort", f + " asc"});
- }
-
- // these queries should be exactly ordered and scores should exactly match
- query(false, new String[] {"q", "*:*", "sort", i1 + " desc"});
- query(false, new String[] {"q", "*:*", "sort", i1 + " asc"});
- query(false, new String[] {"q", "*:*", "sort", i1 + " desc", "fl", "*,score"});
- query(false, new String[] {"q", "*:*", "sort", "n_tl1 asc", "fl", "*,score"});
- query(false, new String[] {"q", "*:*", "sort", "n_tl1 desc"});
- handle.put("maxScore", SKIPVAL);
- query(
- false,
- new String[] {"q", "{!func}" + i1}); // does not expect maxScore. So if it comes ,ignore it.
- // JavaBinCodec.writeSolrDocumentList()
- // is agnostic of request params.
- handle.remove("maxScore");
- query(
- false,
- new String[] {
- "q", "{!func}" + i1, "fl", "*,score"
- }); // even scores should match exactly here
-
- handle.put("highlighting", UNORDERED);
- handle.put("response", UNORDERED);
-
- handle.put("maxScore", SKIPVAL);
- query(false, new String[] {"q", "quick"});
- query(false, new String[] {"q", "all", "fl", "id", "start", "0"});
- query(
- false,
- new String[] {"q", "all", "fl", "foofoofoo", "start", "0"}); // no fields in returned docs
- query(false, new String[] {"q", "all", "fl", "id", "start", "100"});
-
- handle.put("score", SKIPVAL);
- query(false, new String[] {"q", "quick", "fl", "*,score"});
- query(false, new String[] {"q", "all", "fl", "*,score", "start", "1"});
- query(false, new String[] {"q", "all", "fl", "*,score", "start", "100"});
-
- query(
- false,
- new String[] {
- "q", "now their fox sat had put", "fl", "*,score", "hl", "true", "hl.fl", t1
- });
-
- query(
- false,
- new String[] {
- "q", "now their fox sat had put", "fl", "foofoofoo", "hl", "true", "hl.fl", t1
- });
-
- query(false, new String[] {"q", "matchesnothing", "fl", "*,score"});
-
- query(false, new Object[] {"q", "*:*", "rows", 100, "facet", "true", "facet.field", t1});
- query(
- false,
- new Object[] {
- "q",
- "*:*",
- "rows",
- 100,
- "facet",
- "true",
- "facet.field",
- t1,
- "facet.limit",
- -1,
- "facet.sort",
- "count"
- });
- query(
- false,
- new Object[] {
- "q",
- "*:*",
- "rows",
- 100,
- "facet",
- "true",
- "facet.field",
- t1,
- "facet.limit",
- -1,
- "facet.sort",
- "count",
- "facet.mincount",
- 2
- });
- query(
- false,
- new Object[] {
- "q",
- "*:*",
- "rows",
- 100,
- "facet",
- "true",
- "facet.field",
- t1,
- "facet.limit",
- -1,
- "facet.sort",
- "index"
- });
- query(
- false,
- new Object[] {
- "q",
- "*:*",
- "rows",
- 100,
- "facet",
- "true",
- "facet.field",
- t1,
- "facet.limit",
- -1,
- "facet.sort",
- "index",
- "facet.mincount",
- 2
- });
- query(
- false,
- new Object[] {
- "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.limit", 1
- });
- query(
- false,
- new Object[] {
- "q",
- "*:*",
- "rows",
- 100,
- "facet",
- "true",
- "facet.query",
- "quick",
- "facet.query",
- "all",
- "facet.query",
- "*:*"
- });
- query(
- false,
- new Object[] {
- "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.offset", 1
- });
- query(
- false,
- new Object[] {
- "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.mincount", 2
- });
-
- // test faceting multiple things at once
- query(
- false,
- new Object[] {
- "q",
- "*:*",
- "rows",
- 100,
- "facet",
- "true",
- "facet.query",
- "quick",
- "facet.query",
- "all",
- "facet.query",
- "*:*",
- "facet.field",
- t1
- });
-
- // test filter tagging, facet exclusion, and naming (multi-select facet support)
- query(
- false,
- new Object[] {
- "q",
- "*:*",
- "rows",
- 100,
- "facet",
- "true",
- "facet.query",
- "{!key=myquick}quick",
- "facet.query",
- "{!key=myall ex=a}all",
- "facet.query",
- "*:*",
- "facet.field",
- "{!key=mykey ex=a}" + t1,
- "facet.field",
- "{!key=other ex=b}" + t1,
- "facet.field",
- "{!key=again ex=a,b}" + t1,
- "facet.field",
- t1,
- "fq",
- "{!tag=a}id_i1:[1 TO 7]",
- "fq",
- "{!tag=b}id_i1:[3 TO 9]"
- });
- query(
- false,
- new Object[] {
- "q",
- "*:*",
- "facet",
- "true",
- "facet.field",
- "{!ex=t1}SubjectTerms_mfacet",
- "fq",
- "{!tag=t1}SubjectTerms_mfacet:(test 1)",
- "facet.limit",
- "10",
- "facet.mincount",
- "1"
- });
-
- // test field that is valid in schema but missing in all shards
- query(
- false,
- new Object[] {
- "q", "*:*", "rows", 100, "facet", "true", "facet.field", missingField, "facet.mincount", 2
- });
- // test field that is valid in schema and missing in some shards
- query(
- false,
- new Object[] {
- "q", "*:*", "rows", 100, "facet", "true", "facet.field", oddField, "facet.mincount", 2
- });
-
- query(
- false, new Object[] {"q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", i1});
-
- /* TODO: the failure may come back in "exception"
- try {
- // test error produced for field that is invalid for schema
- query("q","*:*", "rows",100, "facet","true", "facet.field",invalidField, "facet.mincount",2);
- TestCase.fail("SolrServerException expected for invalid field that is not in schema");
- } catch (SolrServerException ex) {
- // expected
- }
- */
-
- // Try to get better coverage for refinement queries by turning off over requesting.
- // This makes it much more likely that we may not get the top facet values and hence
- // we turn of that checking.
- handle.put("facet_fields", SKIPVAL);
- query(
- false,
- new Object[] {
- "q",
- "*:*",
- "rows",
- 0,
- "facet",
- "true",
- "facet.field",
- t1,
- "facet.limit",
- 5,
- "facet.shard.limit",
- 5
- });
- // check a complex key name
- query(
- false,
- new Object[] {
- "q",
- "*:*",
- "rows",
- 0,
- "facet",
- "true",
- "facet.field",
- "{!key='a b/c \\' \\} foo'}" + t1,
- "facet.limit",
- 5,
- "facet.shard.limit",
- 5
- });
- handle.remove("facet_fields");
-
- // index the same document to two servers and make sure things
- // don't blow up.
- if (clients.size() >= 2) {
- index(id, 100, i1, 107, t1, "oh no, a duplicate!");
- for (int i = 0; i < clients.size(); i++) {
- index_specific(i, id, 100, i1, 107, t1, "oh no, a duplicate!");
- }
- commit();
- query(false, new Object[] {"q", "duplicate", "hl", "true", "hl.fl", t1});
- query(false, new Object[] {"q", "fox duplicate horses", "hl", "true", "hl.fl", t1});
- query(false, new Object[] {"q", "*:*", "rows", 100});
- }
-
- // test debugging
- handle.put("explain", SKIPVAL);
- handle.put("debug", UNORDERED);
- handle.put("time", SKIPVAL);
- handle.put("track", SKIP);
- query(
- false,
- new Object[] {
- "q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true"
- });
- query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"});
- query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING});
- query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS});
- query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY});
-
- // try add commitWithin
- long before = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
- for (SolrClient client : clients) {
- assertEquals(
- "unexpected pre-commitWithin document count on node: "
- + ((HttpSolrClient) client).getBaseURL(),
- before,
- client.query(new SolrQuery("*:*")).getResults().getNumFound());
- }
-
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("commitWithin", 10);
- add(cloudClient, params, List.of(getDoc("id", 300), getDoc("id", 301)));
-
- newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);
-
- ClusterState clusterState = getCommonCloudSolrClient().getClusterState();
- DocCollection dColl = clusterState.getCollection(DEFAULT_COLLECTION);
-
- assertSliceCounts("should have found 2 docs, 300 and 301", before + 2, dColl);
-
- // try deleteById commitWithin
- UpdateRequest deleteByIdReq = new UpdateRequest();
- deleteByIdReq.deleteById("300");
- deleteByIdReq.setCommitWithin(10);
- deleteByIdReq.process(cloudClient);
-
- newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);
-
- assertSliceCounts("deleteById commitWithin did not work", before + 1, dColl);
-
- // try deleteByQuery commitWithin
- UpdateRequest deleteByQueryReq = new UpdateRequest();
- deleteByQueryReq.deleteByQuery("id:301");
- deleteByQueryReq.setCommitWithin(10);
- deleteByQueryReq.process(cloudClient);
-
- newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false);
-
- assertSliceCounts("deleteByQuery commitWithin did not work", before, dColl);
-
- // TODO: This test currently fails because debug info is obtained only
- // on shards with matches.
- // query("q","matchesnothing","fl","*,score", "debugQuery", "true");
-
- // would be better if these where all separate tests - but much, much
- // slower
- doOptimisticLockingAndUpdating();
- testShardParamVariations();
- testMultipleCollections();
- testANewCollectionInOneInstance();
- testSearchByCollectionName();
- testUpdateByCollectionName();
- testANewCollectionInOneInstanceWithManualShardAssignement();
- testNumberOfCommitsWithCommitAfterAdd();
-
- testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit");
- testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit");
-
- testStopAndStartCoresInOneInstance();
- }
-
- private void testSortableTextFaceting() throws Exception {
- SolrQuery query = new SolrQuery("*:*");
- query.addFacetField(tsort);
- query.setFacetMissing(false);
- QueryResponse resp = queryRandomShard(query);
- List<FacetField> ffs = resp.getFacetFields();
- for (FacetField ff : ffs) {
- if (ff.getName().equals(tsort) == false) continue;
- for (FacetField.Count count : ff.getValues()) {
- long num = count.getCount();
- switch (count.getName()) {
- case "all the kings horses and all the kings men":
- case "An eye for eye only ends up making the whole world blind.":
- case "Great works are performed, not by strength, but by perseverance.":
- case "how now brown cow":
- case "no eggs on wall, lesson learned":
- case "now is the time for all good men":
- case "this too shall pass":
- case "to come to the aid of their country.":
- assertEquals("Should have exactly one facet count for field " + ff.getName(), 1, num);
- break;
- case "the quick fox jumped over the lazy dog":
- assertEquals("Should have 5 docs for the lazy dog", 5, num);
- break;
- default:
- fail("No case for facet '" + ff.getName() + "'");
- }
- }
- }
- }
-
- private void testSortableTextSorting() throws Exception {
- SolrQuery query = new SolrQuery("*:*");
- query.addSort(tsort, SolrQuery.ORDER.desc);
- query.addField("*");
- query.addField("eoe_sortable");
- query.addField(tsort);
- QueryResponse resp = queryRandomShard(query);
-
- SolrDocumentList docs = resp.getResults();
-
- String title = docs.get(0).getFieldValue(tsort).toString();
- for (SolrDocument doc : docs) {
- assertTrue(
- "Docs should be back in sorted order, descending",
- title.compareTo(doc.getFieldValue(tsort).toString()) >= 0);
- title = doc.getFieldValue(tsort).toString();
- }
- }
-
- private void testSortableTextGrouping() throws Exception {
- SolrQuery query = new SolrQuery("*:*");
- query.add("group", "true");
- query.add("group.field", tsort);
- QueryResponse resp = queryRandomShard(query);
- GroupResponse groupResp = resp.getGroupResponse();
- List<GroupCommand> grpCmds = groupResp.getValues();
- for (GroupCommand grpCmd : grpCmds) {
- if (grpCmd.getName().equals(tsort) == false) continue;
- for (Group grp : grpCmd.getValues()) {
- long count = grp.getResult().getNumFound();
- if (grp.getGroupValue() == null)
- continue; // Don't count the groups without an entry as the numnber is variable
- switch (grp.getGroupValue()) {
- case "all the kings horses and all the kings men":
- case "An eye for eye only ends up making the whole world blind.":
- case "Great works are performed, not by strength, but by perseverance.":
- case "how now brown cow":
- case "no eggs on wall, lesson learned":
- case "now is the time for all good men":
- case "this too shall pass":
- case "to come to the aid of their country.":
- assertEquals(
- "Should have exactly one facet count for field " + grpCmd.getName(), 1, count);
- break;
- case "the quick fox jumped over the lazy dog":
- assertEquals("Should have 5 docs for the lazy dog", 5, count);
- break;
- default:
- fail("No case for facet '" + grpCmd.getName() + "'");
- }
- }
- }
- }
-
- private void testTokenizedGrouping() throws Exception {
- SolrException ex =
- expectThrows(
- SolrException.class,
- () -> {
- query(false, new String[] {"q", "*:*", "group", "true", "group.field", t1});
- });
- assertTrue(
- "Expected error from server that SortableTextFields are required",
- ex.getMessage()
- .contains(
- "Sorting on a tokenized field that is not a SortableTextField is not supported in cloud mode"));
- }
-
- private void assertSliceCounts(String msg, long expected, DocCollection dColl) throws Exception {
- long found = checkSlicesSameCounts(dColl);
-
- if (found != expected) {
- // we get one do over in a bad race
- Thread.sleep(1000);
- found = checkSlicesSameCounts(dColl);
- }
-
- assertEquals(msg, expected, checkSlicesSameCounts(dColl));
- }
-
- // Ensure that total docs found is the expected number.
- private void waitForDocCount(long expectedNumFound, long waitMillis, String failureMessage)
- throws Exception {
- AtomicLong total = new AtomicLong(-1);
- try {
- ZkStateReader.from(getCommonCloudSolrClient())
- .waitForState(
- DEFAULT_COLLECTION,
- waitMillis,
- TimeUnit.MILLISECONDS,
- (n, c) -> {
- long docTotal;
- try {
- docTotal = checkSlicesSameCounts(c);
- } catch (SolrServerException | IOException e) {
- throw new RuntimeException(e);
- }
- total.set(docTotal);
- if (docTotal == expectedNumFound) {
- return true;
- }
- return false;
- });
- } catch (TimeoutException | InterruptedException e) {
-
- }
- // We could fail here if we broke out of the above because we exceeded the time allowed.
- assertEquals(failureMessage, expectedNumFound, total.get());
-
- // This should be redundant, but it caught a test error after all.
- for (SolrClient client : clients) {
- assertEquals(
- failureMessage,
- expectedNumFound,
- client.query(new SolrQuery("*:*")).getResults().getNumFound());
- }
- }
-
- // Insure that counts are the same for all replicas in each shard
- // Return the total doc count for the query.
- private long checkSlicesSameCounts(DocCollection dColl) throws SolrServerException, IOException {
- long docTotal = 0; // total number of documents found counting only one replica per slice.
- for (Slice slice : dColl.getActiveSlices()) {
- long sliceDocCount = -1;
- for (Replica rep : slice.getReplicas()) {
- try (SolrClient one = getHttpSolrClient(rep)) {
- SolrQuery query = new SolrQuery("*:*");
- query.setDistrib(false);
- QueryResponse resp = one.query(query);
- long hits = resp.getResults().getNumFound();
- if (sliceDocCount == -1) {
- sliceDocCount = hits;
- docTotal += hits;
- } else {
- if (hits != sliceDocCount) {
- return -1;
- }
- }
- }
- }
- }
- return docTotal;
- }
-
- private void testShardParamVariations() throws Exception {
- SolrQuery query = new SolrQuery("*:*");
- Map<String, Long> shardCounts = new HashMap<>();
-
- for (String shard : shardToJetty.keySet()) {
- // every client should give the same numDocs for this shard
- // shffle the clients in a diff order for each shard
- List<SolrClient> solrclients = new ArrayList<>(this.clients);
- Collections.shuffle(solrclients, random());
- for (SolrClient client : solrclients) {
- query.set("shards", shard);
- long numDocs = client.query(query).getResults().getNumFound();
- assertTrue("numDocs < 0 for shard " + shard + " via " + client, 0 <= numDocs);
- if (!shardCounts.containsKey(shard)) {
- shardCounts.put(shard, numDocs);
- }
- assertEquals(
- "inconsitent numDocs for shard " + shard + " via " + client,
- shardCounts.get(shard).longValue(),
- numDocs);
-
- List<CloudJettyRunner> replicaJetties = new ArrayList<>(shardToJetty.get(shard));
- Collections.shuffle(replicaJetties, random());
-
- // each replica should also give the same numDocs
- ArrayList<String> replicaAlts = new ArrayList<>(replicaJetties.size() * 2);
- for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) {
- String replica = replicaJetty.url;
- query.set("shards", replica);
-
- // replicas already shuffled, use this in the alternative check below
- if (0 == random().nextInt(3) || replicaAlts.size() < 2) {
- replicaAlts.add(replica);
- }
-
- numDocs = client.query(query).getResults().getNumFound();
- assertTrue("numDocs < 0 for replica " + replica + " via " + client, 0 <= numDocs);
- assertEquals(
- "inconsitent numDocs for shard "
- + shard
- + " in replica "
- + replica
- + " via "
- + client,
- shardCounts.get(shard).longValue(),
- numDocs);
- }
-
- // any combination of replica alternatives should give same numDocs
- String replicas = String.join("|", replicaAlts);
- query.set("shards", replicas);
- numDocs = client.query(query).getResults().getNumFound();
- assertTrue("numDocs < 0 for replicas " + replicas + " via " + client, 0 <= numDocs);
- assertEquals(
- "inconsitent numDocs for replicas " + replicas + " via " + client,
- shardCounts.get(shard).longValue(),
- numDocs);
- }
- }
-
- // sums of multiple shards should add up regardless of how we
- // query those shards or which client we use
- long randomShardCountsExpected = 0;
- ArrayList<String> randomShards = new ArrayList<>(shardCounts.size());
- for (Map.Entry<String, Long> shardData : shardCounts.entrySet()) {
- if (random().nextBoolean() || randomShards.size() < 2) {
- String shard = shardData.getKey();
- randomShardCountsExpected += shardData.getValue();
- if (random().nextBoolean()) {
- // use shard id
- randomShards.add(shard);
- } else {
- // use some set explicit replicas
- ArrayList<String> replicas = new ArrayList<>(7);
- for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) {
- if (0 == random().nextInt(3) || 0 == replicas.size()) {
- replicas.add(replicaJetty.url);
- }
- }
- Collections.shuffle(replicas, random());
- randomShards.add(String.join("|", replicas));
- }
- }
- }
- String randShards = String.join(",", randomShards);
- query.set("shards", randShards);
- for (SolrClient client : this.clients) {
- assertEquals(
- "numDocs for " + randShards + " via " + client,
- randomShardCountsExpected,
- client.query(query).getResults().getNumFound());
- }
-
- // total num docs must match sum of every shard's numDocs
- query = new SolrQuery("*:*");
- long totalShardNumDocs = 0;
- for (Long c : shardCounts.values()) {
- totalShardNumDocs += c;
- }
- for (SolrClient client : clients) {
- assertEquals(
- "sum of shard numDocs on client: " + client,
- totalShardNumDocs,
- client.query(query).getResults().getNumFound());
- }
- assertTrue("total numDocs <= 0, WTF? Test is useless", 0 < totalShardNumDocs);
- }
-
- private void testStopAndStartCoresInOneInstance() throws Exception {
- JettySolrRunner jetty = jettys.get(0);
- try (final SolrClient httpSolrClient = (HttpSolrClient) jetty.newClient(15000, 60000)) {
- ThreadPoolExecutor executor = null;
- try {
- executor =
- new ExecutorUtil.MDCAwareThreadPoolExecutor(
- 0,
- Integer.MAX_VALUE,
- 5,
- TimeUnit.SECONDS,
- new SynchronousQueue<Runnable>(),
- new SolrNamedThreadFactory("testExecutor"));
- int cnt = 3;
-
- // create the cores
- createCollectionInOneInstance(
- httpSolrClient, jetty.getNodeName(), executor, "multiunload2", 1, cnt);
- } finally {
- if (executor != null) {
- ExecutorUtil.shutdownAndAwaitTermination(executor);
- }
- }
- }
-
- cloudJettys.get(0).jetty.stop();
- printLayout();
-
- cloudJettys.get(0).jetty.start();
- ZkStateReader.from(cloudClient).forceUpdateCollection("multiunload2");
- try {
- ZkStateReader.from(cloudClient).getLeaderRetry("multiunload2", "shard1", 30000);
- } catch (SolrException e) {
- printLayout();
- throw e;
- }
-
- printLayout();
- }
-
- /** Create a collection in single node */
- public static void createCollectionInOneInstance(
- final SolrClient client,
- String nodeName,
- ThreadPoolExecutor executor,
- final String collection,
- final int numShards,
- int numReplicas) {
- assertNotNull(nodeName);
- try {
- assertEquals(
- 0,
- CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1)
- .setCreateNodeSet("")
- .process(client)
- .getStatus());
- } catch (SolrServerException | IOException e) {
- throw new RuntimeException(e);
- }
- for (int i = 0; i < numReplicas; i++) {
- final int freezeI = i;
- executor.execute(
- () -> {
- try {
- assertTrue(
- CollectionAdminRequest.addReplicaToShard(
- collection, "shard" + ((freezeI % numShards) + 1))
- .setCoreName(collection + freezeI)
- .setNode(nodeName)
- .process(client)
- .isSuccess());
- } catch (SolrServerException | IOException e) {
- throw new RuntimeException(e);
- }
- });
- }
- }
-
- @Override
- protected CollectionAdminResponse createCollection(
- Map<String, List<Integer>> collectionInfos,
- String collectionName,
- String configSetName,
- int numShards,
- int numReplicas,
- SolrClient client,
- String createNodeSetStr)
- throws SolrServerException, IOException {
- // TODO: Use CollectionAdminRequest for this test
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", CollectionAction.CREATE.toString());
-
- params.set(CollectionHandlingUtils.NUM_SLICES, numShards);
- params.set(ZkStateReader.REPLICATION_FACTOR, numReplicas);
- if (createNodeSetStr != null)
- params.set(CollectionHandlingUtils.CREATE_NODE_SET, createNodeSetStr);
-
- int clientIndex = clients.size() > 1 ? random().nextInt(2) : 0;
- List<Integer> list = new ArrayList<>();
- list.add(numShards);
- list.add(numReplicas);
- if (collectionInfos != null) {
- collectionInfos.put(collectionName, list);
- }
- params.set("name", collectionName);
- params.set("collection.configName", configSetName);
- QueryRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- CollectionAdminResponse res = new CollectionAdminResponse();
- if (client == null) {
- final String baseUrl = ((HttpSolrClient) clients.get(clientIndex)).getBaseURL();
-
- try (SolrClient aClient = createNewSolrClient("", baseUrl)) {
- res.setResponse(aClient.request(request));
- }
- } else {
- res.setResponse(client.request(request));
- }
- return res;
- }
-
- @Override
- protected Replica getLeaderFromZk(String collection, String slice) {
- ClusterState clusterState = getCommonCloudSolrClient().getClusterState();
- Replica leader = clusterState.getCollection(collection).getLeader(slice);
- if (leader == null) {
- throw new RuntimeException("Could not find leader:" + collection + " " + slice);
- }
- return leader;
- }
-
- /**
- * Expects a RegexReplaceProcessorFactories in the chain which will "double up" the values in two
- * (stored) string fields.
- *
- * <p>If the values are "double-doubled" or "not-doubled" then we know the processor was not run
- * the appropriate number of times
- */
- private void testUpdateProcessorsRunOnlyOnce(final String chain) throws Exception {
-
- final String fieldA = "regex_dup_A_s";
- final String fieldB = "regex_dup_B_s";
- final String val = "x";
- final String expected = "x_x";
- final ModifiableSolrParams updateParams = new ModifiableSolrParams();
- updateParams.add(UpdateParams.UPDATE_CHAIN, chain);
-
- final int numLoops = atLeast(50);
-
- for (int i = 1; i < numLoops; i++) {
- // add doc to random client
- SolrClient updateClient = clients.get(random().nextInt(clients.size()));
- SolrInputDocument doc = new SolrInputDocument();
- addFields(doc, id, i, fieldA, val, fieldB, val);
- UpdateResponse ures = add(updateClient, updateParams, doc);
- assertEquals(chain + ": update failed", 0, ures.getStatus());
- ures = updateClient.commit();
- assertEquals(chain + ": commit failed", 0, ures.getStatus());
- }
-
- // query for each doc, and check both fields to ensure the value is correct
- for (int i = 1; i < numLoops; i++) {
- final String query = id + ":" + i;
- QueryResponse qres = queryRandomShard(new SolrQuery(query));
- assertEquals(chain + ": query failed: " + query, 0, qres.getStatus());
- assertEquals(
- chain + ": didn't find correct # docs with query: " + query,
- 1,
- qres.getResults().getNumFound());
- SolrDocument doc = qres.getResults().get(0);
-
- for (String field : new String[] {fieldA, fieldB}) {
- assertEquals(
- chain + ": doc#" + i + " has wrong value for " + field,
- expected,
- doc.getFirstValue(field));
- }
- }
- }
-
- // cloud level test mainly needed just to make sure that versions and errors are propagated
- // correctly
- private void doOptimisticLockingAndUpdating() throws Exception {
- log.info("### STARTING doOptimisticLockingAndUpdating");
- printLayout();
-
- final SolrInputDocument sd = sdoc("id", 1000, "_version_", -1);
- indexDoc(sd);
-
- ignoreException("version conflict");
- for (SolrClient client : clients) {
- SolrException e = expectThrows(SolrException.class, () -> client.add(sd));
- assertEquals(409, e.code());
- }
- unIgnoreException("version conflict");
-
- // TODO: test deletes. SolrJ needs a good way to pass version for delete...
-
- final SolrInputDocument sd2 = sdoc("id", 1000, "foo_i", 5);
- clients.get(0).add(sd2);
-
- List<Integer> expected = new ArrayList<>();
- int val = 0;
- for (SolrClient client : clients) {
- val += 10;
- client.add(sdoc("id", 1000, "val_i", map("add", val), "foo_i", val));
- expected.add(val);
- }
-
- QueryRequest qr = new QueryRequest(params("qt", "/get", "id", "1000"));
- for (SolrClient client : clients) {
- val += 10;
- NamedList<?> rsp = client.request(qr);
- String match = JSONTestUtil.matchObj("/val_i", rsp.get("doc"), expected);
- if (match != null) throw new RuntimeException(match);
- }
- }
-
- private void testNumberOfCommitsWithCommitAfterAdd() throws SolrServerException, IOException {
- log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd");
- long startCommits = getNumCommits((HttpSolrClient) clients.get(0));
-
- NamedList<Object> result =
- clients
- .get(0)
- .request(
- new StreamingUpdateRequest(
- "/update", getFile("books_numeric_ids.csv"), "application/csv")
- .setCommitWithin(900000)
- .setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true));
-
- long endCommits = getNumCommits((HttpSolrClient) clients.get(0));
- assertEquals(startCommits + 1L, endCommits);
- }
-
- private Long getNumCommits(HttpSolrClient sourceClient) throws SolrServerException, IOException {
- String collection = sourceClient.getDefaultCollection();
- try (SolrClient client =
- new HttpSolrClient.Builder(sourceClient.getBaseURL())
- .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)
- .withSocketTimeout(60000, TimeUnit.MILLISECONDS)
- .build()) {
- var req =
- new GenericSolrRequest(
- SolrRequest.METHOD.GET,
- "/admin/metrics",
- SolrRequest.SolrRequestType.ADMIN,
- SolrParams.of("wt", "prometheus"));
- req.setResponseParser(new InputStreamResponseParser("prometheus"));
-
- NamedList<Object> resp = client.request(req);
- try (InputStream in = (InputStream) resp.get("stream")) {
- String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);
- String metricName = "solr_core_update_commit_ops";
-
- return (long)
- output
- .lines()
- .filter(
- l ->
- l.startsWith(metricName)
- && l.contains("collection=\"" + collection + "\"")
- && l.contains("ops=\"commits\""))
- .mapToDouble(s -> Double.parseDouble(s.substring(s.lastIndexOf(" "))))
- .sum();
- }
- }
- }
-
- private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception {
- log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement");
- assertEquals(
- 0,
- CollectionAdminRequest.createCollection(oneInstanceCollection2, "conf1", 2, 2)
- .setCreateNodeSet("")
- .process(cloudClient)
- .getStatus());
-
- List<SolrClient> collectionClients = new ArrayList<>();
- for (int i = 0; i < 4; i++) {
- CollectionAdminResponse resp =
- CollectionAdminRequest.addReplicaToShard(oneInstanceCollection2, "shard" + ((i % 2) + 1))
- .setNode(jettys.get(0).getNodeName())
- .process(cloudClient);
- for (String coreName : resp.getCollectionCoresStatus().keySet()) {
- collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString()));
- }
- }
-
- SolrClient client1 = collectionClients.get(0);
- SolrClient client2 = collectionClients.get(1);
- SolrClient client3 = collectionClients.get(2);
- SolrClient client4 = collectionClients.get(3);
-
- // no one should be recovering
- waitForRecoveriesToFinish(
- oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()), false, true);
-
- assertAllActive(oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()));
-
- // printLayout();
-
- // TODO: enable when we don't falsely get slice1...
- // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000);
- // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000);
- client2.add(getDoc(id, "1"));
- client3.add(getDoc(id, "2"));
- client4.add(getDoc(id, "3"));
-
- client1.commit();
- SolrQuery query = new SolrQuery("*:*");
- query.set("distrib", false);
- long oneDocs = client1.query(query).getResults().getNumFound();
- long twoDocs = client2.query(query).getResults().getNumFound();
- long threeDocs = client3.query(query).getResults().getNumFound();
- long fourDocs = client4.query(query).getResults().getNumFound();
-
- query.set("collection", oneInstanceCollection2);
- query.set("distrib", true);
- long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound();
-
- // System.out.println("1:" + oneDocs);
- // System.out.println("2:" + twoDocs);
- // System.out.println("3:" + threeDocs);
- // System.out.println("4:" + fourDocs);
- // System.out.println("All Docs:" + allDocs);
-
- // assertEquals(oneDocs, threeDocs);
- // assertEquals(twoDocs, fourDocs);
- // assertNotSame(oneDocs, twoDocs);
- assertEquals(3, allDocs);
-
- // we added a role of none on these creates - check for it
- ZkStateReader zkStateReader = ZkStateReader.from(getCommonCloudSolrClient());
- zkStateReader.forceUpdateCollection(oneInstanceCollection2);
- Map<String, Slice> slices =
- zkStateReader.getClusterState().getCollection(oneInstanceCollection2).getSlicesMap();
- assertNotNull(slices);
-
- Replica leader =
- getCommonCloudSolrClient()
- .getClusterState()
- .getCollection(oneInstanceCollection2)
- .getLeader("shard1");
-
- // now test that unloading a core gets us a new leader
- try (SolrClient unloadClient =
- new HttpSolrClient.Builder(jettys.get(0).getBaseUrl().toString())
- .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)
- .withSocketTimeout(60000, TimeUnit.MILLISECONDS)
- .build()) {
- Unload unloadCmd = new Unload(true);
- unloadCmd.setCoreName(leader.getCoreName());
-
- String leaderUrl = leader.getCoreUrl();
-
- testExecutor.execute(
- new Runnable() {
-
- @Override
- public void run() {
- try {
- unloadClient.request(unloadCmd);
- } catch (SolrServerException e) {
- throw new RuntimeException(e);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- });
-
- try {
- ZkStateReader.from(getCommonCloudSolrClient())
- .waitForState(
- oneInstanceCollection2,
- 20000,
- TimeUnit.MILLISECONDS,
- (n, c) -> {
- try {
- if (leaderUrl.equals(
- zkStateReader.getLeaderUrl(oneInstanceCollection2, "shard1", 10000))) {
- return false;
- }
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- return true;
- });
- } catch (TimeoutException | InterruptedException e) {
- fail("Leader never changed");
- }
- }
-
- IOUtils.close(collectionClients);
- }
-
- private void testSearchByCollectionName() throws SolrServerException, IOException {
- log.info("### STARTING testSearchByCollectionName");
- SolrClient client = clients.get(0);
- final String baseUrl = ((HttpSolrClient) client).getBaseURL();
-
- // the cores each have different names, but if we add the collection name to the url
- // we should get mapped to the right core
- try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) {
- SolrQuery query = new SolrQuery("*:*");
- long oneDocs = client1.query(query).getResults().getNumFound();
- assertEquals(3, oneDocs);
- }
- }
-
- private void testUpdateByCollectionName() throws SolrServerException, IOException {
- log.info("### STARTING testUpdateByCollectionName");
- SolrClient client = clients.get(0);
- final String baseUrl = ((HttpSolrClient) client).getBaseURL();
-
- // the cores each have different names, but if we add the collection name to the url
- // we should get mapped to the right core
- // test hitting an update url
- try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) {
- client1.commit();
- }
- }
-
- private void testANewCollectionInOneInstance() throws Exception {
- log.info("### STARTING testANewCollectionInOneInstance");
- CollectionAdminResponse response =
- CollectionAdminRequest.createCollection(oneInstanceCollection, "conf1", 2, 2)
- .setCreateNodeSet(jettys.get(0).getNodeName())
- .process(cloudClient);
- assertEquals(0, response.getStatus());
- List<SolrClient> collectionClients = new ArrayList<>();
- for (String coreName : response.getCollectionCoresStatus().keySet()) {
- collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString()));
- }
-
- SolrClient client1 = collectionClients.get(0);
- SolrClient client2 = collectionClients.get(1);
- SolrClient client3 = collectionClients.get(2);
- SolrClient client4 = collectionClients.get(3);
-
- waitForRecoveriesToFinish(
- oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()), false);
- assertAllActive(oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()));
-
- client2.add(getDoc(id, "1"));
- client3.add(getDoc(id, "2"));
- client4.add(getDoc(id, "3"));
-
- client1.commit();
- SolrQuery query = new SolrQuery("*:*");
- query.set("distrib", false);
- long oneDocs = client1.query(query).getResults().getNumFound();
- long twoDocs = client2.query(query).getResults().getNumFound();
- long threeDocs = client3.query(query).getResults().getNumFound();
- long fourDocs = client4.query(query).getResults().getNumFound();
-
- query.set("collection", oneInstanceCollection);
- query.set("distrib", true);
- long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound();
-
- // System.out.println("1:" + oneDocs);
- // System.out.println("2:" + twoDocs);
- // System.out.println("3:" + threeDocs);
- // System.out.println("4:" + fourDocs);
- // System.out.println("All Docs:" + allDocs);
-
- assertEquals(3, allDocs);
- IOUtils.close(collectionClients);
- }
-
- private void createSolrCore(
- final String collection,
- List<SolrClient> collectionClients,
- final String baseUrl,
- final int num,
- final String shardId) {
- Callable<Object> call =
- () -> {
- try (SolrClient client = getHttpSolrClient(baseUrl)) {
- // client.setConnectionTimeout(15000);
- Create createCmd = new Create();
- createCmd.setCoreName(collection + num);
- createCmd.setCollection(collection);
-
- if (random().nextBoolean()) {
- // sometimes we use an explicit core node name
- createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet());
- }
-
- if (shardId == null) {
- createCmd.setNumShards(2);
- }
- createCmd.setDataDir(getDataDir(createTempDir(collection).toString()));
- if (shardId != null) {
- createCmd.setShardId(shardId);
- }
- client.request(createCmd);
- } catch (Exception e) {
- log.error("error creating core", e);
- // fail
- }
- return null;
- };
-
- pending.add(completionService.submit(call));
-
- collectionClients.add(createNewSolrClient(collection + num, baseUrl));
- }
-
- private void testMultipleCollections() throws Exception {
- log.info("### STARTING testMultipleCollections");
- // create another 2 collections and search across them
- createNewCollection("collection2");
- createNewCollection("collection3");
-
- while (pending != null && pending.size() > 0) {
-
- Future<Object> future = completionService.take();
- if (future == null) return;
- pending.remove(future);
- }
-
- indexDoc("collection2", getDoc(id, "10000000"));
- indexDoc("collection2", getDoc(id, "10000001"));
- indexDoc("collection2", getDoc(id, "10000003"));
-
- getSolrClient("collection2").add(getDoc(id, "10000004"));
-
- indexDoc("collection3", getDoc(id, "20000000"));
- indexDoc("collection3", getDoc(id, "20000001"));
-
- getSolrClient("collection3").add(getDoc(id, "10000005"));
-
- otherCollectionClients.get("collection2").get(0).commit();
- otherCollectionClients.get("collection3").get(0).commit();
-
- SolrClient clientForCollection1 = getSolrClient("collection1");
- long collection1Docs =
- clientForCollection1.query(new SolrQuery("*:*")).getResults().getNumFound();
-
- long collection2Docs =
- otherCollectionClients
- .get("collection2")
- .get(0)
- .query(new SolrQuery("*:*"))
- .getResults()
- .getNumFound();
-
- long collection3Docs =
- otherCollectionClients
- .get("collection3")
- .get(0)
- .query(new SolrQuery("*:*"))
- .getResults()
- .getNumFound();
-
- SolrQuery query = new SolrQuery("*:*");
- query.set("collection", "collection2,collection3");
- long found = clients.get(0).query(query).getResults().getNumFound();
- assertEquals(collection2Docs + collection3Docs, found);
-
- query = new SolrQuery("*:*");
- query.set("collection", "collection1,collection2,collection3");
- found = clients.get(0).query(query).getResults().getNumFound();
- assertEquals(collection1Docs + collection2Docs + collection3Docs, found);
-
- // try to search multiple with cloud client
- found = getCommonCloudSolrClient().query(query).getResults().getNumFound();
- assertEquals(collection1Docs + collection2Docs + collection3Docs, found);
-
- query.set("collection", "collection2,collection3");
- found = getCommonCloudSolrClient().query(query).getResults().getNumFound();
- assertEquals(collection2Docs + collection3Docs, found);
-
- query.set("collection", "collection3");
- found = getCommonCloudSolrClient().query(query).getResults().getNumFound();
- assertEquals(collection3Docs, found);
-
- query.remove("collection");
- found = getCommonCloudSolrClient().query(query).getResults().getNumFound();
- assertEquals(collection1Docs, found);
-
- assertEquals(collection3Docs, collection2Docs - 1);
- }
-
- protected void indexDoc(String collection, SolrInputDocument doc)
- throws IOException, SolrServerException {
- List<SolrClient> clients = otherCollectionClients.get(collection);
- int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size();
- SolrClient client = clients.get(which);
- client.add(doc);
- }
-
- @SuppressWarnings({"unchecked"})
- private void createNewCollection(final String collection) throws InterruptedException {
- try {
- assertEquals(
- 0,
- CollectionAdminRequest.createCollection(collection, "conf1", 2, 1)
- .setCreateNodeSet("")
- .process(cloudClient)
- .getStatus());
- } catch (Exception e) {
- log.error("error creating collection", e);
- // fails
- }
- final List<SolrClient> collectionClients = new ArrayList<>();
- otherCollectionClients.put(collection, collectionClients);
- int unique = 0;
- for (final JettySolrRunner runner : jettys) {
- unique++;
- final int frozeUnique = unique;
- Callable<Object> call =
- () -> {
- try {
- assertTrue(
- CollectionAdminRequest.addReplicaToShard(
- collection, "shard" + ((frozeUnique % 2) + 1))
- .setNode(runner.getNodeName())
- .process(cloudClient)
- .isSuccess());
- } catch (Exception e) {
- log.error("error adding replica", e);
- // fails
- }
- return null;
- };
-
- collectionClients.add(createNewSolrClient(collection, runner.getBaseUrl().toString()));
- pending.add(completionService.submit(call));
- while (pending != null && pending.size() > 0) {
-
- Future<Object> future = completionService.take();
- if (future == null) return;
- pending.remove(future);
- }
- }
- }
-
- @Override
- protected SolrClient createNewSolrClient(String collection, String baseUrl) {
-
- SolrClient client = getHttpSolrClient(baseUrl, collection);
-
- return client;
- }
-
- /**
- * @param collection the name of a collection or core to set as the "default" on the created
- * client.
- * @param baseUrl the "base" URL of a Solr node. Should <em>not</em> contain a collection or core
- * name.
- * @param connectionTimeoutMillis the HTTP connection timeout in milliseconds
- * @param socketTimeoutMillis the HTTP socket-read timeout in milliseconds
- */
- protected SolrClient createNewSolrClient(
- String collection, String baseUrl, int connectionTimeoutMillis, int socketTimeoutMillis) {
-
- SolrClient client =
- new HttpSolrClient.Builder(baseUrl)
- .withDefaultCollection(collection)
- .withConnectionTimeout(connectionTimeoutMillis, TimeUnit.MILLISECONDS)
- .withSocketTimeout(socketTimeoutMillis, TimeUnit.MILLISECONDS)
- .build();
-
- return client;
- }
-
- @Override
- protected QueryResponse queryRandomShard(ModifiableSolrParams params)
- throws SolrServerException, IOException {
-
- if (r.nextBoolean()) return super.queryRandomShard(params);
-
- if (r.nextBoolean()) params.set("collection", DEFAULT_COLLECTION);
-
- QueryResponse rsp = getCommonCloudSolrClient().query(params);
- return rsp;
- }
-
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
- if (otherCollectionClients != null) {
- for (List<SolrClient> clientList : otherCollectionClients.values()) {
- IOUtils.close(clientList);
- }
- }
- otherCollectionClients = null;
- List<Runnable> tasks = executor.shutdownNow();
- assertTrue(tasks.isEmpty());
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java
deleted file mode 100644
index 6d55d1f560b..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java
+++ /dev/null
@@ -1,338 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.solr.client.solrj.apache.CloudLegacySolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.SolrQuery;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public abstract class AbstractChaosMonkeyNothingIsSafeTestBase
- extends AbstractFullDistribZkTestBase {
- private static final int FAIL_TOLERANCE = 100;
-
- private static final Integer RUN_LENGTH =
- Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
-
- private final boolean onlyLeaderIndexes = random().nextBoolean();
-
- @BeforeClass
- public static void beforeSuperClass() {
- schemaString = "schema15.xml"; // we need a string id
- System.setProperty("solr.autoCommit.maxTime", "15000");
- System.clearProperty("solr.httpclient.retries");
- System.clearProperty("solr.retries.on.forward");
- System.clearProperty("solr.retries.to.followers");
- setErrorHook();
- }
-
- @AfterClass
- public static void afterSuperClass() {
- System.clearProperty("solr.autoCommit.maxTime");
- clearErrorHook();
- }
-
- @Override
- protected void destroyServers() throws Exception {
-
- super.destroyServers();
- }
-
- protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};
- protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};
-
- private int clientSoTimeout = 60000;
-
- private volatile FullThrottleStoppableIndexingThread ftIndexThread;
-
- private final boolean runFullThrottle;
-
- @Override
- public String[] getFieldNames() {
- return fieldNames;
- }
-
- @Override
- public RandVal[] getRandValues() {
- return randVals;
- }
-
- @Override
- public void distribSetUp() throws Exception {
- super.distribSetUp();
- // can help to hide this when testing and looking at logs
- // ignoreException("shard update error");
- useFactory("solr.StandardDirectoryFactory");
- }
-
- @Override
- public void distribTearDown() throws Exception {
- try {
- ftIndexThread.safeStop();
- } catch (NullPointerException e) {
- // okay
- }
- super.distribTearDown();
- }
-
- public AbstractChaosMonkeyNothingIsSafeTestBase() {
- super();
- sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1"));
- if (sliceCount == -1) {
- sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1;
- }
-
- int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1"));
- if (numShards == -1) {
- // we make sure that there's at least one shard with more than one replica
- // so that the ChaosMonkey has something to kill
- numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1;
- }
- fixShardCount(numShards);
-
- // TODO: we only do this sometimes so that we can sometimes compare against control,
- // it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer
- runFullThrottle = random().nextBoolean();
- }
-
- @Override
- protected boolean useTlogReplicas() {
- return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug
- // and it's TestInjection use
- }
-
- @Override
- protected CloudSolrClient createCloudClient(String defaultCollection) {
- return this.createCloudClient(defaultCollection, this.clientSoTimeout);
- }
-
- protected CloudSolrClient createCloudClient(String defaultCollection, int socketTimeout) {
-
- return getCloudSolrClient(
- zkServer.getZkAddress(), defaultCollection, random().nextBoolean(), 30000, socketTimeout);
- }
-
- @Test
- @SuppressWarnings({"try"})
- public void test() throws Exception {
- // None of the operations used here are particularly costly, so this should work.
- // Using this low timeout will also help us catch index stalling.
- clientSoTimeout = 5000;
-
- boolean testSuccessful = false;
- try (CloudSolrClient ourCloudClient = createCloudClient(DEFAULT_COLLECTION)) {
- handle.clear();
- handle.put("timestamp", SKIPVAL);
- ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
- // make sure we have leaders for each shard
- for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
- } // make sure we again have leaders for each shard
-
- waitForRecoveriesToFinish(false);
-
- // we cannot do delete by query
- // as it's not supported for recovery
- del("*:*");
-
- List<StoppableThread> threads = new ArrayList<>();
- List<StoppableIndexingThread> indexTreads = new ArrayList<>();
- int threadCount = TEST_NIGHTLY ? 3 : 1;
- int i = 0;
- for (i = 0; i < threadCount; i++) {
- StoppableIndexingThread indexThread =
- new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true);
- threads.add(indexThread);
- indexTreads.add(indexThread);
- indexThread.start();
- }
-
- threadCount = 1;
- i = 0;
- for (i = 0; i < threadCount; i++) {
- StoppableSearchThread searchThread = new StoppableSearchThread(cloudClient);
- threads.add(searchThread);
- searchThread.start();
- }
-
- if (runFullThrottle) {
- ftIndexThread =
- new FullThrottleStoppableIndexingThread(
- ((CloudLegacySolrClient) cloudClient).getHttpClient(),
- controlClient,
- cloudClient,
- clients,
- "ft1",
- true,
- this.clientSoTimeout);
- ftIndexThread.start();
- }
-
- chaosMonkey.startTheMonkey(true, 10000);
- try {
- long runLength;
- if (RUN_LENGTH != -1) {
- runLength = RUN_LENGTH;
- } else {
- int[] runTimes;
- if (TEST_NIGHTLY) {
- runTimes =
- new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000};
- } else {
- runTimes = new int[] {5000, 7000, 15000};
- }
- runLength = runTimes[random().nextInt(runTimes.length - 1)];
- }
-
- Thread.sleep(runLength);
- } finally {
- chaosMonkey.stopTheMonkey();
- }
-
- // ideally this should go into chaosMonkey
- restartZk(1000 * (5 + random().nextInt(4)));
-
- if (runFullThrottle) {
- ftIndexThread.safeStop();
- }
-
- for (StoppableThread indexThread : threads) {
- indexThread.safeStop();
- }
-
- // start any downed jetties to be sure we still will end up with a leader per shard...
-
- // wait for stop...
- for (StoppableThread indexThread : threads) {
- indexThread.join();
- }
-
- // try and wait for any replications and what not to finish...
-
- Thread.sleep(2000);
-
- // wait until there are no recoveries...
- waitForThingsToLevelOut();
-
- // make sure we again have leaders for each shard
- for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000);
- }
-
- commit();
-
- // TODO: assert we didn't kill everyone
-
- zkStateReader.updateLiveNodes();
- assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0);
-
- // we expect full throttle fails, but cloud client should not easily fail
- for (StoppableThread indexThread : threads) {
- if (indexThread instanceof StoppableIndexingThread
- && !(indexThread instanceof FullThrottleStoppableIndexingThread)) {
- int failCount = ((StoppableIndexingThread) indexThread).getFailCount();
- assertFalse(
- "There were too many update fails ("
- + failCount
- + " > "
- + FAIL_TOLERANCE
- + ") - we expect it can happen, but shouldn't easily",
- failCount > FAIL_TOLERANCE);
- }
- }
-
- waitForThingsToLevelOut(20, TimeUnit.SECONDS);
-
- commit();
-
- Set<String> addFails = getAddFails(indexTreads);
- Set<String> deleteFails = getDeleteFails(indexTreads);
- // full throttle thread can
- // have request fails
- checkShardConsistency(!runFullThrottle, true, addFails, deleteFails);
-
- long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();
-
- // ensure we have added more than 0 docs
- long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
-
- assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0);
-
- if (VERBOSE)
- System.out.println(
- "control docs:"
- + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()
- + "\n\n");
-
- // try and make a collection to make sure the overseer has survived the expiration and session
- // loss
-
- // sometimes we restart zookeeper as well
- if (random().nextBoolean()) {
- // restartZk(1000 * (5 + random().nextInt(4)));
- }
-
- try (CloudSolrClient client = createCloudClient("collection1", 30000)) {
- createCollection(null, "testcollection", 1, 1, client, null, "conf1");
- }
- List<Integer> numShardsNumReplicas = new ArrayList<>(2);
- numShardsNumReplicas.add(1);
- numShardsNumReplicas.add(1);
- checkForCollection("testcollection", numShardsNumReplicas);
-
- testSuccessful = true;
- } finally {
- if (!testSuccessful) {
- printLayout();
- }
- }
- }
-
- private Set<String> getAddFails(List<StoppableIndexingThread> threads) {
- Set<String> addFails = new HashSet<>();
- for (StoppableIndexingThread thread : threads) {
- addFails.addAll(thread.getAddFails());
- }
- return addFails;
- }
-
- private Set<String> getDeleteFails(List<StoppableIndexingThread> threads) {
- Set<String> deleteFails = new HashSet<>();
- for (StoppableIndexingThread thread : threads) {
- deleteFails.addAll(thread.getDeleteFails());
- }
- return deleteFails;
- }
-
- // skip the randoms - they can deadlock...
- @Override
- protected void indexr(Object... fields) throws Exception {
- SolrInputDocument doc = getDoc(fields);
- indexDoc(doc);
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java
deleted file mode 100644
index 5d94eae05e3..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.SolrQuery;
-import org.apache.solr.common.SolrInputDocument;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class AbstractChaosMonkeySafeLeaderTestBase extends AbstractFullDistribZkTestBase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final Integer RUN_LENGTH =
- Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
-
- @BeforeClass
- public static void beforeSuperClass() {
- schemaString = "schema15.xml"; // we need a string id
- System.setProperty("solr.autoCommit.maxTime", "15000");
- System.clearProperty("solr.httpclient.retries");
- System.clearProperty("solr.retries.on.forward");
- System.clearProperty("solr.retries.to.followers");
- setErrorHook();
- }
-
- @AfterClass
- public static void afterSuperClass() {
- System.clearProperty("solr.autoCommit.maxTime");
- clearErrorHook();
- }
-
- protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};
- protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};
-
- @Override
- public String[] getFieldNames() {
- return fieldNames;
- }
-
- @Override
- public RandVal[] getRandValues() {
- return randVals;
- }
-
- protected abstract String getDirectoryFactory();
-
- @Override
- public void distribSetUp() throws Exception {
- useFactory(getDirectoryFactory());
- super.distribSetUp();
- }
-
- public AbstractChaosMonkeySafeLeaderTestBase() {
- super();
- sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1"));
- if (sliceCount == -1) {
- sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1;
- }
-
- int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1"));
- if (numShards == -1) {
- // we make sure that there's at least one shard with more than one replica
- // so that the ChaosMonkey has something to kill
- numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1;
- }
- fixShardCount(numShards);
- }
-
- @Test
- public void test() throws Exception {
-
- handle.clear();
- handle.put("timestamp", SKIPVAL);
-
- // randomly turn on 1 seconds 'soft' commit
- randomlyEnableAutoSoftCommit();
-
- tryDelete();
-
- List<StoppableIndexingThread> threads = new ArrayList<>();
- int threadCount = 2;
- int batchSize = 1;
- if (random().nextBoolean()) {
- batchSize = random().nextInt(98) + 2;
- }
-
- boolean pauseBetweenUpdates = TEST_NIGHTLY ? random().nextBoolean() : true;
- int maxUpdates = -1;
- if (!pauseBetweenUpdates) {
- maxUpdates = 1000 + random().nextInt(1000);
- } else {
- maxUpdates = 15000;
- }
-
- for (int i = 0; i < threadCount; i++) {
- StoppableIndexingThread indexThread =
- new StoppableIndexingThread(
- controlClient,
- cloudClient,
- Integer.toString(i),
- true,
- maxUpdates,
- batchSize,
- pauseBetweenUpdates); // random().nextInt(999) + 1
- threads.add(indexThread);
- indexThread.start();
- }
-
- chaosMonkey.startTheMonkey(false, 500);
- try {
- long runLength;
- if (RUN_LENGTH != -1) {
- runLength = RUN_LENGTH;
- } else {
- int[] runTimes;
- if (TEST_NIGHTLY) {
- runTimes =
- new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000};
- } else {
- runTimes = new int[] {5000, 7000, 15000};
- }
- runLength = runTimes[random().nextInt(runTimes.length - 1)];
- }
-
- Thread.sleep(runLength);
- } finally {
- chaosMonkey.stopTheMonkey();
- }
-
- for (StoppableIndexingThread indexThread : threads) {
- indexThread.safeStop();
- }
-
- // wait for stop...
- for (StoppableIndexingThread indexThread : threads) {
- indexThread.join();
- }
-
- for (StoppableIndexingThread indexThread : threads) {
- assertEquals(0, indexThread.getFailCount());
- }
-
- // try and wait for any replications and what not to finish...
-
- Thread.sleep(2000);
-
- waitForThingsToLevelOut(3, TimeUnit.MINUTES);
-
- // even if things were leveled out, a jetty may have just been stopped or something
- // we wait again and wait to level out again to make sure the system is not still in flux
-
- Thread.sleep(3000);
-
- waitForThingsToLevelOut(3, TimeUnit.MINUTES);
-
- checkShardConsistency(batchSize == 1, true);
-
- if (VERBOSE)
- System.out.println(
- "control docs:"
- + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()
- + "\n\n");
-
- // try and make a collection to make sure the overseer has survived the expiration and session
- // loss
-
- // sometimes we restart zookeeper as well
- if (random().nextBoolean()) {
- zkServer.shutdown();
- zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());
- zkServer.run(false);
- }
-
- try (CloudSolrClient client = createCloudClient("collection1")) {
- createCollection(null, "testcollection", 1, 1, client, null, "conf1");
- }
- List<Integer> numShardsNumReplicas = new ArrayList<>(2);
- numShardsNumReplicas.add(1);
- numShardsNumReplicas.add(1);
- checkForCollection("testcollection", numShardsNumReplicas);
- }
-
- private void tryDelete() throws Exception {
- long start = System.nanoTime();
- long timeout = start + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS);
- while (System.nanoTime() < timeout) {
- try {
- del("*:*");
- break;
- } catch (SolrServerException e) {
- log.error("cluster may not be up yet", e);
- }
- Thread.sleep(100);
- }
- }
-
- // skip the randoms - they can deadlock...
- @Override
- protected void indexr(Object... fields) throws Exception {
- SolrInputDocument doc = new SolrInputDocument();
- addFields(doc, fields);
- addFields(doc, "rnd_b", true);
- indexDoc(doc);
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java
deleted file mode 100644
index 31bfdb837c6..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java
+++ /dev/null
@@ -1,567 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_FILE_VM_PARAM_NAME;
-import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME;
-import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME;
-import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE;
-
-import java.io.FileWriter;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
-import java.util.Collections;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.cloud.DigestZkACLProvider;
-import org.apache.solr.common.cloud.DigestZkCredentialsProvider;
-import org.apache.solr.common.cloud.SecurityAwareZkACLProvider;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.VMParamsZkCredentialsInjector;
-import org.apache.solr.common.cloud.ZkCredentialsInjector;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException.NoAuthException;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class AbstractDigestZkACLAndCredentialsProvidersTestBase extends SolrTestCaseJ4 {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final Charset DATA_ENCODING = StandardCharsets.UTF_8;
-
- private static final String ALL_USERNAME = "connectAndAllACLUsername";
- private static final String ALL_PASSWORD = "connectAndAllACLPassword";
- private static final String READONLY_USERNAME = "readonlyACLUsername";
- private static final String READONLY_PASSWORD = "readonlyACLPassword";
-
- protected ZkTestServer zkServer;
-
- protected Path zkDir;
-
- @BeforeClass
- public static void beforeClass() {
- System.setProperty("solrcloud.skip.autorecovery", "true");
- }
-
- @AfterClass
- public static void afterClass() {
- System.clearProperty("solrcloud.skip.autorecovery");
- }
-
- @Override
- public void setUp() throws Exception {
- // TODO: Does all of this setup need to happen for each test case, or can it be done once for
- // the class? (i.e. @BeforeClass) and maybe some minor reset logic in setup, instead of full
- // startup and teardown of a new ZkTestServer in each cycle?
- super.setUp();
- if (log.isInfoEnabled()) {
- log.info("####SETUP_START {}", getTestName());
- }
- createTempDir();
-
- zkDir = createTempDir().resolve("zookeeper/server1/data");
- log.info("ZooKeeper dataDir:{}", zkDir);
- setSecuritySystemProperties();
- zkServer = new ZkTestServer(zkDir);
- zkServer.run(false);
-
- System.setProperty("zkHost", zkServer.getZkAddress());
-
- setDigestZkSystemProps();
- System.setProperty(
- SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,
- AllAndReadonlyCredentialZkCredentialsInjector.class.getName());
-
- SolrZkClient zkClient =
- new SolrZkClient.Builder()
- .withUrl(zkServer.getZkHost())
- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
- .withConnTimeOut(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
- .build();
-
- zkClient.makePath("/solr", false, true);
- zkClient.close();
-
- zkClient =
- new SolrZkClient.Builder()
- .withUrl(zkServer.getZkAddress())
- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
- .build();
- zkClient.create(
- "/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
- zkClient.makePath(
- "/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
-
- zkClient.create(
- SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH,
- "content".getBytes(DATA_ENCODING),
- CreateMode.PERSISTENT,
- false);
- zkClient.close();
-
- clearSecuritySystemProperties();
-
- zkClient =
- new SolrZkClient.Builder()
- .withUrl(zkServer.getZkAddress())
- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
- .build();
- // Currently, no credentials on ZK connection, because those same VM-params are used for adding
- // ACLs, and here we want
- // no (or completely open) ACLs added. Therefore, hack your way into being authorized for
- // creating anyway
- zkClient
- .getCuratorFramework()
- .getZookeeperClient()
- .getZooKeeper()
- .addAuthInfo(
- "digest", (ALL_USERNAME + ":" + ALL_PASSWORD).getBytes(StandardCharsets.UTF_8));
- zkClient.create(
- "/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
- zkClient.makePath(
- "/unprotectedMakePathNode",
- "content".getBytes(DATA_ENCODING),
- CreateMode.PERSISTENT,
- false);
- zkClient.close();
-
- setDigestZkSystemProps();
- if (log.isInfoEnabled()) {
- log.info("####SETUP_END {}", getTestName());
- }
- }
-
- private void setDigestZkSystemProps() {
- System.setProperty(
- SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME,
- DigestZkCredentialsProvider.class.getName());
- System.setProperty(
- SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName());
- }
-
- @Override
- public void tearDown() throws Exception {
- zkServer.shutdown();
-
- clearSecuritySystemProperties();
-
- super.tearDown();
- }
-
- @Test
- public void testNoCredentials() throws Exception {
- List<TestZkCredentialsInjector> testZkCredentialsInjectors =
- List.of(
- new TestZkCredentialsInjector(NoCredentialZkCredentialsInjector.class),
- new TestZkCredentialsInjector(VMParamsZkCredentialsInjector.class));
-
- testInjectors(
- testZkCredentialsInjectors,
- false,
- false,
- false,
- false,
- false,
- false,
- false,
- false,
- false,
- false);
- }
-
- @Test
- public void testWrongCredentials() throws Exception {
- List<TestZkCredentialsInjector> testZkCredentialsInjectors =
- List.of(
- new TestZkCredentialsInjector(WrongAllCredentialZkCredentialsInjector.class),
- new TestZkCredentialsInjector(
- VMParamsZkCredentialsInjector.class,
- List.of(
- DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),
- List.of(ALL_USERNAME, ALL_PASSWORD + "Wrong")));
-
- testInjectors(
- testZkCredentialsInjectors,
- false,
- false,
- false,
- false,
- false,
- false,
- false,
- false,
- false,
- false);
- }
-
- @Test
- public void testAllCredentials() throws Exception {
- List<TestZkCredentialsInjector> testZkCredentialsInjectors =
- List.of(
- new TestZkCredentialsInjector(AllCredentialZkCredentialsInjector.class),
- new TestZkCredentialsInjector(
- VMParamsZkCredentialsInjector.class,
- List.of(
- DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),
- List.of(ALL_USERNAME, ALL_PASSWORD)));
-
- testInjectors(
- testZkCredentialsInjectors, true, true, true, true, true, true, true, true, true, true);
- }
-
- @Test
- public void testReadonlyCredentials() throws Exception {
- List<TestZkCredentialsInjector> testZkCredentialsInjectors =
- List.of(
- new TestZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class),
- new TestZkCredentialsInjector(
- VMParamsZkCredentialsInjector.class,
- List.of(
- DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME),
- List.of(READONLY_USERNAME, READONLY_PASSWORD)));
- testInjectors(
- testZkCredentialsInjectors,
- true,
- true,
- false,
- false,
- false,
- false,
- false,
- false,
- false,
- false);
- }
-
- protected void testInjectors(
- List<TestZkCredentialsInjector> testZkCredentialsInjectors,
- boolean getData,
- boolean list,
- boolean create,
- boolean setData,
- boolean delete,
- boolean secureGet,
- boolean secureList,
- boolean secureCreate,
- boolean secureSet,
- boolean secureDelete)
- throws Exception {
- for (TestZkCredentialsInjector testZkCredentialsInjector : testZkCredentialsInjectors) {
- tearDown();
- setUp();
- testZkCredentialsInjector.setSystemProps();
- try (SolrZkClient zkClient =
- new SolrZkClient.Builder()
- .withUrl(zkServer.getZkAddress())
- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
- .build()) {
- doTest(
- zkClient,
- getData,
- list,
- create,
- setData,
- delete,
- secureGet,
- secureList,
- secureCreate,
- secureSet,
- secureDelete);
- }
- }
- }
-
- @Test
- public void testRepairACL() throws Exception {
- clearSecuritySystemProperties();
- try (SolrZkClient zkClient =
- new SolrZkClient.Builder()
- .withUrl(zkServer.getZkAddress())
- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
- .build()) {
- // Currently, no credentials on ZK connection, because those same VM-params are used for
- // adding ACLs, and here we want
- // no (or completely open) ACLs added. Therefore, hack your way into being authorized for
- // creating anyway
- zkClient
- .getCuratorFramework()
- .getZookeeperClient()
- .getZooKeeper()
- .addAuthInfo(
- "digest",
- ("connectAndAllACLUsername:connectAndAllACLPassword")
- .getBytes(StandardCharsets.UTF_8));
-
- zkClient.create(
- "/security.json", "{}".getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, false);
- assertEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false));
- }
-
- setSecuritySystemProperties();
- try (SolrZkClient zkClient =
- new SolrZkClient.Builder()
- .withUrl(zkServer.getZkAddress())
- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
- .build()) {
- ZkController.createClusterZkNodes(zkClient);
- assertNotEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false));
- }
-
- useZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class);
- // useReadonlyCredentials();
- try (SolrZkClient zkClient =
- new SolrZkClient.Builder()
- .withUrl(zkServer.getZkAddress())
- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
- .build()) {
- NoAuthException e =
- assertThrows(
- NoAuthException.class, () -> zkClient.getData("/security.json", null, null, false));
- assertEquals("/solr/security.json", e.getPath());
- }
- }
-
- private void useZkCredentialsInjector(Class<?> zkCredentialsInjectorClass) {
- clearSecuritySystemProperties();
- setDigestZkSystemProps();
- System.setProperty(
- SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,
- zkCredentialsInjectorClass.getName());
- }
-
- private void setSecuritySystemProperties() {
- System.setProperty(
- SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME,
- DigestZkCredentialsProvider.class.getName());
- System.setProperty(
- SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName());
- System.setProperty(
- SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,
- AllAndReadonlyCredentialZkCredentialsInjector.class.getName());
- }
-
- private void clearSecuritySystemProperties() {
- System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME);
- System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME);
- System.clearProperty(SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME);
- }
-
- public static void doTest(
- SolrZkClient zkClient,
- boolean getData,
- boolean list,
- boolean create,
- boolean setData,
- boolean delete,
- boolean secureGet,
- boolean secureList,
- boolean secureCreate,
- boolean secureSet,
- boolean secureDelete)
- throws Exception {
- doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete);
- doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete);
- doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete);
- doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete);
- doTest(
- zkClient,
- SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH,
- secureGet,
- secureList,
- secureCreate,
- secureSet,
- secureDelete);
- }
-
- protected static void doTest(
- SolrZkClient zkClient,
- String path,
- boolean getData,
- boolean list,
- boolean create,
- boolean setData,
- boolean delete)
- throws Exception {
- doTest(getData, () -> zkClient.getData(path, null, null, false));
- doTest(list, () -> zkClient.getChildren(path, null, false));
-
- doTest(
- create,
- () -> {
- zkClient.create(path + "/subnode", null, CreateMode.PERSISTENT, false);
- zkClient.delete(path + "/subnode", -1, false);
- });
- doTest(
- create,
- () -> {
- zkClient.makePath(path + "/subnode/subsubnode", false);
- zkClient.delete(path + "/subnode/subsubnode", -1, false);
- zkClient.delete(path + "/subnode", -1, false);
- });
-
- doTest(setData, () -> zkClient.setData(path, (byte[]) null, false));
-
- // Actually about the ACLs on /solr, but that is protected
- doTest(delete, () -> zkClient.delete(path, -1, false));
- }
-
- interface ExceptingRunnable {
- void run() throws Exception;
- }
-
- private static void doTest(boolean shouldSucceed, ExceptingRunnable action) throws Exception {
- if (shouldSucceed) {
- action.run();
- } else {
- expectThrows(NoAuthException.class, action::run);
- }
- }
-
- @Test
- public void testVMParamsAllCredentialsFromFile() throws Exception {
- useVMParamsAllCredentialsFromFile();
-
- try (SolrZkClient zkClient =
- new SolrZkClient.Builder()
- .withUrl(zkServer.getZkAddress())
- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
- .build()) {
- doTest(zkClient, true, true, true, true, true, true, true, true, true, true);
- }
- }
-
- @Test
- public void testVMParamsReadonlyCredentialsFromFile() throws Exception {
- useVMParamsReadonlyCredentialsFromFile();
-
- try (SolrZkClient zkClient =
- new SolrZkClient.Builder()
- .withUrl(zkServer.getZkAddress())
- .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS)
- .build()) {
- doTest(zkClient, true, true, false, false, false, false, false, false, false, false);
- }
- }
-
- private void useVMParamsAllCredentialsFromFile() throws IOException {
- useVMParamsCredentialsFromFile("connectAndAllACLUsername", "connectAndAllACLPassword");
- }
-
- private void useVMParamsReadonlyCredentialsFromFile() throws IOException {
- useVMParamsCredentialsFromFile("readonlyACLUsername", "readonlyACLPassword");
- }
-
- private void useVMParamsCredentialsFromFile(String username, String password) throws IOException {
- Properties props = new Properties();
- props.setProperty(
- VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, username);
- props.setProperty(
- VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, password);
- String credsFile = saveCredentialsFile(props);
-
- useZkCredentialsInjector(VMParamsZkCredentialsInjector.class);
- System.setProperty(DEFAULT_DIGEST_FILE_VM_PARAM_NAME, credsFile);
- }
-
- private String saveCredentialsFile(Properties props) throws IOException {
- Path tmp = createTempFile("zk-creds", "properties");
- try (FileWriter w = new FileWriter(tmp.toFile(), StandardCharsets.UTF_8)) {
- props.store(w, "test");
- }
- return tmp.toAbsolutePath().toString();
- }
-
- public static class NoCredentialZkCredentialsInjector implements ZkCredentialsInjector {
- @Override
- public List<ZkCredential> getZkCredentials() {
- return Collections.emptyList();
- }
- }
-
- public static class AllAndReadonlyCredentialZkCredentialsInjector
- implements ZkCredentialsInjector {
- @Override
- public List<ZkCredential> getZkCredentials() {
- return List.of(
- new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL),
- new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.READ));
- }
- }
-
- public static class ConnectWithReadonlyCredsInjector implements ZkCredentialsInjector {
- @Override
- public List<ZkCredential> getZkCredentials() {
- return List.of(
- new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.ALL));
- }
- }
-
- public static class AllCredentialZkCredentialsInjector implements ZkCredentialsInjector {
- @Override
- public List<ZkCredential> getZkCredentials() {
- return List.of(new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL));
- }
- }
-
- public static class WrongAllCredentialZkCredentialsInjector implements ZkCredentialsInjector {
- @Override
- public List<ZkCredential> getZkCredentials() {
- return List.of(
- new ZkCredential(ALL_USERNAME, ALL_PASSWORD + "Wrong", ZkCredential.Perms.ALL));
- }
- }
-
- class TestZkCredentialsInjector {
- private final Class<?> zkCredentialsInjectorClass;
- private final List<String> systemPropsKeys;
- private final List<String> systemPropsValues;
-
- public TestZkCredentialsInjector(Class<?> zkCredentialsInjectorClass) {
- this(zkCredentialsInjectorClass, Collections.emptyList(), Collections.emptyList());
- }
-
- public TestZkCredentialsInjector(
- Class<?> zkCredentialsInjectorClass,
- List<String> systemPropsKeys,
- List<String> systemPropsValues) {
- this.zkCredentialsInjectorClass = zkCredentialsInjectorClass;
- this.systemPropsKeys = systemPropsKeys;
- this.systemPropsValues = systemPropsValues;
- }
-
- private void setSystemProps() {
- clearSecuritySystemProperties();
- setDigestZkSystemProps();
- System.setProperty(
- SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME,
- zkCredentialsInjectorClass.getName());
- int i = 0;
- for (String key : systemPropsKeys) {
- System.setProperty(key, systemPropsValues.get(i++));
- }
- }
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
index e123fa6d53f..5a24a1f2f4d 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
@@ -133,6 +133,12 @@ public abstract class AbstractFullDistribZkTestBase extends BaseDistributedSearc
protected volatile ZkTestServer zkServer;
private final AtomicInteger homeCount = new AtomicInteger();
+ @BeforeClass
+ public static void beforeThisClass() throws Exception {
+ // Only For Manual Testing: this will force an fs based dir factory
+ // useFactory(null);
+ }
+
@BeforeClass
public static void beforeFullSolrCloudTest() {}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java
deleted file mode 100644
index da9ed3da1fe..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java
+++ /dev/null
@@ -1,410 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import org.apache.solr.client.api.model.CoreStatusResponse;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.CoreAdminRequest;
-import org.apache.solr.client.solrj.request.SolrQuery;
-import org.apache.solr.client.solrj.response.CoreAdminResponse;
-import org.apache.solr.client.solrj.response.RequestStatusState;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.util.IdUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class AbstractMoveReplicaTestBase extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- // used by MoveReplicaHDFSTest
- protected boolean inPlaceMove = true;
- protected boolean isCollectionApiDistributed = false;
-
- protected String getConfigSet() {
- return "cloud-dynamic";
- }
-
- @Before
- public void beforeTest() throws Exception {
- inPlaceMove = true;
-
- configureCluster(4)
- .addConfig("conf1", configset(getConfigSet()))
- .addConfig("conf2", configset(getConfigSet()))
- .withSolrXml(TEST_PATH().resolve("solr.xml"))
- .configure();
-
- // If Collection API is distributed let's not wait for Overseer.
- isCollectionApiDistributed =
- new CollectionAdminRequest.RequestApiDistributedProcessing()
- .process(cluster.getSolrClient())
- .getIsCollectionApiDistributed();
- if (isCollectionApiDistributed) {
- return;
- }
-
- NamedList<Object> overSeerStatus =
- cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- JettySolrRunner overseerJetty = null;
- String overseerLeader = (String) overSeerStatus.get("leader");
- for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
- if (jetty.getNodeName().equals(overseerLeader)) {
- overseerJetty = jetty;
- break;
- }
- }
- if (overseerJetty == null) {
- fail("no overseer leader!");
- }
- }
-
- @After
- public void afterTest() throws Exception {
- try {
- shutdownCluster();
- } finally {
- super.tearDown();
- }
- }
-
- @Test
- public void test() throws Exception {
- String coll = getTestClass().getSimpleName() + "_coll_" + inPlaceMove;
- if (log.isInfoEnabled()) {
- log.info("total_jettys: {}", cluster.getJettySolrRunners().size());
- }
- int REPLICATION = 2;
-
- CloudSolrClient cloudClient = cluster.getSolrClient();
-
- // random create tlog or pull type replicas with nrt
- boolean isTlog = random().nextBoolean();
- CollectionAdminRequest.Create create =
- CollectionAdminRequest.createCollection(
- coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0);
- cloudClient.request(create);
-
- addDocs(coll, 100);
-
- Replica replica = getRandomReplica(coll, cloudClient);
- Set<String> liveNodes = cloudClient.getClusterState().getLiveNodes();
- ArrayList<String> l = new ArrayList<>(liveNodes);
- Collections.shuffle(l, random());
- String targetNode = null;
- for (String node : liveNodes) {
- if (!replica.getNodeName().equals(node)) {
- targetNode = node;
- break;
- }
- }
- assertNotNull(targetNode);
- String shardId = null;
- for (Slice slice : cloudClient.getClusterState().getCollection(coll).getSlices()) {
- if (slice.getReplicas().contains(replica)) {
- shardId = slice.getName();
- }
- }
-
- int sourceNumCores =
- getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name());
- int targetNumCores = getNumOfCores(cloudClient, targetNode, coll, replica.getType().name());
-
- CollectionAdminRequest.MoveReplica moveReplica =
- createMoveReplicaRequest(coll, replica, targetNode);
- moveReplica.setInPlaceMove(inPlaceMove);
- String asyncId = IdUtils.randomId();
- moveReplica.processAsync(asyncId, cloudClient);
- CollectionAdminRequest.RequestStatus requestStatus =
- CollectionAdminRequest.requestStatus(asyncId);
- // wait for async request success
- boolean success = false;
- for (int i = 0; i < 200; i++) {
- CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
- if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) {
- success = true;
- break;
- }
- assertNotSame(rsp.getRequestStatus(), RequestStatusState.FAILED);
- Thread.sleep(500);
- }
- assertTrue(success);
- assertEquals(
- "should be one less core on the source node!",
- sourceNumCores - 1,
- getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name()));
- assertEquals(
- "should be one more core on target node!",
- targetNumCores + 1,
- getNumOfCores(cloudClient, targetNode, coll, replica.getType().name()));
- // wait for recovery
- boolean recovered = false;
- for (int i = 0; i < 300; i++) {
- DocCollection collState = getCollectionState(coll);
- log.debug("###### {}", collState);
- Collection<Replica> replicas = collState.getSlice(shardId).getReplicas();
- boolean allActive = true;
- boolean hasLeaders = true;
- if (replicas != null && !replicas.isEmpty()) {
- for (Replica r : replicas) {
- if (!r.getNodeName().equals(targetNode)) {
- continue;
- }
- if (!r.isActive(Collections.singleton(targetNode))) {
- log.info("Not active: {}", r);
- allActive = false;
- }
- }
- } else {
- allActive = false;
- }
- for (Slice slice : collState.getSlices()) {
- if (slice.getLeader() == null) {
- hasLeaders = false;
- }
- }
- if (allActive && hasLeaders) {
- // check the number of active replicas
- assertEquals("total number of replicas", REPLICATION, replicas.size());
- recovered = true;
- break;
- } else {
- log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders);
- Thread.sleep(1000);
- }
- }
- assertTrue("replica never fully recovered", recovered);
-
- assertEquals(
- 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
-
- moveReplica = createMoveReplicaRequest(coll, replica, targetNode, shardId);
- moveReplica.setInPlaceMove(inPlaceMove);
- moveReplica.process(cloudClient);
- checkNumOfCores(cloudClient, replica.getNodeName(), coll, sourceNumCores);
- // wait for recovery
- recovered = false;
- for (int i = 0; i < 300; i++) {
- DocCollection collState = getCollectionState(coll);
- log.debug("###### {}", collState);
- Collection<Replica> replicas = collState.getSlice(shardId).getReplicas();
- boolean allActive = true;
- boolean hasLeaders = true;
- if (replicas != null && !replicas.isEmpty()) {
- for (Replica r : replicas) {
- if (!r.getNodeName().equals(replica.getNodeName())) {
- continue;
- }
- if (!r.isActive(Collections.singleton(replica.getNodeName()))) {
- log.info("Not active yet: {}", r);
- allActive = false;
- }
- }
- } else {
- allActive = false;
- }
- for (Slice slice : collState.getSlices()) {
- if (slice.getLeader() == null) {
- hasLeaders = false;
- }
- }
- if (allActive && hasLeaders) {
- assertEquals("total number of replicas", REPLICATION, replicas.size());
- recovered = true;
- break;
- } else {
- Thread.sleep(1000);
- }
- }
- assertTrue("replica never fully recovered", recovered);
-
- assertEquals(
- 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
- }
-
- @Test
- public void testFailedMove() throws Exception {
- String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove;
- int REPLICATION = 2;
-
- CloudSolrClient cloudClient = cluster.getSolrClient();
-
- // random create tlog or pull type replicas with nrt
- boolean isTlog = random().nextBoolean();
- CollectionAdminRequest.Create create =
- CollectionAdminRequest.createCollection(
- coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0);
- cloudClient.request(create);
-
- addDocs(coll, 100);
-
- NamedList<Object> overSeerStatus =
- cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- String overseerLeader = (String) overSeerStatus.get("leader");
-
- // don't kill overseer in this test
- Replica replica;
- int count = 10;
- do {
- replica = getRandomReplica(coll, cloudClient);
- } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0);
- assertNotNull("could not find non-overseer replica???", replica);
- Set<String> liveNodes = cloudClient.getClusterState().getLiveNodes();
- ArrayList<String> l = new ArrayList<>(liveNodes);
- Collections.shuffle(l, random());
- String targetNode = null;
- for (String node : liveNodes) {
- if (!replica.getNodeName().equals(node)
- && (isCollectionApiDistributed || !overseerLeader.equals(node))) {
- targetNode = node;
- break;
- }
- }
- assertNotNull(targetNode);
- CollectionAdminRequest.MoveReplica moveReplica =
- createMoveReplicaRequest(coll, replica, targetNode);
- moveReplica.setInPlaceMove(inPlaceMove);
- // start moving
- String asyncId = IdUtils.randomId();
- moveReplica.processAsync(asyncId, cloudClient);
- // shut down target node
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) {
- JettySolrRunner j = cluster.stopJettySolrRunner(i);
- cluster.waitForJettyToStop(j);
- break;
- }
- }
- CollectionAdminRequest.RequestStatus requestStatus =
- CollectionAdminRequest.requestStatus(asyncId);
- // wait for async request success
- boolean success = true;
- for (int i = 0; i < 200; i++) {
- CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
- assertNotSame(
- rsp.getRequestStatus().toString(), rsp.getRequestStatus(), RequestStatusState.COMPLETED);
- if (rsp.getRequestStatus() == RequestStatusState.FAILED) {
- success = false;
- break;
- }
- Thread.sleep(500);
- }
- assertFalse(success);
-
- if (log.isInfoEnabled()) {
- log.info(
- "--- current collection state: {}", cloudClient.getClusterState().getCollection(coll));
- }
- assertEquals(
- 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
- }
-
- private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(
- String coll, Replica replica, String targetNode, String shardId) {
- return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName());
- }
-
- private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(
- String coll, Replica replica, String targetNode) {
- return new CollectionAdminRequest.MoveReplica(coll, replica.getName(), targetNode);
- }
-
- private Replica getRandomReplica(String coll, CloudSolrClient cloudClient) throws IOException {
- List<Replica> replicas = cloudClient.getClusterState().getCollection(coll).getReplicas();
- Collections.shuffle(replicas, random());
- return replicas.get(0);
- }
-
- private void checkNumOfCores(
- CloudSolrClient cloudClient, String nodeName, String collectionName, int expectedCores)
- throws IOException, SolrServerException {
- assertEquals(
- nodeName + " does not have expected number of cores",
- expectedCores,
- getNumOfCores(cloudClient, nodeName, collectionName));
- }
-
- private int getNumOfCores(CloudSolrClient cloudClient, String nodeName, String collectionName)
- throws IOException, SolrServerException {
- return getNumOfCores(cloudClient, nodeName, collectionName, null);
- }
-
- private int getNumOfCores(
- CloudSolrClient cloudClient, String nodeName, String collectionName, String replicaType)
- throws IOException, SolrServerException {
- try (SolrClient coreclient =
- getHttpSolrClient(ZkStateReader.from(cloudClient).getBaseUrlForNodeName(nodeName))) {
- CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient);
- if (status.getCoreStatus().size() == 0) {
- return 0;
- }
- if (collectionName == null && replicaType == null) {
- return status.getCoreStatus().size();
- }
- // filter size by collection name
- int size = 0;
- for (Map.Entry<String, CoreStatusResponse.SingleCoreData> coreStatusEntry :
- status.getCoreStatus().entrySet()) {
- if (collectionName != null) {
- String coll = coreStatusEntry.getValue().cloud.collection;
- if (!collectionName.equals(coll)) {
- continue;
- }
- }
- if (replicaType != null) {
- String type = coreStatusEntry.getValue().cloud.replicaType;
- if (!replicaType.equals(type)) {
- continue;
- }
- }
- size++;
- }
- return size;
- }
- }
-
- protected void addDocs(String collection, int numDocs) throws Exception {
- SolrClient solrClient = cluster.getSolrClient();
- for (int docId = 1; docId <= numDocs; docId++) {
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField("id", docId);
- solrClient.add(collection, doc);
- }
- solrClient.commit(collection);
- Thread.sleep(5000);
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java
deleted file mode 100644
index 545a9486c41..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.apache.CloudLegacySolrClient;
-import org.apache.solr.client.solrj.apache.HttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.SolrQuery;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class AbstractRecoveryZkTestBase extends SolrCloudTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- cluster = configureCluster(2).addConfig("conf", configset("cloud-minimal")).configure();
- }
-
- private final List<StoppableIndexingThread> threads = new ArrayList<>();
-
- @After
- public void stopThreads() throws InterruptedException {
- for (StoppableIndexingThread t : threads) {
- t.safeStop();
- }
- for (StoppableIndexingThread t : threads) {
- t.join();
- }
- threads.clear();
- }
-
- @Test
- public void test() throws Exception {
-
- final String collection = "recoverytest";
-
- CollectionAdminRequest.createCollection(collection, "conf", 1, 2)
- .process(cluster.getSolrClient());
- waitForState(
- "Expected a collection with one shard and two replicas", collection, clusterShape(1, 2));
-
- // start a couple indexing threads
-
- int[] maxDocList = new int[] {300, 700, 1200, 1350, 3000};
- int[] maxDocNightlyList = new int[] {3000, 7000, 12000, 30000, 45000, 60000};
-
- int maxDoc;
- if (!TEST_NIGHTLY) {
- maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];
- } else {
- maxDoc = maxDocNightlyList[random().nextInt(maxDocList.length - 1)];
- }
- log.info("Indexing {} documents", maxDoc);
-
- try (SolrClient solrClient =
- cluster.basicSolrClientBuilder().withDefaultCollection(collection).build(); ) {
- final StoppableIndexingThread indexThread =
- new StoppableIndexingThread(null, solrClient, "1", true, maxDoc, 1, true);
- threads.add(indexThread);
- indexThread.start();
-
- final StoppableIndexingThread indexThread2 =
- new StoppableIndexingThread(null, solrClient, "2", true, maxDoc, 1, true);
- threads.add(indexThread2);
- indexThread2.start();
-
- // give some time to index...
- int[] waitTimes = new int[] {200, 2000, 3000};
- Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
-
- // bring shard replica down
- DocCollection state = getCollectionState(collection);
- Replica leader = state.getLeader("shard1");
- Replica replica = getRandomReplica(state.getSlice("shard1"), (r) -> !leader.equals(r));
-
- JettySolrRunner jetty = cluster.getReplicaJetty(replica);
- jetty.stop();
-
- // wait a moment - lets allow some docs to be indexed so replication time is non 0
- Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
-
- // bring shard replica up
- jetty.start();
-
- // make sure replication can start
- Thread.sleep(3000);
-
- // stop indexing threads
- indexThread.safeStop();
- indexThread2.safeStop();
-
- indexThread.join();
- indexThread2.join();
-
- new UpdateRequest().commit(solrClient, collection);
-
- cluster
- .getZkStateReader()
- .waitForState(collection, 120, TimeUnit.SECONDS, clusterShape(1, 2));
-
- // test that leader and replica have same doc count
- state = getCollectionState(collection);
- assertShardConsistency(state.getSlice("shard1"), true);
- }
- }
-
- private void assertShardConsistency(Slice shard, boolean expectDocs) throws Exception {
- List<Replica> replicas = shard.getReplicas(r -> r.getState() == Replica.State.ACTIVE);
- long[] numCounts = new long[replicas.size()];
- int i = 0;
- for (Replica replica : replicas) {
- try (var client =
- new HttpSolrClient.Builder(replica.getBaseUrl())
- .withDefaultCollection(replica.getCoreName())
- .withHttpClient(((CloudLegacySolrClient) cluster.getSolrClient()).getHttpClient())
- .build()) {
- numCounts[i] =
- client.query(new SolrQuery("*:*").add("distrib", "false")).getResults().getNumFound();
- i++;
- }
- }
- for (int j = 1; j < replicas.size(); j++) {
- if (numCounts[j] != numCounts[j - 1])
- fail("Mismatch in counts between replicas"); // TODO improve this!
- if (numCounts[j] == 0 && expectDocs)
- fail("Expected docs on shard " + shard.getName() + " but found none");
- }
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java
deleted file mode 100644
index 38733b30fbe..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public abstract class AbstractRestartWhileUpdatingTestBase extends AbstractFullDistribZkTestBase {
-
- // private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain";
- private List<StoppableIndexingThread> threads;
-
- private volatile boolean stopExpire = false;
-
- public AbstractRestartWhileUpdatingTestBase() throws Exception {
- super();
- sliceCount = 1;
- fixShardCount(3);
- schemaString = "schema15.xml"; // we need a string id
- useFactory("solr.StandardDirectoryFactory");
- }
-
- public static String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"};
- public static RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate};
-
- @Override
- protected String[] getFieldNames() {
- return fieldNames;
- }
-
- @Override
- protected RandVal[] getRandValues() {
- return randVals;
- }
-
- @BeforeClass
- public static void beforeRestartWhileUpdatingTest() {
- System.setProperty("leaderVoteWait", "300000");
- System.setProperty("solr.autoCommit.maxTime", "30000");
- System.setProperty("solr.autoSoftCommit.maxTime", "3000");
- // SOLR-13212 // TestInjection.nonGracefullClose = "true:60";
- // SOLR-13189 // TestInjection.failReplicaRequests = "true:03";
- }
-
- @AfterClass
- public static void afterRestartWhileUpdatingTest() {
- System.clearProperty("leaderVoteWait");
- System.clearProperty("solr.autoCommit.maxTime");
- System.clearProperty("solr.autoSoftCommit.maxTime");
- }
-
- @Test
- public void test() throws Exception {
- handle.clear();
- handle.put("timestamp", SKIPVAL);
-
- // start a couple indexing threads
-
- int[] maxDocList = new int[] {5000, 10000};
-
- int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];
-
- int numThreads = random().nextInt(4) + 1;
-
- threads = new ArrayList<>(numThreads);
-
- Thread expireThread =
- new Thread("expireThread") {
- @Override
- public void run() {
- while (!stopExpire) {
- try {
- Thread.sleep(random().nextInt(15000));
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
-
- // try {
- // chaosMonkey.expireRandomSession();
- // } catch (KeeperException e) {
- // throw new RuntimeException(e);
- // } catch (InterruptedException e) {
- // throw new RuntimeException(e);
- // }
- }
- }
- };
-
- // Currently unused
- // expireThread.start();
-
- StoppableIndexingThread indexThread;
- for (int i = 0; i < numThreads; i++) {
- indexThread =
- new StoppableIndexingThread(
- controlClient, cloudClient, Integer.toString(i), true, maxDoc, 1, true);
- threads.add(indexThread);
- indexThread.start();
- }
-
- Thread.sleep(2000);
-
- int restartTimes = 1; // random().nextInt(4) + 1;;
- for (int i = 0; i < restartTimes; i++) {
- Thread.sleep(random().nextInt(30000));
- stopAndStartAllReplicas();
- Thread.sleep(random().nextInt(30000));
- }
-
- Thread.sleep(2000);
-
- // stop indexing threads
- for (StoppableIndexingThread thread : threads) {
- thread.safeStop();
- }
- stopExpire = true;
- expireThread.join();
-
- Thread.sleep(1000);
-
- waitForThingsToLevelOut(320, TimeUnit.SECONDS);
-
- Thread.sleep(2000);
-
- waitForThingsToLevelOut(30, TimeUnit.SECONDS);
-
- Thread.sleep(5000);
-
- waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true);
-
- for (StoppableIndexingThread thread : threads) {
- thread.join();
- }
-
- checkShardConsistency(false, false);
- }
-
- public void stopAndStartAllReplicas() throws Exception, InterruptedException {
- chaosMonkey.stopAll(random().nextInt(1));
-
- if (random().nextBoolean()) {
- for (StoppableIndexingThread thread : threads) {
- thread.safeStop();
- }
- }
- Thread.sleep(1000);
-
- chaosMonkey.startAll();
- }
-
- @Override
- protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException {
- cloudClient.add(doc);
- }
-
- @Override
- public void distribTearDown() throws Exception {
- // make sure threads have been stopped...
- if (threads != null) {
- for (StoppableIndexingThread thread : threads) {
- thread.safeStop();
- thread.safeStop();
- }
- }
-
- super.distribTearDown();
- }
-
- // skip the randoms - they can deadlock...
- @Override
- protected void indexr(Object... fields) throws Exception {
- SolrInputDocument doc = new SolrInputDocument();
- addFields(doc, fields);
- addFields(doc, "rnd_b", true);
- indexDoc(doc);
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java
deleted file mode 100644
index c0c41f2eb22..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.apache.HttpSolrClient;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
-import org.apache.solr.client.solrj.request.SolrQuery;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CollectionParams.CollectionAction;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.util.LogLevel;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** Test sync phase that occurs when Leader goes down and a new Leader is elected. */
-@LogLevel("org.apache.solr.update.processor.DistributedZkUpdateProcessor=WARN")
-public abstract class AbstractSyncSliceTestBase extends AbstractFullDistribZkTestBase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private boolean success = false;
-
- @Override
- public void distribTearDown() throws Exception {
- if (!success) {
- printLayoutOnTearDown = true;
- }
- super.distribTearDown();
- }
-
- public AbstractSyncSliceTestBase() {
- super();
- sliceCount = 1;
- fixShardCount(TEST_NIGHTLY ? 7 : 4);
- }
-
- @Test
- public void test() throws Exception {
-
- handle.clear();
- handle.put("timestamp", SKIPVAL);
-
- waitForThingsToLevelOut(30, TimeUnit.SECONDS);
-
- del("*:*");
- List<CloudJettyRunner> skipServers = new ArrayList<>();
- int docId = 0;
- indexDoc(
- skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");
-
- indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "old haven was blue.");
-
- skipServers.add(shardToJetty.get("shard1").get(1));
-
- indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "but the song was fancy.");
-
- skipServers.add(shardToJetty.get("shard1").get(2));
-
- indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "under the moon and over the lake");
-
- commit();
-
- waitForRecoveriesToFinish(false);
-
- // shard should be inconsistent
- String shardFailMessage = checkShardConsistency("shard1", true, false);
- assertNotNull(shardFailMessage);
-
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", CollectionAction.SYNCSHARD.toString());
- params.set("collection", "collection1");
- params.set("shard", "shard1");
- final var request =
- new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/collections", params);
-
- String baseUrl = shardToJetty.get(SHARD1).get(2).jetty.getBaseUrl().toString();
-
- // we only set the connect timeout, not so timeout
- try (SolrClient baseClient =
- new HttpSolrClient.Builder(baseUrl)
- .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)
- .build()) {
- baseClient.request(request);
- }
-
- waitForThingsToLevelOut(15, TimeUnit.SECONDS);
-
- checkShardConsistency(false, true);
-
- long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
- assertEquals(4, cloudClientDocs);
-
- // kill the leader - new leader could have all the docs or be missing one
- CloudJettyRunner leaderJetty = shardToLeaderJetty.get("shard1");
-
- skipServers = getRandomOtherJetty(leaderJetty, null); // but not the leader
-
- // this doc won't be on one node
- indexDoc(
- skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");
- commit();
-
- Set<CloudJettyRunner> jetties = new HashSet<>();
- jetties.addAll(shardToJetty.get("shard1"));
- jetties.remove(leaderJetty);
- assertEquals(getShardCount() - 1, jetties.size());
-
- leaderJetty.jetty.stop();
-
- Thread.sleep(3000);
-
- waitForNoShardInconsistency();
-
- Thread.sleep(1000);
-
- checkShardConsistency(false, true);
-
- cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
- assertEquals(5, cloudClientDocs);
-
- CloudJettyRunner deadJetty = leaderJetty;
-
- // let's get the latest leader
- while (deadJetty.equals(leaderJetty)) {
- updateMappingsFromZk(this.jettys, this.clients);
- leaderJetty = shardToLeaderJetty.get("shard1");
- }
-
- // bring back dead node
- deadJetty.jetty.start(); // he is not the leader anymore
-
- waitTillAllNodesActive();
-
- skipServers = getRandomOtherJetty(leaderJetty, deadJetty);
- skipServers.addAll(getRandomOtherJetty(leaderJetty, deadJetty));
- // skip list should be
-
- // System.out.println("leader:" + leaderJetty.url);
- // System.out.println("dead:" + deadJetty.url);
- // System.out.println("skip list:" + skipServers);
-
- // we are skipping 2 nodes
- assertEquals(2, skipServers.size());
-
- // more docs than can peer sync
- for (int i = 0; i < 300; i++) {
- indexDoc(
- skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country.");
- }
-
- commit();
- waitForRecoveriesToFinish(false);
-
- // shard should be inconsistent
- shardFailMessage = waitTillInconsistent();
- assertNotNull(
- "Test Setup Failure: shard1 should have just been set up to be inconsistent - but it's still consistent. Leader:"
- + leaderJetty.url
- + " Dead Guy:"
- + deadJetty.url
- + "skip list:"
- + skipServers,
- shardFailMessage);
-
- // good place to test compareResults
- boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient);
- assertTrue("A test that compareResults is working correctly failed", shouldFail);
-
- jetties = new HashSet<>();
- jetties.addAll(shardToJetty.get("shard1"));
- jetties.remove(leaderJetty);
- assertEquals(getShardCount() - 1, jetties.size());
-
- // kill the current leader
- leaderJetty.jetty.stop();
-
- waitForNoShardInconsistency();
-
- checkShardConsistency(true, true);
-
- success = true;
- }
-
- private void waitTillAllNodesActive() throws InterruptedException, TimeoutException {
- ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
-
- zkStateReader.waitForState(
- "collection1",
- 3,
- TimeUnit.MINUTES,
- (n, c) -> {
- Collection<Replica> replicas = c.getSlice("shard1").getReplicas();
- Set<String> nodes =
- replicas.stream().map(Replica::getNodeName).collect(Collectors.toSet());
- return replicas.stream().map(Replica::getState).allMatch(Replica.State.ACTIVE::equals)
- && n.containsAll(nodes);
- });
- }
-
- private String waitTillInconsistent() throws Exception, InterruptedException {
- String shardFailMessage = null;
-
- shardFailMessage = pollConsistency(shardFailMessage, 0);
- shardFailMessage = pollConsistency(shardFailMessage, 3000);
- shardFailMessage = pollConsistency(shardFailMessage, 5000);
- shardFailMessage = pollConsistency(shardFailMessage, 15000);
-
- return shardFailMessage;
- }
-
- private String pollConsistency(String shardFailMessage, int sleep)
- throws InterruptedException, Exception {
- try {
- commit();
- } catch (Throwable t) {
- log.error("commit error", t);
- }
- if (shardFailMessage == null) {
- // try again
- Thread.sleep(sleep);
- shardFailMessage = checkShardConsistency("shard1", true, false);
- }
- return shardFailMessage;
- }
-
- private List<CloudJettyRunner> getRandomOtherJetty(
- CloudJettyRunner leader, CloudJettyRunner down) {
- List<CloudJettyRunner> skipServers = new ArrayList<>();
- List<CloudJettyRunner> candidates = new ArrayList<>();
- candidates.addAll(shardToJetty.get("shard1"));
-
- if (leader != null) {
- candidates.remove(leader);
- }
-
- if (down != null) {
- candidates.remove(down);
- }
-
- CloudJettyRunner cjetty = candidates.get(random().nextInt(candidates.size()));
- skipServers.add(cjetty);
- return skipServers;
- }
-
- protected void indexDoc(List<CloudJettyRunner> skipServers, Object... fields)
- throws IOException, SolrServerException {
- SolrInputDocument doc = new SolrInputDocument();
-
- addFields(doc, fields);
- addFields(doc, "rnd_b", true);
-
- controlClient.add(doc);
-
- UpdateRequest ureq = new UpdateRequest();
- ureq.add(doc);
- for (CloudJettyRunner skip : skipServers) {
- ureq.getParams().add("test.distrib.skip.servers", skip.url + "/");
- }
- ureq.process(cloudClient);
- }
-
- // skip the randoms - they can deadlock...
- @Override
- protected void indexr(Object... fields) throws Exception {
- SolrInputDocument doc = new SolrInputDocument();
- addFields(doc, fields);
- addFields(doc, "rnd_b", true);
- indexDoc(doc);
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java
deleted file mode 100644
index ee73ea8e817..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.util.TestInjection;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public abstract class AbstractTlogReplayBufferedWhileIndexingTestBase
- extends AbstractFullDistribZkTestBase {
-
- private List<StoppableIndexingThread> threads;
-
- public AbstractTlogReplayBufferedWhileIndexingTestBase() throws Exception {
- super();
- sliceCount = 1;
- fixShardCount(2);
- schemaString = "schema15.xml"; // we need a string id
- }
-
- @BeforeClass
- public static void beforeRestartWhileUpdatingTest() throws Exception {
- System.setProperty("leaderVoteWait", "300000");
- System.setProperty("solr.autoCommit.maxTime", "10000");
- System.setProperty("solr.autoSoftCommit.maxTime", "3000");
- TestInjection.updateLogReplayRandomPause = "true:10";
- TestInjection.updateRandomPause = "true:10";
- useFactory("solr.StandardDirectoryFactory");
- }
-
- @AfterClass
- public static void afterRestartWhileUpdatingTest() {
- System.clearProperty("leaderVoteWait");
- System.clearProperty("solr.autoCommit.maxTime");
- System.clearProperty("solr.autoSoftCommit.maxTime");
- }
-
- @Test
- public void test() throws Exception {
- handle.clear();
- handle.put("timestamp", SKIPVAL);
-
- waitForRecoveriesToFinish(false);
-
- int numThreads = 3;
-
- threads = new ArrayList<>(numThreads);
-
- ArrayList<JettySolrRunner> allJetty = new ArrayList<>();
- allJetty.addAll(jettys);
- allJetty.remove(shardToLeaderJetty.get("shard1").jetty);
- assert allJetty.size() == 1 : allJetty.size();
- allJetty.get(0).stop();
-
- StoppableIndexingThread indexThread;
- for (int i = 0; i < numThreads; i++) {
- boolean pauseBetweenUpdates = random().nextBoolean();
- int batchSize = random().nextInt(4) + 1;
- indexThread =
- new StoppableIndexingThread(
- controlClient,
- cloudClient,
- Integer.toString(i),
- true,
- 900,
- batchSize,
- pauseBetweenUpdates);
- threads.add(indexThread);
- indexThread.start();
- }
-
- Thread.sleep(2000);
-
- allJetty.get(0).start();
-
- Thread.sleep(45000);
-
- waitForThingsToLevelOut(); // we can insert random update delays, so this can take a while,
- // especially when beasting this test
-
- Thread.sleep(2000);
-
- waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true);
-
- for (StoppableIndexingThread thread : threads) {
- thread.safeStop();
- }
-
- waitForThingsToLevelOut(30, TimeUnit.SECONDS);
-
- checkShardConsistency(false, false);
- }
-
- @Override
- protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException {
- cloudClient.add(doc);
- }
-
- @Override
- public void distribTearDown() throws Exception {
- // make sure threads have been stopped...
- if (threads != null) {
- for (StoppableIndexingThread thread : threads) {
- thread.safeStop();
- }
-
- for (StoppableIndexingThread thread : threads) {
- thread.join();
- }
- }
-
- super.distribTearDown();
- }
-
- // skip the randoms - they can deadlock...
- @Override
- protected void indexr(Object... fields) throws Exception {
- SolrInputDocument doc = new SolrInputDocument();
- addFields(doc, fields);
- addFields(doc, "rnd_b", true);
- indexDoc(doc);
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java
deleted file mode 100644
index 6477af7aac6..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java
+++ /dev/null
@@ -1,463 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Path;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.apache.HttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;
-import org.apache.solr.client.solrj.request.SolrQuery;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.ExecutorUtil;
-import org.apache.solr.common.util.SolrNamedThreadFactory;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrPaths;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.util.TestInjection;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This test simply does a bunch of basic things in solrcloud mode and asserts things work as
- * expected.
- */
-public abstract class AbstractUnloadDistributedZkTestBase extends AbstractFullDistribZkTestBase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public AbstractUnloadDistributedZkTestBase() {
- super();
- fixShardCount(4); // needs at least 4 servers
- }
-
- @Override
- protected String getSolrXml() {
- return "solr.xml";
- }
-
- @Test
- public void test() throws Exception {
- jettys.forEach(
- j -> {
- Set<Path> allowPath = j.getCoreContainer().getAllowPaths();
- allowPath.clear();
- allowPath.add(SolrPaths.ALL_PATH); // Allow non-standard core instance path
- });
- log.info("###Starting testCoreUnloadAndLeaders");
- testCoreUnloadAndLeaders(); // long
- log.info("###Starting testUnloadOfCores");
- testUnloadLotsOfCores(); // long
- log.info("###Starting testUnloadShardAndCollection");
- testUnloadShardAndCollection();
- }
-
- /**
- * @param url a Solr node base URL. Should <em>not</em> contain a core or collection name.
- */
- private SolrClient newSolrClient(String url) {
- return new HttpSolrClient.Builder(url)
- .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)
- .withSocketTimeout(30000, TimeUnit.MILLISECONDS)
- .build();
- }
-
- private void checkCoreNamePresenceAndSliceCount(
- String collectionName, String coreName, boolean shouldBePresent, int expectedSliceCount)
- throws Exception {
- ZkStateReader reader = ZkStateReader.from(cloudClient);
- try {
- reader.waitForState(
- collectionName,
- 45,
- TimeUnit.SECONDS,
- c -> {
- final Collection<Slice> slices = (c != null) ? c.getSlices() : Collections.emptyList();
- if (expectedSliceCount == slices.size()) {
- for (Slice slice : slices) {
- for (Replica replica : slice.getReplicas()) {
- if (coreName.equals(replica.get("core"))) {
- return shouldBePresent;
- }
- }
- }
- return !shouldBePresent;
- } else {
- return false;
- }
- });
- } catch (TimeoutException e) {
- printLayout();
- fail(
- "checkCoreNamePresenceAndSliceCount failed:"
- + " collection="
- + collectionName
- + " CoreName="
- + coreName
- + " shouldBePresent="
- + shouldBePresent);
- }
- }
-
- private void testUnloadShardAndCollection() throws Exception {
- final int numShards = 2;
-
- final String collection = "test_unload_shard_and_collection";
-
- final String coreName1 = collection + "_1";
- final String coreName2 = collection + "_2";
-
- assertEquals(
- 0,
- CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1)
- .setCreateNodeSet("")
- .process(cloudClient)
- .getStatus());
- assertTrue(
- CollectionAdminRequest.addReplicaToShard(collection, "shard1")
- .setCoreName(coreName1)
- .setNode(jettys.get(0).getNodeName())
- .process(cloudClient)
- .isSuccess());
-
- assertTrue(
- CollectionAdminRequest.addReplicaToShard(collection, "shard2")
- .setCoreName(coreName2)
- .setNode(jettys.get(0).getNodeName())
- .process(cloudClient)
- .isSuccess());
-
- // does not mean they are active and up yet :*
- waitForRecoveriesToFinish(collection, false);
-
- final boolean unloadInOrder = random().nextBoolean();
- final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2);
- final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1);
-
- try (SolrClient adminClient = getHttpSolrClient(buildUrl(jettys.get(0).getLocalPort()))) {
- // now unload one of the two
- Unload unloadCmd = new Unload(false);
- unloadCmd.setCoreName(unloadCmdCoreName1);
- adminClient.request(unloadCmd);
-
- // there should still be two shards (as of SOLR-5209)
- checkCoreNamePresenceAndSliceCount(
- collection,
- unloadCmdCoreName1,
- false /* shouldBePresent */,
- numShards /* expectedSliceCount */);
-
- // now unload one of the other
- unloadCmd = new Unload(false);
- unloadCmd.setCoreName(unloadCmdCoreName2);
- adminClient.request(unloadCmd);
- checkCoreNamePresenceAndSliceCount(
- collection,
- unloadCmdCoreName2,
- false /* shouldBePresent */,
- numShards /* expectedSliceCount */);
- }
-
- // printLayout();
- // the collection should still be present (as of SOLR-5209 replica removal does not cascade to
- // remove the slice and collection)
- getCommonCloudSolrClient();
- assertTrue(
- "No longer found collection " + collection,
- cloudClient.getClusterState().hasCollection(collection));
- }
-
- protected SolrCore getFirstCore(String collection, JettySolrRunner jetty) {
- SolrCore solrCore = null;
- for (SolrCore core : jetty.getCoreContainer().getCores()) {
- if (core.getName().startsWith(collection)) {
- solrCore = core;
- }
- }
- return solrCore;
- }
-
- /**
- * @throws Exception on any problem
- */
- private void testCoreUnloadAndLeaders() throws Exception {
- JettySolrRunner jetty1 = jettys.get(0);
-
- assertEquals(
- 0,
- CollectionAdminRequest.createCollection("unloadcollection", "conf1", 1, 1)
- .setCreateNodeSet(jetty1.getNodeName())
- .process(cloudClient)
- .getStatus());
- getCommonCloudSolrClient();
- ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
-
- zkStateReader.forceUpdateCollection("unloadcollection");
-
- int slices =
- zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size();
- assertEquals(1, slices);
- SolrCore solrCore = getFirstCore("unloadcollection", jetty1);
- String core1DataDir = solrCore.getDataDir();
-
- assertTrue(
- CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")
- .setCoreName("unloadcollection_shard1_replica2")
- .setNode(jettys.get(1).getNodeName())
- .process(cloudClient)
- .isSuccess());
- zkStateReader.forceUpdateCollection("unloadcollection");
- slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size();
- assertEquals(1, slices);
-
- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-
- Replica leader = getLeaderFromZk("unloadcollection", "shard1");
-
- Random random = random();
- if (random.nextBoolean()) {
- try (SolrClient collectionClient =
- getHttpSolrClient(leader.getBaseUrl(), leader.getCoreName())) {
- // lets try and use the solrj client to index and retrieve a couple
- // documents
- SolrInputDocument doc1 =
- getDoc(id, 6, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");
- SolrInputDocument doc2 =
- getDoc(id, 7, i1, -600, tlong, 600, t1, "humpty dumpy3 sat on a walls");
- SolrInputDocument doc3 =
- getDoc(id, 8, i1, -600, tlong, 600, t1, "humpty dumpy2 sat on a walled");
- collectionClient.add(doc1);
- collectionClient.add(doc2);
- collectionClient.add(doc3);
- collectionClient.commit();
- }
- }
-
- assertTrue(
- CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")
- .setCoreName("unloadcollection_shard1_replica3")
- .setNode(jettys.get(2).getNodeName())
- .process(cloudClient)
- .isSuccess());
-
- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-
- // so that we start with some versions when we reload...
- TestInjection.skipIndexWriterCommitOnClose = true;
-
- try (SolrClient addClient =
- new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString())
- .withDefaultCollection("unloadcollection_shard1_replica3")
- .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)
- .build()) {
-
- // add a few docs
- for (int x = 20; x < 100; x++) {
- SolrInputDocument doc1 =
- getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");
- addClient.add(doc1);
- }
- }
- // don't commit so they remain in the tran log
- // collectionClient.commit();
-
- // unload the leader
- try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) {
-
- Unload unloadCmd = new Unload(false);
- unloadCmd.setCoreName(leader.getCoreName());
- ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams();
-
- collectionClient.request(unloadCmd);
- }
- // Thread.currentThread().sleep(500);
- // printLayout();
-
- int tries = 50;
- while (leader
- .getCoreUrl()
- .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
- Thread.sleep(100);
- if (tries-- == 0) {
- fail("Leader never changed");
- }
- }
-
- // ensure there is a leader
- zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
-
- try (SolrClient addClient =
- new HttpSolrClient.Builder(jettys.get(1).getBaseUrl().toString())
- .withDefaultCollection("unloadcollection_shard1_replica2")
- .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)
- .withSocketTimeout(90000, TimeUnit.MILLISECONDS)
- .build()) {
-
- // add a few docs while the leader is down
- for (int x = 101; x < 200; x++) {
- SolrInputDocument doc1 =
- getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");
- addClient.add(doc1);
- }
- }
-
- assertTrue(
- CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")
- .setCoreName("unloadcollection_shard1_replica4")
- .setNode(jettys.get(3).getNodeName())
- .process(cloudClient)
- .isSuccess());
-
- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-
- // unload the leader again
- leader = getLeaderFromZk("unloadcollection", "shard1");
- try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) {
-
- Unload unloadCmd = new Unload(false);
- unloadCmd.setCoreName(leader.getCoreName());
- collectionClient.request(unloadCmd);
- }
- tries = 50;
- while (leader
- .getCoreUrl()
- .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
- Thread.sleep(100);
- if (tries-- == 0) {
- fail("Leader never changed");
- }
- }
-
- zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
-
- TestInjection.skipIndexWriterCommitOnClose = false; // set this back
- assertTrue(
- CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1")
- .setCoreName(leader.getCoreName())
- .setDataDir(core1DataDir)
- .setNode(leader.getNodeName())
- .process(cloudClient)
- .isSuccess());
-
- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-
- long found1, found3;
-
- try (SolrClient adminClient =
- newSolrClient((jettys.get(1).getBaseUrl() + "/unloadcollection_shard1_replica2"))) {
- adminClient.commit();
- SolrQuery q = new SolrQuery("*:*");
- q.set("distrib", false);
- found1 = adminClient.query(q).getResults().getNumFound();
- }
-
- try (SolrClient adminClient =
- new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString())
- .withDefaultCollection("unloadcollection_shard1_replica3")
- .withConnectionTimeout(15000, TimeUnit.MILLISECONDS)
- .withSocketTimeout(30000, TimeUnit.MILLISECONDS)
- .build()) {
- adminClient.commit();
- SolrQuery q = new SolrQuery("*:*");
- q.set("distrib", false);
- found3 = adminClient.query(q).getResults().getNumFound();
- }
-
- try (SolrClient adminClient =
- newSolrClient((jettys.get(3).getBaseUrl() + "/unloadcollection_shard1_replica4"))) {
- adminClient.commit();
- SolrQuery q = new SolrQuery("*:*");
- q.set("distrib", false);
- long found4 = adminClient.query(q).getResults().getNumFound();
-
- // all 3 shards should now have the same number of docs
- assertEquals(found1, found3);
- assertEquals(found3, found4);
- }
- }
-
- private void testUnloadLotsOfCores() throws Exception {
- JettySolrRunner jetty = jettys.get(0);
- int shards = TEST_NIGHTLY ? 2 : 1;
- try (final SolrClient adminClient = jetty.newClient(15000, 60000)) {
- int numReplicas = atLeast(3);
- ThreadPoolExecutor executor =
- new ExecutorUtil.MDCAwareThreadPoolExecutor(
- 0,
- Integer.MAX_VALUE,
- 5,
- TimeUnit.SECONDS,
- new SynchronousQueue<>(),
- new SolrNamedThreadFactory("testExecutor"));
- try {
- // create the cores
- AbstractBasicDistributedZkTestBase.createCollectionInOneInstance(
- adminClient, jetty.getNodeName(), executor, "multiunload", shards, numReplicas);
- } finally {
- ExecutorUtil.shutdownAndAwaitTermination(executor);
- }
-
- if (TEST_NIGHTLY == false) {
- // with nightly tests, we can try doing the unloads before the creates are done
- // it still works, but takes much longer since we end up waiting for a timeout
- waitForRecoveriesToFinish("multiunload", false);
- }
-
- executor =
- new ExecutorUtil.MDCAwareThreadPoolExecutor(
- 0,
- Integer.MAX_VALUE,
- 5,
- TimeUnit.SECONDS,
- new SynchronousQueue<>(),
- new SolrNamedThreadFactory("testExecutor"));
- try {
- for (int j = 0; j < numReplicas; j++) {
- final int freezeJ = j;
- executor.execute(
- () -> {
- Unload unloadCmd = new Unload(true);
- unloadCmd.setCoreName("multiunload" + freezeJ);
- try {
- adminClient.request(unloadCmd);
- } catch (SolrServerException | IOException e) {
- throw new RuntimeException(e);
- }
- });
- Thread.sleep(random().nextInt(50));
- }
- } finally {
- ExecutorUtil.shutdownAndAwaitTermination(executor);
- }
- }
- }
-}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
index 79dc6bde071..5dc62e94ed2 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
@@ -47,7 +47,6 @@ import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -173,7 +172,6 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
}
testBackupAndRestore(getCollectionName());
- testConfigBackupOnly("conf1", getCollectionName());
testInvalidPath(getCollectionName());
}
@@ -230,22 +228,6 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
}
}
- /**
- * This test validates the backup of collection configuration using {@linkplain
- * CollectionAdminParams#NO_INDEX_BACKUP_STRATEGY}.
- *
- * @param configName The config name for the collection to be backed up.
- * @param collectionName The name of the collection to be backed up.
- * @throws Exception in case of errors.
- */
- protected void testConfigBackupOnly(String configName, String collectionName) throws Exception {
- // This is deliberately no-op since we want to run this test only for one of the backup
- // repository
- // implementation (mainly to avoid redundant test execution). Currently HDFS backup repository
- // test
- // implements this.
- }
-
// This test verifies the system behavior when the backup location cluster property is configured
// with an invalid
// value for the specified repository (and the default backup location is not configured in