Apache Solr open-source search software
SOLR-18126 Benchmark, refactor: MiniClusterState -> SolrBenchState (#4196)
David Smileyaed1c3ef79ad
diff --git a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java b/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.javadeleted file mode 100755index 4c4946a3ae8..00000000000--- a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java+++ /dev/null@@ -1,587 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.bench;--import static org.apache.commons.io.file.PathUtils.deleteDirectory;-import static org.apache.solr.bench.BaseBenchState.log;--import com.codahale.metrics.Meter;-import java.io.IOException;-import java.io.OutputStream;-import java.io.PrintStream;-import java.lang.management.ManagementFactory;-import java.net.URL;-import java.nio.charset.StandardCharsets;-import java.nio.file.FileSystems;-import java.nio.file.Files;-import java.nio.file.Path;-import java.util.ArrayList;-import java.util.List;-import java.util.SplittableRandom;-import java.util.concurrent.ExecutorService;-import java.util.concurrent.Executors;-import java.util.concurrent.ScheduledExecutorService;-import java.util.concurrent.TimeUnit;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.jetty.HttpJettySolrClient;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.request.QueryRequest;-import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.client.solrj.request.UpdateRequest;-import org.apache.solr.cloud.MiniSolrCloudCluster;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.util.IOUtils;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.common.util.SolrNamedThreadFactory;-import org.apache.solr.common.util.SuppressForbidden;-import org.apache.solr.embedded.JettySolrRunner;-import org.apache.solr.util.SolrTestNonSecureRandomProvider;-import org.openjdk.jmh.annotations.Level;-import org.openjdk.jmh.annotations.Scope;-import org.openjdk.jmh.annotations.Setup;-import org.openjdk.jmh.annotations.State;-import org.openjdk.jmh.annotations.TearDown;-import org.openjdk.jmh.infra.BenchmarkParams;-import org.openjdk.jmh.infra.Control;--/** The base class for Solr JMH benchmarks that operate against a {@code MiniSolrCloudCluster}. */-public class MiniClusterState {-- /** The constant PROC_COUNT. */- public static final int PROC_COUNT =- ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors();-- /** The type Mini cluster bench state. */- @State(Scope.Benchmark)- public static class MiniClusterBenchState {-- /** The Metrics enabled. */- boolean metricsEnabled = true;-- /** The Nodes. */- public List<String> nodes;-- public String zkHost;-- /** The Cluster. */- MiniSolrCloudCluster cluster;-- /** The Client. */- public HttpJettySolrClient client;-- /** The Run cnt. */- int runCnt = 0;-- /** The Create collection and index. */- boolean createCollectionAndIndex = true;-- /** The Delete mini cluster. */- boolean deleteMiniCluster = true;-- /** Unless overridden we ensure SecureRandoms do not block. */- boolean doNotWeakenSecureRandom = Boolean.getBoolean("doNotWeakenSecureRandom");-- /** The Mini cluster base dir. */- Path miniClusterBaseDir;-- /** To Allow cluster reuse. */- boolean allowClusterReuse = false;-- /** The Is warmup. */- boolean isWarmup;-- private SplittableRandom random;- private String workDir;-- private boolean useHttp1 = Boolean.getBoolean("solr.http1");-- /**- * Tear down.- *- * @param benchmarkParams the benchmark params- * @throws Exception the exception- */- @TearDown(Level.Iteration)- public void tearDown(BenchmarkParams benchmarkParams) throws Exception {-- // dump Solr metrics- Path metricsResults =- Path.of(- workDir,- "metrics-results",- benchmarkParams.id(),- String.valueOf(runCnt++),- benchmarkParams.getBenchmark() + ".txt");- Files.createDirectories(metricsResults.getParent());-- cluster.dumpMetrics(metricsResults.getParent(), metricsResults.getFileName().toString());- }-- /**- * Check warm up.- *- * @param control the control- * @throws Exception the exception- */- @Setup(Level.Iteration)- public void checkWarmUp(Control control) throws Exception {- isWarmup = control.stopMeasurement;- }-- /**- * Shutdown mini cluster.- *- * @param benchmarkParams the benchmark params- * @throws Exception the exception- */- @TearDown(Level.Trial)- public void shutdownMiniCluster(BenchmarkParams benchmarkParams, BaseBenchState baseBenchState)- throws Exception {- BaseBenchState.dumpHeap(benchmarkParams);-- IOUtils.closeQuietly(client);- cluster.shutdown();- logClusterDirectorySize();- }-- private void logClusterDirectorySize() throws IOException {- log("");- Files.list(miniClusterBaseDir.toAbsolutePath())- .forEach(- (node) -> {- try {- long clusterSize =- Files.walk(node)- .filter(Files::isRegularFile)- .mapToLong(- file -> {- try {- return Files.size(file);- } catch (IOException e) {- throw new RuntimeException(e);- }- })- .sum();- log("mini cluster node size (bytes) " + node + " " + clusterSize);- } catch (IOException e) {- throw new RuntimeException(e);- }- });- }-- /**- * Do setup.- *- * @param benchmarkParams the benchmark params- * @param baseBenchState the base bench state- * @throws Exception the exception- */- @Setup(Level.Trial)- public void doSetup(BenchmarkParams benchmarkParams, BaseBenchState baseBenchState)- throws Exception {-- if (!doNotWeakenSecureRandom) {- // remove all blocking from all secure randoms- SolrTestNonSecureRandomProvider.injectProvider();- }-- workDir = System.getProperty("workBaseDir", "build/work");-- Path currentRelativePath = Path.of("");- String s = currentRelativePath.toAbsolutePath().toString();- log("current relative path is: " + s);- log("work path is: " + workDir);-- System.setProperty("doNotWaitForMergesOnIWClose", "true");-- System.setProperty("pkiHandlerPrivateKeyPath", "");- System.setProperty("pkiHandlerPublicKeyPath", "");-- System.setProperty("solr.configset.default.confdir", "../server/solr/configsets/_default");-- this.random = new SplittableRandom(BaseBenchState.getRandomSeed());-- // not currently usable, but would enable JettySolrRunner's ill-conceived jetty.testMode and- // allow using SSL-- // System.getProperty("jetty.testMode", "true");- // SolrCloudTestCase.sslConfig = SolrTestCaseJ4.buildSSLConfig();-- String baseDirSysProp = System.getProperty("miniClusterBaseDir");- if (baseDirSysProp != null) {- deleteMiniCluster = false;- miniClusterBaseDir = Path.of(baseDirSysProp);- if (Files.exists(miniClusterBaseDir)) {- createCollectionAndIndex = false;- allowClusterReuse = true;- }- } else {- miniClusterBaseDir = Path.of(workDir, "mini-cluster");- }-- System.setProperty("metricsEnabled", String.valueOf(metricsEnabled));- }-- /**- * Metrics enabled.- *- * @param metricsEnabled the metrics enabled- */- public void metricsEnabled(boolean metricsEnabled) {- this.metricsEnabled = metricsEnabled;- }-- /**- * Start mini cluster.- *- * @param nodeCount the node count- */- public void startMiniCluster(int nodeCount) {- log("starting mini cluster at base directory: " + miniClusterBaseDir.toAbsolutePath());-- if (!allowClusterReuse && Files.exists(miniClusterBaseDir)) {- log("mini cluster base directory exists, removing ...");- try {- deleteDirectory(miniClusterBaseDir);- } catch (IOException e) {- throw new RuntimeException(e);- }- createCollectionAndIndex = true;- } else if (Files.exists(miniClusterBaseDir)) {- createCollectionAndIndex = false;- deleteMiniCluster = false;- }-- try {- cluster =- new MiniSolrCloudCluster.Builder(nodeCount, miniClusterBaseDir)- .formatZkServer(false)- .addConfig("conf", getFile("src/resources/configs/cloud-minimal/conf"))- .configure();- } catch (Exception e) {- if (Files.exists(miniClusterBaseDir)) {- try {- deleteDirectory(miniClusterBaseDir);- } catch (IOException ex) {- e.addSuppressed(ex);- }- }- throw new RuntimeException(e);- }-- nodes = new ArrayList<>(nodeCount);- List<JettySolrRunner> jetties = cluster.getJettySolrRunners();- for (JettySolrRunner runner : jetties) {- nodes.add(runner.getBaseUrl().toString());- }- zkHost = cluster.getZkServer().getZkAddress();-- client = new HttpJettySolrClient.Builder(nodes.get(0)).useHttp1_1(useHttp1).build();-- log("done starting mini cluster");- log("");- }-- /**- * Gets random.- *- * @return the random- */- public SplittableRandom getRandom() {- return random;- }-- /**- * Create collection.- *- * @param collection the collection- * @param numShards the num shards- * @param numReplicas the num replicas- * @throws Exception the exception- */- public void createCollection(String collection, int numShards, int numReplicas)- throws Exception {- if (createCollectionAndIndex) {- try {-- CollectionAdminRequest.Create request =- CollectionAdminRequest.createCollection(collection, "conf", numShards, numReplicas);- client.requestWithBaseUrl(- nodes.get(random.nextInt(cluster.getJettySolrRunners().size())), request, null);-- cluster.waitForActiveCollection(- collection, 15, TimeUnit.SECONDS, numShards, numShards * numReplicas);- } catch (Exception e) {- if (Files.exists(miniClusterBaseDir)) {- deleteDirectory(miniClusterBaseDir);- }- throw e;- }- }- }-- /** Setting useHttp1 to true will make the {@link #client} use http1 */- public void setUseHttp1(boolean useHttp1) {- if (client != null) {- throw new IllegalStateException(- "You can only change this setting before starting the Mini Cluster");- }- this.useHttp1 = useHttp1;- }-- @SuppressForbidden(reason = "This module does not need to deal with logging context")- public void index(String collection, Docs docs, int docCount) throws Exception {- index(collection, docs, docCount, true);- }-- /**- * Index.- *- * @param collection the collection- * @param docs the docs- * @param docCount the doc count- * @throws Exception the exception- */- public void index(String collection, Docs docs, int docCount, boolean parallel)- throws Exception {- if (createCollectionAndIndex) {- log("indexing data for benchmark...");- if (parallel) {- indexParallel(collection, docs, docCount);- } else {- indexBatch(collection, docs, docCount, 10000);- }- log("done indexing data for benchmark");-- log("committing data ...");- UpdateRequest commitRequest = new UpdateRequest();- final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size()));- commitRequest.setAction(UpdateRequest.ACTION.COMMIT, false, true);- client.requestWithBaseUrl(url, commitRequest, collection);- log("done committing data");- } else {- cluster.waitForActiveCollection(collection, 15, TimeUnit.SECONDS);- }-- QueryRequest queryRequest = new QueryRequest(new SolrQuery("q", "*:*", "rows", "1"));- final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size()));- NamedList<Object> result = client.requestWithBaseUrl(url, queryRequest, collection);-- log("sanity check of single row query result: " + result);- log("");-- log("Dump Core Info");- dumpCoreInfo();- }-- @SuppressForbidden(reason = "This module does not need to deal with logging context")- private void indexParallel(String collection, Docs docs, int docCount)- throws InterruptedException {- Meter meter = new Meter();- ExecutorService executorService =- Executors.newFixedThreadPool(- Runtime.getRuntime().availableProcessors(),- new SolrNamedThreadFactory("SolrJMH Indexer"));- ScheduledExecutorService scheduledExecutor =- Executors.newSingleThreadScheduledExecutor(- new SolrNamedThreadFactory("SolrJMH Indexer Progress"));- scheduledExecutor.scheduleAtFixedRate(- () -> {- if (meter.getCount() == docCount) {- scheduledExecutor.shutdown();- } else {- log(meter.getCount() + " docs at " + meter.getMeanRate() + " doc/s");- }- },- 10,- 10,- TimeUnit.SECONDS);- for (int i = 0; i < docCount; i++) {- executorService.execute(- new Runnable() {- final SplittableRandom threadRandom = random.split();-- @Override- public void run() {- UpdateRequest updateRequest = new UpdateRequest();- final var url =- nodes.get(threadRandom.nextInt(cluster.getJettySolrRunners().size()));- SolrInputDocument doc = docs.inputDocument();- // log("add doc " + doc);- updateRequest.add(doc);- meter.mark();-- try {- client.requestWithBaseUrl(url, updateRequest, collection);- } catch (Exception e) {- throw new RuntimeException(e);- }- }- });- }-- log("done adding docs, waiting for executor to terminate...");-- executorService.shutdown();- boolean result = false;- while (!result) {- result = executorService.awaitTermination(600, TimeUnit.MINUTES);- }-- scheduledExecutor.shutdown();- }-- private void indexBatch(String collection, Docs docs, int docCount, int batchSize)- throws SolrServerException, IOException {- Meter meter = new Meter();- List<SolrInputDocument> batch = new ArrayList<>(batchSize);- for (int i = 1; i <= docCount; i++) {- batch.add(docs.inputDocument());- if (i % batchSize == 0) {- UpdateRequest updateRequest = new UpdateRequest();- updateRequest.add(batch);- client.requestWithBaseUrl(nodes.get(0), updateRequest, collection);- meter.mark(batch.size());- batch.clear();- log(meter.getCount() + " docs at " + (long) meter.getMeanRate() + " doc/s");- }- }- if (!batch.isEmpty()) {- UpdateRequest updateRequest = new UpdateRequest();- updateRequest.add(batch);- client.requestWithBaseUrl(nodes.get(0), updateRequest, collection);- meter.mark(batch.size());- batch = null;- }- log(meter.getCount() + " docs at " + (long) meter.getMeanRate() + " doc/s");- }-- /**- * Wait for merges.- *- * @param collection the collection- * @throws Exception the exception- */- public void waitForMerges(String collection) throws Exception {- forceMerge(collection, Integer.MAX_VALUE);- }-- /**- * Force merge.- *- * @param collection the collection- * @param maxMergeSegments the max merge segments- * @throws Exception the exception- */- public void forceMerge(String collection, int maxMergeSegments) throws Exception {- if (createCollectionAndIndex) {- // we control segment count for a more informative benchmark *and* because background- // merging would continue after- // indexing and overlap with the benchmark- if (maxMergeSegments == Integer.MAX_VALUE) {- log("waiting for merges to finish...\n");- } else {- log("merging segments to " + maxMergeSegments + " segments ...\n");- }-- UpdateRequest optimizeRequest = new UpdateRequest();- final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size()));- optimizeRequest.setAction(UpdateRequest.ACTION.OPTIMIZE, false, true, maxMergeSegments);- client.requestWithBaseUrl(url, optimizeRequest, collection);- }- }-- /**- * Dump core info.- *- * @throws IOException the io exception- */- @SuppressForbidden(reason = "JMH uses std out for user output")- public void dumpCoreInfo() throws IOException {- cluster.dumpCoreInfo(- !BaseBenchState.QUIET_LOG- ? System.out- : new PrintStream(OutputStream.nullOutputStream(), false, StandardCharsets.UTF_8));- }- }-- /**- * Params modifiable solr params.- *- * @param moreParams the more params- * @return the modifiable solr params- */- public static ModifiableSolrParams params(String... moreParams) {- ModifiableSolrParams params = new ModifiableSolrParams();- for (int i = 0; i < moreParams.length; i += 2) {- params.add(moreParams[i], moreParams[i + 1]);- }- return params;- }-- /**- * Params modifiable solr params.- *- * @param params the params- * @param moreParams the more params- * @return the modifiable solr params- */- public static ModifiableSolrParams params(ModifiableSolrParams params, String... moreParams) {- for (int i = 0; i < moreParams.length; i += 2) {- params.add(moreParams[i], moreParams[i + 1]);- }- return params;- }-- /**- * Gets file.- *- * @param name the name- * @return the file- */- public static Path getFile(String name) {- final URL url =- MiniClusterState.class- .getClassLoader()- .getResource(name.replace(FileSystems.getDefault().getSeparator(), "/"));- if (url != null) {- try {- return Path.of(url.toURI());- } catch (Exception e) {- throw new RuntimeException(- "Resource was found on classpath, but cannot be resolved to a "- + "normal file (maybe it is part of a JAR file): "- + name);- }- }- Path file = Path.of(name);- if (Files.exists(file)) {- return file;- } else {- file = Path.of("../../../", name);- if (Files.exists(file)) {- return file;- }- }- throw new RuntimeException(- "Cannot find resource in classpath or in file-system (relative to CWD): "- + name- + " CWD="- + Path.of("").toAbsolutePath());- }-}diff --git a/solr/benchmark/src/java/org/apache/solr/bench/SolrBenchState.java b/solr/benchmark/src/java/org/apache/solr/bench/SolrBenchState.javanew file mode 100755index 00000000000..1aade6389dd--- /dev/null+++ b/solr/benchmark/src/java/org/apache/solr/bench/SolrBenchState.java@@ -0,0 +1,580 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.bench;++import static org.apache.commons.io.file.PathUtils.deleteDirectory;+import static org.apache.solr.bench.BaseBenchState.log;++import com.codahale.metrics.Meter;+import java.io.IOException;+import java.io.OutputStream;+import java.io.PrintStream;+import java.lang.management.ManagementFactory;+import java.net.URL;+import java.nio.charset.StandardCharsets;+import java.nio.file.FileSystems;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;+import java.util.SplittableRandom;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.TimeUnit;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.jetty.HttpJettySolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.cloud.MiniSolrCloudCluster;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.util.IOUtils;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.SolrNamedThreadFactory;+import org.apache.solr.common.util.SuppressForbidden;+import org.apache.solr.embedded.JettySolrRunner;+import org.apache.solr.util.SolrTestNonSecureRandomProvider;+import org.openjdk.jmh.annotations.Level;+import org.openjdk.jmh.annotations.Scope;+import org.openjdk.jmh.annotations.Setup;+import org.openjdk.jmh.annotations.State;+import org.openjdk.jmh.annotations.TearDown;+import org.openjdk.jmh.infra.BenchmarkParams;+import org.openjdk.jmh.infra.Control;++/** JMH {@link State} class for Solr benchmarks. This is the benchmark's facade to Solr. */+@State(Scope.Benchmark)+public class SolrBenchState {++ /** The constant PROC_COUNT. */+ public static final int PROC_COUNT =+ ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors();++ /** The Metrics enabled. */+ boolean metricsEnabled = true;++ /** The Nodes. */+ public List<String> nodes;++ public String zkHost;++ /** The Cluster. */+ MiniSolrCloudCluster cluster;++ /** The Client. */+ public HttpJettySolrClient client;++ /** The Run cnt. */+ int runCnt = 0;++ /** The Create collection and index. */+ boolean createCollectionAndIndex = true;++ /** The Delete mini cluster. */+ boolean deleteMiniCluster = true;++ /** Unless overridden we ensure SecureRandoms do not block. */+ boolean doNotWeakenSecureRandom = Boolean.getBoolean("doNotWeakenSecureRandom");++ /** The Mini cluster base dir. */+ Path miniClusterBaseDir;++ /** To Allow cluster reuse. */+ boolean allowClusterReuse = false;++ /** The Is warmup. */+ boolean isWarmup;++ private SplittableRandom random;+ private String workDir;++ private boolean useHttp1 = Boolean.getBoolean("solr.http1");++ /**+ * Tear down.+ *+ * @param benchmarkParams the benchmark params+ * @throws Exception the exception+ */+ @TearDown(Level.Iteration)+ public void tearDown(BenchmarkParams benchmarkParams) throws Exception {++ // dump Solr metrics+ Path metricsResults =+ Path.of(+ workDir,+ "metrics-results",+ benchmarkParams.id(),+ String.valueOf(runCnt++),+ benchmarkParams.getBenchmark() + ".txt");+ Files.createDirectories(metricsResults.getParent());++ cluster.dumpMetrics(metricsResults.getParent(), metricsResults.getFileName().toString());+ }++ /**+ * Check warm up.+ *+ * @param control the control+ * @throws Exception the exception+ */+ @Setup(Level.Iteration)+ public void checkWarmUp(Control control) throws Exception {+ isWarmup = control.stopMeasurement;+ }++ /**+ * Shutdown Solr.+ *+ * @param benchmarkParams the benchmark params+ * @throws Exception the exception+ */+ @TearDown(Level.Trial)+ public void shutdownSolr(BenchmarkParams benchmarkParams, BaseBenchState baseBenchState)+ throws Exception {+ BaseBenchState.dumpHeap(benchmarkParams);++ IOUtils.closeQuietly(client);+ cluster.shutdown();+ logClusterDirectorySize();+ }++ private void logClusterDirectorySize() throws IOException {+ log("");+ Files.list(miniClusterBaseDir.toAbsolutePath())+ .forEach(+ (node) -> {+ try {+ long clusterSize =+ Files.walk(node)+ .filter(Files::isRegularFile)+ .mapToLong(+ file -> {+ try {+ return Files.size(file);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ })+ .sum();+ log("mini cluster node size (bytes) " + node + " " + clusterSize);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ });+ }++ /**+ * Do setup.+ *+ * @param benchmarkParams the benchmark params+ * @param baseBenchState the base bench state+ * @throws Exception the exception+ */+ @Setup(Level.Trial)+ public void doSetup(BenchmarkParams benchmarkParams, BaseBenchState baseBenchState)+ throws Exception {++ if (!doNotWeakenSecureRandom) {+ // remove all blocking from all secure randoms+ SolrTestNonSecureRandomProvider.injectProvider();+ }++ workDir = System.getProperty("workBaseDir", "build/work");++ Path currentRelativePath = Path.of("");+ String s = currentRelativePath.toAbsolutePath().toString();+ log("current relative path is: " + s);+ log("work path is: " + workDir);++ System.setProperty("doNotWaitForMergesOnIWClose", "true");++ System.setProperty("pkiHandlerPrivateKeyPath", "");+ System.setProperty("pkiHandlerPublicKeyPath", "");++ System.setProperty("solr.configset.default.confdir", "../server/solr/configsets/_default");++ this.random = new SplittableRandom(BaseBenchState.getRandomSeed());++ // not currently usable, but would enable JettySolrRunner's ill-conceived jetty.testMode and+ // allow using SSL++ // System.getProperty("jetty.testMode", "true");+ // SolrCloudTestCase.sslConfig = SolrTestCaseJ4.buildSSLConfig();++ String baseDirSysProp = System.getProperty("miniClusterBaseDir");+ if (baseDirSysProp != null) {+ deleteMiniCluster = false;+ miniClusterBaseDir = Path.of(baseDirSysProp);+ if (Files.exists(miniClusterBaseDir)) {+ createCollectionAndIndex = false;+ allowClusterReuse = true;+ }+ } else {+ miniClusterBaseDir = Path.of(workDir, "mini-cluster");+ }++ System.setProperty("metricsEnabled", String.valueOf(metricsEnabled));+ }++ /**+ * Metrics enabled.+ *+ * @param metricsEnabled the metrics enabled+ */+ public void metricsEnabled(boolean metricsEnabled) {+ this.metricsEnabled = metricsEnabled;+ }++ /**+ * Start Solr.+ *+ * @param nodeCount the node count+ */+ public void startSolr(int nodeCount) {+ log("starting mini cluster at base directory: " + miniClusterBaseDir.toAbsolutePath());++ if (!allowClusterReuse && Files.exists(miniClusterBaseDir)) {+ log("mini cluster base directory exists, removing ...");+ try {+ deleteDirectory(miniClusterBaseDir);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ createCollectionAndIndex = true;+ } else if (Files.exists(miniClusterBaseDir)) {+ createCollectionAndIndex = false;+ deleteMiniCluster = false;+ }++ try {+ cluster =+ new MiniSolrCloudCluster.Builder(nodeCount, miniClusterBaseDir)+ .formatZkServer(false)+ .addConfig("conf", getFile("src/resources/configs/cloud-minimal/conf"))+ .configure();+ } catch (Exception e) {+ if (Files.exists(miniClusterBaseDir)) {+ try {+ deleteDirectory(miniClusterBaseDir);+ } catch (IOException ex) {+ e.addSuppressed(ex);+ }+ }+ throw new RuntimeException(e);+ }++ nodes = new ArrayList<>(nodeCount);+ List<JettySolrRunner> jetties = cluster.getJettySolrRunners();+ for (JettySolrRunner runner : jetties) {+ nodes.add(runner.getBaseUrl().toString());+ }+ zkHost = cluster.getZkServer().getZkAddress();++ client = new HttpJettySolrClient.Builder(nodes.get(0)).useHttp1_1(useHttp1).build();++ log("done starting mini cluster");+ log("");+ }++ /**+ * Gets random.+ *+ * @return the random+ */+ public SplittableRandom getRandom() {+ return random;+ }++ /**+ * Create collection.+ *+ * @param collection the collection+ * @param numShards the num shards+ * @param numReplicas the num replicas+ * @throws Exception the exception+ */+ public void createCollection(String collection, int numShards, int numReplicas) throws Exception {+ if (createCollectionAndIndex) {+ try {++ CollectionAdminRequest.Create request =+ CollectionAdminRequest.createCollection(collection, "conf", numShards, numReplicas);+ client.requestWithBaseUrl(+ nodes.get(random.nextInt(cluster.getJettySolrRunners().size())), request, null);++ cluster.waitForActiveCollection(+ collection, 15, TimeUnit.SECONDS, numShards, numShards * numReplicas);+ } catch (Exception e) {+ if (Files.exists(miniClusterBaseDir)) {+ deleteDirectory(miniClusterBaseDir);+ }+ throw e;+ }+ }+ }++ /** Setting useHttp1 to true will make the {@link #client} use http1 */+ public void setUseHttp1(boolean useHttp1) {+ if (client != null) {+ throw new IllegalStateException(+ "You can only change this setting before starting the Mini Cluster");+ }+ this.useHttp1 = useHttp1;+ }++ @SuppressForbidden(reason = "This module does not need to deal with logging context")+ public void index(String collection, Docs docs, int docCount) throws Exception {+ index(collection, docs, docCount, true);+ }++ /**+ * Index.+ *+ * @param collection the collection+ * @param docs the docs+ * @param docCount the doc count+ * @throws Exception the exception+ */+ public void index(String collection, Docs docs, int docCount, boolean parallel) throws Exception {+ if (createCollectionAndIndex) {+ log("indexing data for benchmark...");+ if (parallel) {+ indexParallel(collection, docs, docCount);+ } else {+ indexBatch(collection, docs, docCount, 10000);+ }+ log("done indexing data for benchmark");++ log("committing data ...");+ UpdateRequest commitRequest = new UpdateRequest();+ final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size()));+ commitRequest.setAction(UpdateRequest.ACTION.COMMIT, false, true);+ client.requestWithBaseUrl(url, commitRequest, collection);+ log("done committing data");+ } else {+ cluster.waitForActiveCollection(collection, 15, TimeUnit.SECONDS);+ }++ QueryRequest queryRequest = new QueryRequest(new SolrQuery("q", "*:*", "rows", "1"));+ final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size()));+ NamedList<Object> result = client.requestWithBaseUrl(url, queryRequest, collection);++ log("sanity check of single row query result: " + result);+ log("");++ log("Dump Core Info");+ dumpCoreInfo();+ }++ @SuppressForbidden(reason = "This module does not need to deal with logging context")+ private void indexParallel(String collection, Docs docs, int docCount)+ throws InterruptedException {+ Meter meter = new Meter();+ ExecutorService executorService =+ Executors.newFixedThreadPool(+ Runtime.getRuntime().availableProcessors(),+ new SolrNamedThreadFactory("SolrJMH Indexer"));+ ScheduledExecutorService scheduledExecutor =+ Executors.newSingleThreadScheduledExecutor(+ new SolrNamedThreadFactory("SolrJMH Indexer Progress"));+ scheduledExecutor.scheduleAtFixedRate(+ () -> {+ if (meter.getCount() == docCount) {+ scheduledExecutor.shutdown();+ } else {+ log(meter.getCount() + " docs at " + meter.getMeanRate() + " doc/s");+ }+ },+ 10,+ 10,+ TimeUnit.SECONDS);+ for (int i = 0; i < docCount; i++) {+ executorService.execute(+ new Runnable() {+ final SplittableRandom threadRandom = random.split();++ @Override+ public void run() {+ UpdateRequest updateRequest = new UpdateRequest();+ final var url = nodes.get(threadRandom.nextInt(cluster.getJettySolrRunners().size()));+ SolrInputDocument doc = docs.inputDocument();+ // log("add doc " + doc);+ updateRequest.add(doc);+ meter.mark();++ try {+ client.requestWithBaseUrl(url, updateRequest, collection);+ } catch (Exception e) {+ throw new RuntimeException(e);+ }+ }+ });+ }++ log("done adding docs, waiting for executor to terminate...");++ executorService.shutdown();+ boolean result = false;+ while (!result) {+ result = executorService.awaitTermination(600, TimeUnit.MINUTES);+ }++ scheduledExecutor.shutdown();+ }++ private void indexBatch(String collection, Docs docs, int docCount, int batchSize)+ throws SolrServerException, IOException {+ Meter meter = new Meter();+ List<SolrInputDocument> batch = new ArrayList<>(batchSize);+ for (int i = 1; i <= docCount; i++) {+ batch.add(docs.inputDocument());+ if (i % batchSize == 0) {+ UpdateRequest updateRequest = new UpdateRequest();+ updateRequest.add(batch);+ client.requestWithBaseUrl(nodes.get(0), updateRequest, collection);+ meter.mark(batch.size());+ batch.clear();+ log(meter.getCount() + " docs at " + (long) meter.getMeanRate() + " doc/s");+ }+ }+ if (!batch.isEmpty()) {+ UpdateRequest updateRequest = new UpdateRequest();+ updateRequest.add(batch);+ client.requestWithBaseUrl(nodes.get(0), updateRequest, collection);+ meter.mark(batch.size());+ batch = null;+ }+ log(meter.getCount() + " docs at " + (long) meter.getMeanRate() + " doc/s");+ }++ /**+ * Wait for merges.+ *+ * @param collection the collection+ * @throws Exception the exception+ */+ public void waitForMerges(String collection) throws Exception {+ forceMerge(collection, Integer.MAX_VALUE);+ }++ /**+ * Force merge.+ *+ * @param collection the collection+ * @param maxMergeSegments the max merge segments+ * @throws Exception the exception+ */+ public void forceMerge(String collection, int maxMergeSegments) throws Exception {+ if (createCollectionAndIndex) {+ // we control segment count for a more informative benchmark *and* because background+ // merging would continue after+ // indexing and overlap with the benchmark+ if (maxMergeSegments == Integer.MAX_VALUE) {+ log("waiting for merges to finish...\n");+ } else {+ log("merging segments to " + maxMergeSegments + " segments ...\n");+ }++ UpdateRequest optimizeRequest = new UpdateRequest();+ final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size()));+ optimizeRequest.setAction(UpdateRequest.ACTION.OPTIMIZE, false, true, maxMergeSegments);+ client.requestWithBaseUrl(url, optimizeRequest, collection);+ }+ }++ /**+ * Dump core info.+ *+ * @throws IOException the io exception+ */+ @SuppressForbidden(reason = "JMH uses std out for user output")+ public void dumpCoreInfo() throws IOException {+ cluster.dumpCoreInfo(+ !BaseBenchState.QUIET_LOG+ ? System.out+ : new PrintStream(OutputStream.nullOutputStream(), false, StandardCharsets.UTF_8));+ }++ /**+ * Params modifiable solr params.+ *+ * @param moreParams the more params+ * @return the modifiable solr params+ */+ public static ModifiableSolrParams params(String... moreParams) {+ ModifiableSolrParams params = new ModifiableSolrParams();+ for (int i = 0; i < moreParams.length; i += 2) {+ params.add(moreParams[i], moreParams[i + 1]);+ }+ return params;+ }++ /**+ * Params modifiable solr params.+ *+ * @param params the params+ * @param moreParams the more params+ * @return the modifiable solr params+ */+ public static ModifiableSolrParams params(ModifiableSolrParams params, String... moreParams) {+ for (int i = 0; i < moreParams.length; i += 2) {+ params.add(moreParams[i], moreParams[i + 1]);+ }+ return params;+ }++ /**+ * Gets file.+ *+ * @param name the name+ * @return the file+ */+ public static Path getFile(String name) {+ final URL url =+ SolrBenchState.class+ .getClassLoader()+ .getResource(name.replace(FileSystems.getDefault().getSeparator(), "/"));+ if (url != null) {+ try {+ return Path.of(url.toURI());+ } catch (Exception e) {+ throw new RuntimeException(+ "Resource was found on classpath, but cannot be resolved to a "+ + "normal file (maybe it is part of a JAR file): "+ + name);+ }+ }+ Path file = Path.of(name);+ if (Files.exists(file)) {+ return file;+ } else {+ file = Path.of("../../../", name);+ if (Files.exists(file)) {+ return file;+ }+ }+ throw new RuntimeException(+ "Cannot find resource in classpath or in file-system (relative to CWD): "+ + name+ + " CWD="+ + Path.of("").toAbsolutePath());+ }+}diff --git a/solr/benchmark/src/java/org/apache/solr/bench/index/CloudIndexing.java b/solr/benchmark/src/java/org/apache/solr/bench/index/CloudIndexing.javaindex 5f6c6526690..770699d2a54 100755--- a/solr/benchmark/src/java/org/apache/solr/bench/index/CloudIndexing.java+++ b/solr/benchmark/src/java/org/apache/solr/bench/index/CloudIndexing.java@@ -24,7 +24,7 @@ import static org.apache.solr.bench.generators.SourceDSL.strings;import java.util.Iterator;import java.util.concurrent.TimeUnit;import org.apache.solr.bench.Docs;-import org.apache.solr.bench.MiniClusterState;+import org.apache.solr.bench.SolrBenchState;import org.apache.solr.client.solrj.request.UpdateRequest;import org.apache.solr.common.SolrInputDocument;import org.openjdk.jmh.annotations.Benchmark;@@ -114,22 +114,20 @@ public class CloudIndexing {}@Setup(Level.Trial)- public void doSetup(MiniClusterState.MiniClusterBenchState miniClusterState) throws Exception {+ public void doSetup(SolrBenchState solrBenchState) throws Exception {preGenerate();System.setProperty("mergePolicyFactory", "org.apache.solr.index.NoMergePolicyFactory");- miniClusterState.startMiniCluster(nodeCount);- miniClusterState.createCollection(COLLECTION, numShards, numReplicas);+ solrBenchState.startSolr(nodeCount);+ solrBenchState.createCollection(COLLECTION, numShards, numReplicas);}}@Benchmark- public Object indexDoc(MiniClusterState.MiniClusterBenchState miniClusterState, BenchState state)- throws Exception {+ public Object indexDoc(SolrBenchState solrBenchState, BenchState state) throws Exception {UpdateRequest updateRequest = new UpdateRequest();updateRequest.add(state.getNextDoc());- final var url =- miniClusterState.nodes.get(miniClusterState.getRandom().nextInt(state.nodeCount));- return miniClusterState.client.requestWithBaseUrl(url, updateRequest, BenchState.COLLECTION);+ final var url = solrBenchState.nodes.get(solrBenchState.getRandom().nextInt(state.nodeCount));+ return solrBenchState.client.requestWithBaseUrl(url, updateRequest, BenchState.COLLECTION);}}diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.javaindex 0b7b4a5ff5a..0365fe00410 100644--- a/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java@@ -24,7 +24,7 @@ import java.util.Map;import java.util.Set;import java.util.concurrent.TimeUnit;import org.apache.solr.bench.Docs;-import org.apache.solr.bench.MiniClusterState;+import org.apache.solr.bench.SolrBenchState;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.client.solrj.response.QueryResponse;@@ -67,9 +67,8 @@ public class ExitableDirectoryReaderSearch {int WORDS = NUM_DOCS / 100;@Setup(Level.Trial)- public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState)- throws Exception {- miniClusterState.setUseHttp1(true);+ public void setupTrial(SolrBenchState solrBenchState) throws Exception {+ solrBenchState.setUseHttp1(true);System.setProperty("documentCache.enabled", "false");System.setProperty("queryResultCache.enabled", "false");System.setProperty("filterCache.enabled", "false");@@ -78,9 +77,9 @@ public class ExitableDirectoryReaderSearch {System.setProperty("segmentsPerTier", "200");System.setProperty("maxBufferedDocs", "100");- miniClusterState.startMiniCluster(1);+ solrBenchState.startSolr(1);log("######### Creating index ...");- miniClusterState.createCollection(COLLECTION, 1, 1);+ solrBenchState.createCollection(COLLECTION, 1, 1);// create a lot of large-ish fields to scan positionsDocs docs =Docs.docs(1234567890L)@@ -103,9 +102,9 @@ public class ExitableDirectoryReaderSearch {.field("f9_ts",strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10));- miniClusterState.index(COLLECTION, docs, NUM_DOCS, true);- miniClusterState.forceMerge(COLLECTION, 200);- miniClusterState.dumpCoreInfo();+ solrBenchState.index(COLLECTION, docs, NUM_DOCS, true);+ solrBenchState.forceMerge(COLLECTION, 200);+ solrBenchState.dumpCoreInfo();}// this adds significant processing time to the checking of query limits@@ -116,13 +115,13 @@ public class ExitableDirectoryReaderSearch {private static final String matchExpression = "ExitableTermsEnum:-1";@Setup(Level.Iteration)- public void setupQueries(MiniClusterState.MiniClusterBenchState state) throws Exception {+ public void setupQueries(SolrBenchState solrBenchState) throws Exception {if (verifyEDRInUse) {TestInjection.queryTimeout = new CallerSpecificQueryLimit(Set.of(matchExpression));}// reload collection to force searcher / reader refreshCollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION);- state.client.request(reload);+ solrBenchState.client.request(reload);queryFields =Docs.docs(1234567890L)@@ -154,25 +153,23 @@ public class ExitableDirectoryReaderSearch {private static ModifiableSolrParams createInitialParams() {ModifiableSolrParams params =- MiniClusterState.params("rows", "100", "timeAllowed", "1000", "fl", "*");+ SolrBenchState.params("rows", "100", "timeAllowed", "1000", "fl", "*");return params;}@Benchmark- public void testShortQuery(- MiniClusterState.MiniClusterBenchState miniClusterState, Blackhole bh, BenchState state)+ public void testShortQuery(SolrBenchState solrBenchState, Blackhole bh, BenchState state)throws Exception {SolrInputDocument queryDoc = state.queryFields.inputDocument();ModifiableSolrParams params = createInitialParams();params.set("q", "f1_ts:" + queryDoc.getFieldValue("f1_ts").toString());QueryRequest queryRequest = new QueryRequest(params);- QueryResponse rsp = queryRequest.process(miniClusterState.client, COLLECTION);+ QueryResponse rsp = queryRequest.process(solrBenchState.client, COLLECTION);bh.consume(rsp);}@Benchmark- public void testLongQuery(- MiniClusterState.MiniClusterBenchState miniClusterState, Blackhole bh, BenchState state)+ public void testLongQuery(SolrBenchState solrBenchState, Blackhole bh, BenchState state)throws Exception {SolrInputDocument queryDoc = state.queryFields.inputDocument();ModifiableSolrParams params = createInitialParams();@@ -186,7 +183,7 @@ public class ExitableDirectoryReaderSearch {}params.set("q", query.toString());QueryRequest queryRequest = new QueryRequest(params);- QueryResponse rsp = queryRequest.process(miniClusterState.client, COLLECTION);+ QueryResponse rsp = queryRequest.process(solrBenchState.client, COLLECTION);bh.consume(rsp);}}diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java b/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.javaindex 197a346bb3d..f736aa4bf80 100644--- a/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java@@ -24,7 +24,7 @@ import java.net.URI;import java.nio.charset.StandardCharsets;import org.apache.solr.bench.BaseBenchState;import org.apache.solr.bench.Docs;-import org.apache.solr.bench.MiniClusterState;+import org.apache.solr.bench.SolrBenchState;import org.apache.solr.bench.SolrRandomnessSource;import org.apache.solr.bench.generators.SolrGen;import org.apache.solr.client.solrj.SolrServerException;@@ -71,8 +71,7 @@ public class FilterCache {String baseUrl;@Setup(Level.Trial)- public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState)- throws Exception {+ public void setupTrial(SolrBenchState solrBenchState) throws Exception {String cacheEnabled = cacheEnabledAsyncSize.split(":")[0];String asyncCache = cacheEnabledAsyncSize.split(":")[1];String cacheSize = cacheEnabledAsyncSize.split(":")[2];@@ -81,8 +80,8 @@ public class FilterCache {System.setProperty("filterCache.initialSize", cacheSize);System.setProperty("filterCache.async", asyncCache);- miniClusterState.startMiniCluster(1);- miniClusterState.createCollection(COLLECTION, 1, 1);+ solrBenchState.startSolr(1);+ solrBenchState.createCollection(COLLECTION, 1, 1);Docs docs = Docs.docs().field("id", integers().incrementing());@@ -100,24 +99,24 @@ public class FilterCache {docs.field("Ea_b", booleans);docs.field("FB_b", booleans);- miniClusterState.index(COLLECTION, docs, 30 * 1000);- baseUrl = miniClusterState.nodes.get(0);+ solrBenchState.index(COLLECTION, docs, 30 * 1000);+ baseUrl = solrBenchState.nodes.get(0);}@Setup(Level.Iteration)- public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState)+ public void setupIteration(SolrBenchState solrBenchState)throws SolrServerException, IOException {// Reload the collection/core to drop existing cachesCollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION);- miniClusterState.client.requestWithBaseUrl(miniClusterState.nodes.get(0), reload, null);+ solrBenchState.client.requestWithBaseUrl(solrBenchState.nodes.get(0), reload, null);}@TearDown(Level.Iteration)- public void dumpMetrics(MiniClusterState.MiniClusterBenchState miniClusterState) {+ public void dumpMetrics(SolrBenchState solrBenchState) {// TODO add a verbose flagString url =- miniClusterState.nodes.get(0)+ solrBenchState.nodes.get(0)+ "/admin/metrics?prefix=CACHE.searcher.filterCache&omitHeader=true";HttpURLConnection conn = null;try {@@ -134,20 +133,17 @@ public class FilterCache {}@Benchmark- public Object filterCacheMultipleQueries(- BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState)+ public Object filterCacheMultipleQueries(BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {- return miniClusterState.client.requestWithBaseUrl(+ return solrBenchState.client.requestWithBaseUrl(benchState.baseUrl,- miniClusterState.getRandom().nextBoolean() ? benchState.q1 : benchState.q2,+ solrBenchState.getRandom().nextBoolean() ? benchState.q1 : benchState.q2,COLLECTION);}@Benchmark- public Object filterCacheSingleQuery(- BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState)+ public Object filterCacheSingleQuery(BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {- return miniClusterState.client.requestWithBaseUrl(- benchState.baseUrl, benchState.q1, COLLECTION);+ return solrBenchState.client.requestWithBaseUrl(benchState.baseUrl, benchState.q1, COLLECTION);}}diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java b/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.javaindex 9482f8a588b..7c1216b41b5 100755--- a/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java@@ -24,7 +24,7 @@ import java.util.SplittableRandom;import java.util.concurrent.TimeUnit;import org.apache.solr.bench.BaseBenchState;import org.apache.solr.bench.Docs;-import org.apache.solr.bench.MiniClusterState;+import org.apache.solr.bench.SolrBenchState;import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;@@ -99,16 +99,15 @@ public class JsonFaceting {private ModifiableSolrParams params;@Setup(Level.Trial)- public void setup(- BenchmarkParams benchmarkParams, MiniClusterState.MiniClusterBenchState miniClusterState)+ public void setup(BenchmarkParams benchmarkParams, SolrBenchState solrBenchState)throws Exception {System.setProperty("maxMergeAtOnce", "50");System.setProperty("segmentsPerTier", "50");- miniClusterState.startMiniCluster(nodeCount);+ solrBenchState.startSolr(nodeCount);- miniClusterState.createCollection(collection, numShards, numReplicas);+ solrBenchState.createCollection(collection, numShards, numReplicas);// Define random documentsDocs docs =@@ -132,12 +131,12 @@ public class JsonFaceting {.field(integers().allWithMaxCardinality(facetCard2)).field(integers().allWithMaxCardinality(facetCard2));- miniClusterState.index(collection, docs, docCount);- miniClusterState.forceMerge(collection, 25);+ solrBenchState.index(collection, docs, docCount);+ solrBenchState.forceMerge(collection, 25);params = new ModifiableSolrParams();- MiniClusterState.params(+ SolrBenchState.params(params,"q","*:*",@@ -167,7 +166,7 @@ public class JsonFaceting {params.set("timeAllowed", "5000");}- // MiniClusterState.log("params: " + params + "\n");+ // SolrBenchState.log("params: " + params + "\n");}@State(Scope.Thread)@@ -185,17 +184,17 @@ public class JsonFaceting {@Benchmark@Timeout(time = 500, timeUnit = TimeUnit.SECONDS)public void jsonFacet(- MiniClusterState.MiniClusterBenchState miniClusterState,+ SolrBenchState solrBenchState,BenchState state,BenchState.ThreadState threadState,Blackhole bh)throws Exception {- final var url = miniClusterState.nodes.get(threadState.random.nextInt(state.nodeCount));+ final var url = solrBenchState.nodes.get(threadState.random.nextInt(state.nodeCount));QueryRequest queryRequest = new QueryRequest(state.params);NamedList<Object> result =- miniClusterState.client.requestWithBaseUrl(url, queryRequest, state.collection);+ solrBenchState.client.requestWithBaseUrl(url, queryRequest, state.collection);- // MiniClusterState.log("result: " + result);+ // SolrBenchState.log("result: " + result);bh.consume(result);}diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.javaindex e7f574cd341..70d307963a6 100644--- a/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java@@ -27,7 +27,7 @@ import java.util.Set;import java.util.stream.Collectors;import org.apache.solr.bench.CircularIterator;import org.apache.solr.bench.Docs;-import org.apache.solr.bench.MiniClusterState;+import org.apache.solr.bench.SolrBenchState;import org.apache.solr.bench.generators.SolrGen;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.request.CollectionAdminRequest;@@ -68,11 +68,10 @@ public class NumericSearch {Iterator<String> queries;@Setup(Level.Trial)- public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState)- throws Exception {- miniClusterState.setUseHttp1(true);- miniClusterState.startMiniCluster(1);- miniClusterState.createCollection(COLLECTION, 1, 1);+ public void setupTrial(SolrBenchState solrBenchState) throws Exception {+ solrBenchState.setUseHttp1(true);+ solrBenchState.startSolr(1);+ solrBenchState.createCollection(COLLECTION, 1, 1);int maxCardinality = 10000;int numDocs = 2000000;setValues = integers().allWithMaxCardinality(maxCardinality);@@ -93,16 +92,15 @@ public class NumericSearch {.field("term_high_s", highCardinalityTerms);// .field("numbers_dt", setValues);- miniClusterState.index(COLLECTION, docs, numDocs, false);- basePath = miniClusterState.nodes.get(0);+ solrBenchState.index(COLLECTION, docs, numDocs, false);+ basePath = solrBenchState.nodes.get(0);SolrQuery q = new SolrQuery("*:*");q.setParam("facet", "true");q.setParam("rows", "0");q.setParam("facet.field", "numbers_i_dv", "term_low_s", "term_high_s");q.setParam("facet.limit", String.valueOf(maxCardinality));QueryRequest req = new QueryRequest(q);- QueryResponse response =- req.processWithBaseUrl(miniClusterState.client, basePath, COLLECTION);+ QueryResponse response = req.processWithBaseUrl(solrBenchState.client, basePath, COLLECTION);Set<String> numbers =response.getFacetField("numbers_i_dv").getValues().stream().map(FacetField.Count::getName)@@ -140,11 +138,11 @@ public class NumericSearch {}@Setup(Level.Iteration)- public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState)+ public void setupIteration(SolrBenchState solrBenchState)throws SolrServerException, IOException {// Reload the collection/core to drop existing cachesCollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION);- miniClusterState.client.requestWithBaseUrl(miniClusterState.nodes.get(0), reload, null);+ solrBenchState.client.requestWithBaseUrl(solrBenchState.nodes.get(0), reload, null);}public QueryRequest intSetQuery(boolean dvs) {@@ -176,97 +174,75 @@ public class NumericSearch {}@Benchmark- public Object intSet(- Blackhole blackhole,- BenchState benchState,- MiniClusterState.MiniClusterBenchState miniClusterState)+ public Object intSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {QueryResponse response =- benchState.intSetQuery(false).process(miniClusterState.client, COLLECTION);+ benchState.intSetQuery(false).process(solrBenchState.client, COLLECTION);blackhole.consume(response);return response;}@Benchmark- public Object longSet(- Blackhole blackhole,- BenchState benchState,- MiniClusterState.MiniClusterBenchState miniClusterState)+ public Object longSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {QueryResponse response =- benchState.longSetQuery(false).process(miniClusterState.client, COLLECTION);+ benchState.longSetQuery(false).process(solrBenchState.client, COLLECTION);blackhole.consume(response);return response;}@Benchmark- public Object floatSet(- Blackhole blackhole,- BenchState benchState,- MiniClusterState.MiniClusterBenchState miniClusterState)+ public Object floatSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {QueryResponse response =- benchState.floatSetQuery(false).process(miniClusterState.client, COLLECTION);+ benchState.floatSetQuery(false).process(solrBenchState.client, COLLECTION);blackhole.consume(response);return response;}@Benchmark- public Object doubleSet(- Blackhole blackhole,- BenchState benchState,- MiniClusterState.MiniClusterBenchState miniClusterState)+ public Object doubleSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {QueryResponse response =- benchState.doubleSetQuery(false).process(miniClusterState.client, COLLECTION);+ benchState.doubleSetQuery(false).process(solrBenchState.client, COLLECTION);blackhole.consume(response);return response;}@Benchmark- public Object intDvSet(- Blackhole blackhole,- BenchState benchState,- MiniClusterState.MiniClusterBenchState miniClusterState)+ public Object intDvSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {QueryResponse response =- benchState.intSetQuery(true).process(miniClusterState.client, COLLECTION);+ benchState.intSetQuery(true).process(solrBenchState.client, COLLECTION);blackhole.consume(response);return response;}@Benchmark- public Object longDvSet(- Blackhole blackhole,- BenchState benchState,- MiniClusterState.MiniClusterBenchState miniClusterState)+ public Object longDvSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {QueryResponse response =- benchState.longSetQuery(true).process(miniClusterState.client, COLLECTION);+ benchState.longSetQuery(true).process(solrBenchState.client, COLLECTION);blackhole.consume(response);return response;}@Benchmarkpublic Object floatDvSet(- Blackhole blackhole,- BenchState benchState,- MiniClusterState.MiniClusterBenchState miniClusterState)+ Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {QueryResponse response =- benchState.floatSetQuery(true).process(miniClusterState.client, COLLECTION);+ benchState.floatSetQuery(true).process(solrBenchState.client, COLLECTION);blackhole.consume(response);return response;}@Benchmarkpublic Object doubleDvSet(- Blackhole blackhole,- BenchState benchState,- MiniClusterState.MiniClusterBenchState miniClusterState)+ Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {QueryResponse response =- benchState.doubleSetQuery(true).process(miniClusterState.client, COLLECTION);+ benchState.doubleSetQuery(true).process(solrBenchState.client, COLLECTION);blackhole.consume(response);return response;}diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.javaindex 4b6118fed27..6929d2810a5 100644--- a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java@@ -23,8 +23,7 @@ import static org.apache.solr.bench.generators.SourceDSL.strings;import java.io.IOException;import org.apache.solr.bench.Docs;-import org.apache.solr.bench.MiniClusterState;-import org.apache.solr.bench.MiniClusterState.MiniClusterBenchState;+import org.apache.solr.bench.SolrBenchState;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.client.solrj.response.InputStreamResponseParser;@@ -64,10 +63,10 @@ public class QueryResponseWriters {private QueryRequest q;@Setup(Level.Trial)- public void setup(MiniClusterBenchState miniClusterState) throws Exception {+ public void setup(SolrBenchState solrBenchState) throws Exception {- miniClusterState.startMiniCluster(1);- miniClusterState.createCollection(collection, 1, 1);+ solrBenchState.startSolr(1);+ solrBenchState.createCollection(collection, 1, 1);// only stored fields are needed to cover the response writers perfDocs docGen =@@ -76,8 +75,8 @@ public class QueryResponseWriters {.field("text2_ts", strings().basicLatinAlphabet().multi(25).ofLengthBetween(30, 64)).field("bools_b", booleans().all()).field("int1_is", integers().all());- miniClusterState.index(collection, docGen, docs);- miniClusterState.forceMerge(collection, 5);+ solrBenchState.index(collection, docGen, docs);+ solrBenchState.forceMerge(collection, 5);ModifiableSolrParams params = new ModifiableSolrParams();params.set(CommonParams.Q, "*:*");@@ -85,14 +84,13 @@ public class QueryResponseWriters {params.set(CommonParams.ROWS, docs);q = new QueryRequest(params);q.setResponseParser(new InputStreamResponseParser(wt));- String base = miniClusterState.nodes.get(0);+ String base = solrBenchState.nodes.get(0);}}@Benchmark- public Object query(- BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState)+ public Object query(BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {- return miniClusterState.client.request(benchState.q, collection);+ return solrBenchState.client.request(benchState.q, collection);}}diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/SimpleSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/SimpleSearch.javaindex 2e888b8d878..9159b322197 100644--- a/solr/benchmark/src/java/org/apache/solr/bench/search/SimpleSearch.java+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/SimpleSearch.java@@ -19,7 +19,7 @@ package org.apache.solr.bench.search;import java.io.IOException;import java.util.concurrent.atomic.AtomicLong;import org.apache.solr.bench.BaseBenchState;-import org.apache.solr.bench.MiniClusterState;+import org.apache.solr.bench.SolrBenchState;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.QueryRequest;@@ -60,19 +60,18 @@ public class SimpleSearch {QueryRequest q = new QueryRequest(new SolrQuery("q", "id:0")); // no match is OK@Setup(Level.Trial)- public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState)- throws Exception {- miniClusterState.setUseHttp1(useHttp1);- miniClusterState.startMiniCluster(1);- miniClusterState.createCollection(COLLECTION, 1, 1);+ public void setupTrial(SolrBenchState solrBenchState) throws Exception {+ solrBenchState.setUseHttp1(useHttp1);+ solrBenchState.startSolr(1);+ solrBenchState.createCollection(COLLECTION, 1, 1);}@Setup(Level.Iteration)- public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState)+ public void setupIteration(SolrBenchState solrBenchState)throws SolrServerException, IOException {// Reload the collection/core to drop existing cachesCollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION);- miniClusterState.client.request(reload);+ solrBenchState.client.request(reload);total = new AtomicLong();err = new AtomicLong();@@ -140,16 +139,15 @@ public class SimpleSearch {* }</pre>*/@Benchmark- public Object query(- BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState, Blackhole bh)+ public Object query(BenchState benchState, SolrBenchState solrBenchState, Blackhole bh)throws SolrServerException, IOException {if (benchState.strict) {- return miniClusterState.client.request(benchState.q, COLLECTION);+ return solrBenchState.client.request(benchState.q, COLLECTION);}// non strict run ignores exceptionstry {- return miniClusterState.client.request(benchState.q, COLLECTION);+ return solrBenchState.client.request(benchState.q, COLLECTION);} catch (SolrServerException e) {bh.consume(e);benchState.err.getAndIncrement();diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.javaindex a9860763dbe..c354377eaf2 100644--- a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java@@ -24,8 +24,7 @@ import java.io.IOException;import java.util.ArrayList;import java.util.List;import org.apache.solr.bench.Docs;-import org.apache.solr.bench.MiniClusterState;-import org.apache.solr.bench.MiniClusterState.MiniClusterBenchState;+import org.apache.solr.bench.SolrBenchState;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.io.SolrClientCache;import org.apache.solr.client.solrj.io.Tuple;@@ -71,20 +70,20 @@ public class StreamingSearch {private HttpJettySolrClient httpJettySolrClient;@Setup(Level.Trial)- public void setup(MiniClusterBenchState miniClusterState) throws Exception {+ public void setup(SolrBenchState solrBenchState) throws Exception {- miniClusterState.startMiniCluster(3);- miniClusterState.createCollection(collection, 3, 1);+ solrBenchState.startSolr(3);+ solrBenchState.createCollection(collection, 3, 1);Docs docGen =docs().field("id", integers().incrementing()).field("text2_ts", strings().basicLatinAlphabet().multi(312).ofLengthBetween(30, 64)).field("text3_ts", strings().basicLatinAlphabet().multi(312).ofLengthBetween(30, 64)).field("int1_i_dv", integers().all());- miniClusterState.index(collection, docGen, docs);- miniClusterState.waitForMerges(collection);+ solrBenchState.index(collection, docGen, docs);+ solrBenchState.waitForMerges(collection);- zkHost = miniClusterState.zkHost;+ zkHost = solrBenchState.zkHost;params = new ModifiableSolrParams();params.set(CommonParams.Q, "*:*");@@ -94,7 +93,7 @@ public class StreamingSearch {}@Setup(Level.Iteration)- public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState)+ public void setupIteration(SolrBenchState solrBenchState)throws SolrServerException, IOException {SolrClientCache solrClientCache;// TODO tune params?@@ -115,8 +114,7 @@ public class StreamingSearch {}@Benchmark- public Object stream(- BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState)+ public Object stream(BenchState benchState, SolrBenchState solrBenchState)throws SolrServerException, IOException {CloudSolrStream stream = new CloudSolrStream(benchState.zkHost, collection, benchState.params);stream.setStreamContext(benchState.streamContext);diff --git a/solr/benchmark/src/test/org/apache/solr/bench/MiniClusterBenchStateTest.java b/solr/benchmark/src/test/org/apache/solr/bench/SolrBenchStateTest.javasimilarity index 84%rename from solr/benchmark/src/test/org/apache/solr/bench/MiniClusterBenchStateTest.javarename to solr/benchmark/src/test/org/apache/solr/bench/SolrBenchStateTest.javaindex 82708f6a34e..6aabc965720 100644--- a/solr/benchmark/src/test/org/apache/solr/bench/MiniClusterBenchStateTest.java+++ b/solr/benchmark/src/test/org/apache/solr/bench/SolrBenchStateTest.java@@ -42,18 +42,18 @@ import org.openjdk.jmh.runner.WorkloadParams;import org.openjdk.jmh.runner.options.TimeValue;@ThreadLeakLingering(linger = 10)-public class MiniClusterBenchStateTest extends SolrTestCaseJ4 {- private MiniClusterState.MiniClusterBenchState miniBenchState;+public class SolrBenchStateTest extends SolrTestCaseJ4 {+ private SolrBenchState solrBenchState;private BaseBenchState baseBenchState;private BenchmarkParams benchParams;@Test- public void testMiniClusterState() throws Exception {+ public void test() throws Exception {System.setProperty("workBaseDir", createTempDir("work").toString());System.setProperty("random.counts", "true");- miniBenchState = new MiniClusterState.MiniClusterBenchState();+ solrBenchState = new SolrBenchState();benchParams =new BenchmarkParams("benchmark",@@ -79,14 +79,14 @@ public class MiniClusterBenchStateTest extends SolrTestCaseJ4 {TimeValue.seconds(10));baseBenchState = new BaseBenchState();baseBenchState.doSetup(benchParams);- miniBenchState.doSetup(benchParams, baseBenchState);+ solrBenchState.doSetup(benchParams, baseBenchState);int nodeCount = 3;- miniBenchState.startMiniCluster(nodeCount);+ solrBenchState.startSolr(nodeCount);String collection = "collection1";int numShards = 1;int numReplicas = 1;- miniBenchState.createCollection(collection, numShards, numReplicas);+ solrBenchState.createCollection(collection, numShards, numReplicas);Docs docs =docs()@@ -110,13 +110,13 @@ public class MiniClusterBenchStateTest extends SolrTestCaseJ4 {int numDocs = 50;docs.preGenerate(numDocs);- miniBenchState.index(collection, docs, numDocs);+ solrBenchState.index(collection, docs, numDocs);- miniBenchState.forceMerge(collection, 15);+ solrBenchState.forceMerge(collection, 15);- ModifiableSolrParams params = MiniClusterState.params("q", "*:*");+ ModifiableSolrParams params = SolrBenchState.params("q", "*:*");QueryRequest queryRequest = new QueryRequest(params);- QueryResponse result = queryRequest.process(miniBenchState.client, collection);+ QueryResponse result = queryRequest.process(solrBenchState.client, collection);BaseBenchState.log("match all query result=" + result);@@ -127,7 +127,7 @@ public class MiniClusterBenchStateTest extends SolrTestCaseJ4 {public void after() throws Exception {BaseBenchState.doTearDown(benchParams);- miniBenchState.tearDown(benchParams);- miniBenchState.shutdownMiniCluster(benchParams, baseBenchState);+ solrBenchState.tearDown(benchParams);+ solrBenchState.shutdownSolr(benchParams, baseBenchState);}}