Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-14965: add overseer queue size metrics (#2040)

saatchibhalla19ed90337767
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index cb21fa6f7dc..abf6f1bcf09 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -189,6 +189,10 @@ Improvements
* SOLR-15015 : Add interleaving algorithm parameter support in Learning To Rank (Alessandro Benedetti)
+* SOLR-14965: metrics: Adds two metrics to the SolrCloud Overseer: solr_metrics_overseer_stateUpdateQueueSize
+ and solr_metrics_overseer_collectionWorkQueueSize with corresponding entries in the the Prometheus exporter's
+ default/stock configuration. (Saatchi Bhalla, Megan Carey, Andrzej Białecki, David Smiley)
+
Optimizations
---------------------
* SOLR-14975: Optimize CoreContainer.getAllCoreNames, getLoadedCoreNames and getCoreDescriptors. (Bruno Roustant)
diff --git a/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml b/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml
index e20680c1669..bfcc54a4529 100644
--- a/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml
+++ b/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml
@@ -280,6 +280,40 @@
value : $value
}
</str>
+ <!--
+ overseer metrics
+ -->
+ <str>
+ .metrics | to_entries | .[] | select(.key | startswith("solr.overseer")) as $object |
+ $object.value as $value | $value | to_entries | .[] |
+ select(.key | startswith("queue.") and endswith("collectionWorkQueueSize")) as $object |
+ $object.value as $value |
+ {
+ name : "solr_metrics_overseer_collectionWorkQueueSize",
+ type : "GAUGE",
+ help : "See following URL: https://lucene.apache.org/solr/guide/metrics-reporting.html",
+ label_names : [],
+ label_values : [],
+ value : $value
+ }
+ </str>
+ <str>
+ .metrics | to_entries | .[] | select(.key | startswith("solr.overseer")) as $object |
+ $object.value as $value | $value | to_entries | .[] |
+ select(.key | startswith("queue.") and endswith("stateUpdateQueueSize")) as $object |
+ $object.value as $value |
+ {
+ name : "solr_metrics_overseer_stateUpdateQueueSize",
+ type : "GAUGE",
+ help : "See following URL: https://lucene.apache.org/solr/guide/metrics-reporting.html",
+ label_names : [],
+ label_values : [],
+ value : $value
+ }
+ </str>
+ <!--
+ node metrics
+ -->
<str>
.metrics["solr.node"] | to_entries | .[] | select(.key | endswith(".clientErrors")) as $object |
$object.key | split(".")[0] as $category |
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index fc0d0eb6f53..5cd553c47f5 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -66,9 +66,12 @@ import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CloudConfig;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.component.HttpShardHandler;
import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.metrics.SolrMetricProducer;
+import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.update.UpdateShardHandler;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -143,6 +146,9 @@ public class Overseer implements SolrCloseable {
public static final int NUM_RESPONSES_TO_STORE = 10000;
public static final String OVERSEER_ELECT = "/overseer_elect";
+ private SolrMetricsContext solrMetricsContext;
+ private volatile String metricTag = SolrMetricProducer.getUniqueMetricTag(this, null);
+
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
enum LeaderStatus {DONT_KNOW, NO, YES}
@@ -173,6 +179,8 @@ public class Overseer implements SolrCloseable {
private final Stats zkStats;
+ private SolrMetricsContext clusterStateUpdaterMetricContext;
+
private boolean isClosed = false;
public ClusterStateUpdater(final ZkStateReader reader, final String myId, Stats zkStats) {
@@ -185,6 +193,9 @@ public class Overseer implements SolrCloseable {
this.completedMap = getCompletedMap(zkClient);
this.myId = myId;
this.reader = reader;
+
+ clusterStateUpdaterMetricContext = solrMetricsContext.getChildContext(this);
+ clusterStateUpdaterMetricContext.gauge(() -> stateUpdateQueue.getZkStats().getQueueLength(), true, "stateUpdateQueueSize", "queue" );
}
public Stats getStateUpdateQueueStats() {
@@ -544,6 +555,7 @@ public class Overseer implements SolrCloseable {
@Override
public void close() {
this.isClosed = true;
+ clusterStateUpdaterMetricContext.unregister();
}
}
@@ -616,6 +628,8 @@ public class Overseer implements SolrCloseable {
this.zkController = zkController;
this.stats = new Stats();
this.config = config;
+
+ this.solrMetricsContext = new SolrMetricsContext(zkController.getCoreContainer().getMetricManager(), SolrInfoBean.Group.overseer.toString(), metricTag);
}
public synchronized void start(String id) {
@@ -636,7 +650,7 @@ public class Overseer implements SolrCloseable {
ThreadGroup ccTg = new ThreadGroup("Overseer collection creation process.");
OverseerNodePrioritizer overseerPrioritizer = new OverseerNodePrioritizer(reader, getStateUpdateQueue(), adminPath, shardHandler.getShardHandlerFactory());
- overseerCollectionConfigSetProcessor = new OverseerCollectionConfigSetProcessor(reader, id, shardHandler, adminPath, stats, Overseer.this, overseerPrioritizer);
+ overseerCollectionConfigSetProcessor = new OverseerCollectionConfigSetProcessor(reader, id, shardHandler, adminPath, stats, Overseer.this, overseerPrioritizer, solrMetricsContext);
ccThread = new OverseerThread(ccTg, overseerCollectionConfigSetProcessor, "OverseerCollectionConfigSetProcessor-" + id);
ccThread.setDaemon(true);
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java
index 78ddc824f93..d3819d0cda6 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java
@@ -26,6 +26,7 @@ import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.handler.component.HttpShardHandler;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
+import org.apache.solr.metrics.SolrMetricsContext;
/**
* An {@link OverseerTaskProcessor} that handles:
@@ -37,7 +38,7 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessor
public OverseerCollectionConfigSetProcessor(ZkStateReader zkStateReader, String myId,
final HttpShardHandler shardHandler,
String adminPath, Stats stats, Overseer overseer,
- OverseerNodePrioritizer overseerNodePrioritizer) {
+ OverseerNodePrioritizer overseerNodePrioritizer, SolrMetricsContext solrMetricsContext) {
this(
zkStateReader,
myId,
@@ -49,7 +50,8 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessor
overseer.getCollectionQueue(zkStateReader.getZkClient(), stats),
Overseer.getRunningMap(zkStateReader.getZkClient()),
Overseer.getCompletedMap(zkStateReader.getZkClient()),
- Overseer.getFailureMap(zkStateReader.getZkClient())
+ Overseer.getFailureMap(zkStateReader.getZkClient()),
+ solrMetricsContext
);
}
@@ -62,7 +64,7 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessor
OverseerTaskQueue workQueue,
DistributedMap runningMap,
DistributedMap completedMap,
- DistributedMap failureMap) {
+ DistributedMap failureMap, SolrMetricsContext solrMetricsContext) {
super(
zkStateReader,
myId,
@@ -73,7 +75,8 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessor
workQueue,
runningMap,
completedMap,
- failureMap);
+ failureMap,
+ solrMetricsContext);
}
private static OverseerMessageHandlerSelector getOverseerMessageHandlerSelector(
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
index 2464a467c5e..e496d14045b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
@@ -46,6 +46,7 @@ import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.common.util.SolrNamedThreadFactory;
+import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
@@ -94,6 +95,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
private boolean isClosed;
private volatile Stats stats;
+ private SolrMetricsContext overseerTaskProcessorMetricsContext;
/**
* Set of tasks that have been picked up for processing but not cleaned up from zk work-queue.
@@ -134,13 +136,14 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
private String thisNode;
public OverseerTaskProcessor(ZkStateReader zkStateReader, String myId,
- Stats stats,
- OverseerMessageHandlerSelector selector,
- OverseerNodePrioritizer prioritizer,
- OverseerTaskQueue workQueue,
- DistributedMap runningMap,
- DistributedMap completedMap,
- DistributedMap failureMap) {
+ Stats stats,
+ OverseerMessageHandlerSelector selector,
+ OverseerNodePrioritizer prioritizer,
+ OverseerTaskQueue workQueue,
+ DistributedMap runningMap,
+ DistributedMap completedMap,
+ DistributedMap failureMap,
+ SolrMetricsContext solrMetricsContext) {
this.zkStateReader = zkStateReader;
this.myId = myId;
this.stats = stats;
@@ -154,6 +157,9 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
this.runningTasks = ConcurrentHashMap.newKeySet();
this.completedTasks = new ConcurrentHashMap<>();
thisNode = Utils.getMDCNode();
+
+ overseerTaskProcessorMetricsContext = solrMetricsContext.getChildContext(this);
+ overseerTaskProcessorMetricsContext.gauge(() -> workQueue.getZkStats().getQueueLength(), true, "collectionWorkQueueSize", "queue");
}
@Override
@@ -386,6 +392,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
public void close() {
isClosed = true;
+ overseerTaskProcessorMetricsContext.unregister();
if (tpe != null) {
if (!tpe.isShutdown()) {
ExecutorUtil.shutdownAndAwaitTermination(tpe);
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java
index c811bf81020..5b18ab07f07 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java
@@ -63,6 +63,7 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.component.HttpShardHandler;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.update.UpdateShardHandler;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.CreateMode;
@@ -121,6 +122,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
private static CoreContainer coreContainerMock;
private static UpdateShardHandler updateShardHandlerMock;
private static HttpClient httpClientMock;
+ private static SolrMetricsContext solrMetricsContextMock;
private static ObjectCache objectCache;
private Map<String, byte[]> zkClientData = new HashMap<>();
@@ -144,8 +146,9 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
OverseerTaskQueue workQueue, DistributedMap runningMap,
Overseer overseer,
DistributedMap completedMap,
- DistributedMap failureMap) {
- super(zkStateReader, myId, shardHandlerFactory, adminPath, new Stats(), overseer, new OverseerNodePrioritizer(zkStateReader, overseer.getStateUpdateQueue(), adminPath, shardHandlerFactory), workQueue, runningMap, completedMap, failureMap);
+ DistributedMap failureMap,
+ SolrMetricsContext solrMetricsContext) {
+ super(zkStateReader, myId, shardHandlerFactory, adminPath, new Stats(), overseer, new OverseerNodePrioritizer(zkStateReader, overseer.getStateUpdateQueue(), adminPath, shardHandlerFactory), workQueue, runningMap, completedMap, failureMap, solrMetricsContext);
}
@Override
@@ -180,6 +183,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
coreContainerMock = mock(CoreContainer.class);
updateShardHandlerMock = mock(UpdateShardHandler.class);
httpClientMock = mock(HttpClient.class);
+ solrMetricsContextMock = mock(SolrMetricsContext.class);
}
@AfterClass
@@ -204,6 +208,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
coreContainerMock = null;
updateShardHandlerMock = null;
httpClientMock = null;
+ solrMetricsContextMock = null;
}
@Before
@@ -233,6 +238,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
reset(coreContainerMock);
reset(updateShardHandlerMock);
reset(httpClientMock);
+ reset(solrMetricsContextMock);
zkClientData.clear();
collectionsSet.clear();
@@ -493,7 +499,9 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
}}).when(distribStateManagerMock).makePath(anyString());
zkClientData.put("/configs/myconfig", new byte[1]);
-
+
+ when(solrMetricsContextMock.getChildContext(any(Object.class))).thenReturn(solrMetricsContextMock);
+
return liveNodes;
}
@@ -728,7 +736,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
underTest = new OverseerCollectionConfigSetProcessorToBeTested(zkStateReaderMock,
"1234", shardHandlerFactoryMock, ADMIN_PATH, workQueueMock, runningMapMock,
- overseerMock, completedMapMock, failureMapMock);
+ overseerMock, completedMapMock, failureMapMock, solrMetricsContextMock);
if (log.isInfoEnabled()) {
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
index 0fff3395e5a..91d0f95c51a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
@@ -73,6 +73,7 @@ import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.component.HttpShardHandler;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
+import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.update.UpdateShardHandler;
import org.apache.solr.update.UpdateShardHandlerConfig;
import org.apache.solr.util.TimeOut;
@@ -1410,6 +1411,8 @@ public class OverseerTest extends SolrTestCaseJ4 {
CoreContainer mockAlwaysUpCoreContainer = mock(CoreContainer.class,
Mockito.withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS));
+ SolrMetricManager mockMetricManager = mock(SolrMetricManager.class);
+ when(mockAlwaysUpCoreContainer.getMetricManager()).thenReturn(mockMetricManager);
when(mockAlwaysUpCoreContainer.isShutDown()).thenReturn(testDone); // Allow retry on session expiry
when(mockAlwaysUpCoreContainer.getResourceLoader()).thenReturn(new SolrResourceLoader(createTempDir()));
ClusterSingletons singletons = new ClusterSingletons(() -> true, r -> r.run());
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
index 603c4143544..f92039afb73 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
@@ -37,6 +37,7 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.core.*;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
+import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.update.UpdateShardHandler;
import org.apache.solr.update.UpdateShardHandlerConfig;
import org.apache.solr.util.LogLevel;
@@ -48,6 +49,7 @@ import org.junit.Test;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.mockito.Mockito.mock;
@Slow
@SolrTestCaseJ4.SuppressSSL
@@ -333,6 +335,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 {
private static class MockCoreContainer extends CoreContainer {
UpdateShardHandler updateShardHandler = new UpdateShardHandler(UpdateShardHandlerConfig.DEFAULT);
+ SolrMetricManager metricManager;
public MockCoreContainer() {
super(SolrXmlConfig.fromString(TEST_PATH(), "<solr/>"));
@@ -340,6 +343,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 {
httpShardHandlerFactory.init(new PluginInfo("shardHandlerFactory", Collections.emptyMap()));
this.shardHandlerFactory = httpShardHandlerFactory;
this.coreAdminHandler = new CoreAdminHandler();
+ this.metricManager = mock(SolrMetricManager.class);
}
@Override
@@ -356,5 +360,10 @@ public class ZkControllerTest extends SolrTestCaseJ4 {
updateShardHandler.close();
super.shutdown();
}
- }
+
+ @Override
+ public SolrMetricManager getMetricManager() {
+ return metricManager;
+ }
+ }
}
diff --git a/solr/solr-ref-guide/src/metrics-reporting.adoc b/solr/solr-ref-guide/src/metrics-reporting.adoc
index 96241658489..ccc033fce87 100644
--- a/solr/solr-ref-guide/src/metrics-reporting.adoc
+++ b/solr/solr-ref-guide/src/metrics-reporting.adoc
@@ -57,6 +57,12 @@ This registry is returned at `solr.jvm` and includes the following information.
* System properties such as Java information, various installation directory paths, ports, and similar information. You can control what appears here by modifying `solr.xml`.
// TODO for 7.0 fix this
+=== Overseer Registry
+
+This registry is returned at `solr.overseer` when run in SolrCloud mode and includes the following information. When making requests with the <<Metrics API>>, you can specify `&group=overseer` to limit to only these metrics.
+
+* size of the Overseer queues (collection work queue and cluster state update queue)
+
=== Node / CoreContainer Registry
This registry is returned at `solr.node` and includes the following information. When making requests with the <<Metrics API>>, you can specify `&group=node` to limit to only these metrics.