Apache Solr open-source search software
SOLR-14965: add overseer queue size metrics (#2040)
saatchibhalla19ed90337767
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex cb21fa6f7dc..abf6f1bcf09 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -189,6 +189,10 @@ Improvements* SOLR-15015 : Add interleaving algorithm parameter support in Learning To Rank (Alessandro Benedetti)+* SOLR-14965: metrics: Adds two metrics to the SolrCloud Overseer: solr_metrics_overseer_stateUpdateQueueSize+ and solr_metrics_overseer_collectionWorkQueueSize with corresponding entries in the the Prometheus exporter's+ default/stock configuration. (Saatchi Bhalla, Megan Carey, Andrzej Białecki, David Smiley)+Optimizations---------------------* SOLR-14975: Optimize CoreContainer.getAllCoreNames, getLoadedCoreNames and getCoreDescriptors. (Bruno Roustant)diff --git a/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml b/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xmlindex e20680c1669..bfcc54a4529 100644--- a/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml+++ b/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml@@ -280,6 +280,40 @@value : $value}</str>+ <!--+ overseer metrics+ -->+ <str>+ .metrics | to_entries | .[] | select(.key | startswith("solr.overseer")) as $object |+ $object.value as $value | $value | to_entries | .[] |+ select(.key | startswith("queue.") and endswith("collectionWorkQueueSize")) as $object |+ $object.value as $value |+ {+ name : "solr_metrics_overseer_collectionWorkQueueSize",+ type : "GAUGE",+ help : "See following URL: https://lucene.apache.org/solr/guide/metrics-reporting.html",+ label_names : [],+ label_values : [],+ value : $value+ }+ </str>+ <str>+ .metrics | to_entries | .[] | select(.key | startswith("solr.overseer")) as $object |+ $object.value as $value | $value | to_entries | .[] |+ select(.key | startswith("queue.") and endswith("stateUpdateQueueSize")) as $object |+ $object.value as $value |+ {+ name : "solr_metrics_overseer_stateUpdateQueueSize",+ type : "GAUGE",+ help : "See following URL: https://lucene.apache.org/solr/guide/metrics-reporting.html",+ label_names : [],+ label_values : [],+ value : $value+ }+ </str>+ <!--+ node metrics+ --><str>.metrics["solr.node"] | to_entries | .[] | select(.key | endswith(".clientErrors")) as $object |$object.key | split(".")[0] as $category |diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.javaindex fc0d0eb6f53..5cd553c47f5 100644--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java@@ -66,9 +66,12 @@ import org.apache.solr.common.util.Pair;import org.apache.solr.common.util.Utils;import org.apache.solr.core.CloudConfig;import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.SolrInfoBean;import org.apache.solr.handler.admin.CollectionsHandler;import org.apache.solr.handler.component.HttpShardHandler;import org.apache.solr.logging.MDCLoggingContext;+import org.apache.solr.metrics.SolrMetricProducer;+import org.apache.solr.metrics.SolrMetricsContext;import org.apache.solr.update.UpdateShardHandler;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;@@ -143,6 +146,9 @@ public class Overseer implements SolrCloseable {public static final int NUM_RESPONSES_TO_STORE = 10000;public static final String OVERSEER_ELECT = "/overseer_elect";+ private SolrMetricsContext solrMetricsContext;+ private volatile String metricTag = SolrMetricProducer.getUniqueMetricTag(this, null);+private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());enum LeaderStatus {DONT_KNOW, NO, YES}@@ -173,6 +179,8 @@ public class Overseer implements SolrCloseable {private final Stats zkStats;+ private SolrMetricsContext clusterStateUpdaterMetricContext;+private boolean isClosed = false;public ClusterStateUpdater(final ZkStateReader reader, final String myId, Stats zkStats) {@@ -185,6 +193,9 @@ public class Overseer implements SolrCloseable {this.completedMap = getCompletedMap(zkClient);this.myId = myId;this.reader = reader;++ clusterStateUpdaterMetricContext = solrMetricsContext.getChildContext(this);+ clusterStateUpdaterMetricContext.gauge(() -> stateUpdateQueue.getZkStats().getQueueLength(), true, "stateUpdateQueueSize", "queue" );}public Stats getStateUpdateQueueStats() {@@ -544,6 +555,7 @@ public class Overseer implements SolrCloseable {@Overridepublic void close() {this.isClosed = true;+ clusterStateUpdaterMetricContext.unregister();}}@@ -616,6 +628,8 @@ public class Overseer implements SolrCloseable {this.zkController = zkController;this.stats = new Stats();this.config = config;++ this.solrMetricsContext = new SolrMetricsContext(zkController.getCoreContainer().getMetricManager(), SolrInfoBean.Group.overseer.toString(), metricTag);}public synchronized void start(String id) {@@ -636,7 +650,7 @@ public class Overseer implements SolrCloseable {ThreadGroup ccTg = new ThreadGroup("Overseer collection creation process.");OverseerNodePrioritizer overseerPrioritizer = new OverseerNodePrioritizer(reader, getStateUpdateQueue(), adminPath, shardHandler.getShardHandlerFactory());- overseerCollectionConfigSetProcessor = new OverseerCollectionConfigSetProcessor(reader, id, shardHandler, adminPath, stats, Overseer.this, overseerPrioritizer);+ overseerCollectionConfigSetProcessor = new OverseerCollectionConfigSetProcessor(reader, id, shardHandler, adminPath, stats, Overseer.this, overseerPrioritizer, solrMetricsContext);ccThread = new OverseerThread(ccTg, overseerCollectionConfigSetProcessor, "OverseerCollectionConfigSetProcessor-" + id);ccThread.setDaemon(true);diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.javaindex 78ddc824f93..d3819d0cda6 100644--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java@@ -26,6 +26,7 @@ import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.handler.component.HttpShardHandler;import org.apache.solr.handler.component.HttpShardHandlerFactory;+import org.apache.solr.metrics.SolrMetricsContext;/*** An {@link OverseerTaskProcessor} that handles:@@ -37,7 +38,7 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessorpublic OverseerCollectionConfigSetProcessor(ZkStateReader zkStateReader, String myId,final HttpShardHandler shardHandler,String adminPath, Stats stats, Overseer overseer,- OverseerNodePrioritizer overseerNodePrioritizer) {+ OverseerNodePrioritizer overseerNodePrioritizer, SolrMetricsContext solrMetricsContext) {this(zkStateReader,myId,@@ -49,7 +50,8 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessoroverseer.getCollectionQueue(zkStateReader.getZkClient(), stats),Overseer.getRunningMap(zkStateReader.getZkClient()),Overseer.getCompletedMap(zkStateReader.getZkClient()),- Overseer.getFailureMap(zkStateReader.getZkClient())+ Overseer.getFailureMap(zkStateReader.getZkClient()),+ solrMetricsContext);}@@ -62,7 +64,7 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessorOverseerTaskQueue workQueue,DistributedMap runningMap,DistributedMap completedMap,- DistributedMap failureMap) {+ DistributedMap failureMap, SolrMetricsContext solrMetricsContext) {super(zkStateReader,myId,@@ -73,7 +75,8 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessorworkQueue,runningMap,completedMap,- failureMap);+ failureMap,+ solrMetricsContext);}private static OverseerMessageHandlerSelector getOverseerMessageHandlerSelector(diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.javaindex 2464a467c5e..e496d14045b 100644--- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java@@ -46,6 +46,7 @@ import org.apache.solr.common.util.StrUtils;import org.apache.solr.common.util.Utils;import org.apache.solr.logging.MDCLoggingContext;import org.apache.solr.common.util.SolrNamedThreadFactory;+import org.apache.solr.metrics.SolrMetricsContext;import org.apache.zookeeper.KeeperException;import org.apache.zookeeper.data.Stat;import org.slf4j.Logger;@@ -94,6 +95,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable {private boolean isClosed;private volatile Stats stats;+ private SolrMetricsContext overseerTaskProcessorMetricsContext;/*** Set of tasks that have been picked up for processing but not cleaned up from zk work-queue.@@ -134,13 +136,14 @@ public class OverseerTaskProcessor implements Runnable, Closeable {private String thisNode;public OverseerTaskProcessor(ZkStateReader zkStateReader, String myId,- Stats stats,- OverseerMessageHandlerSelector selector,- OverseerNodePrioritizer prioritizer,- OverseerTaskQueue workQueue,- DistributedMap runningMap,- DistributedMap completedMap,- DistributedMap failureMap) {+ Stats stats,+ OverseerMessageHandlerSelector selector,+ OverseerNodePrioritizer prioritizer,+ OverseerTaskQueue workQueue,+ DistributedMap runningMap,+ DistributedMap completedMap,+ DistributedMap failureMap,+ SolrMetricsContext solrMetricsContext) {this.zkStateReader = zkStateReader;this.myId = myId;this.stats = stats;@@ -154,6 +157,9 @@ public class OverseerTaskProcessor implements Runnable, Closeable {this.runningTasks = ConcurrentHashMap.newKeySet();this.completedTasks = new ConcurrentHashMap<>();thisNode = Utils.getMDCNode();++ overseerTaskProcessorMetricsContext = solrMetricsContext.getChildContext(this);+ overseerTaskProcessorMetricsContext.gauge(() -> workQueue.getZkStats().getQueueLength(), true, "collectionWorkQueueSize", "queue");}@Override@@ -386,6 +392,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable {public void close() {isClosed = true;+ overseerTaskProcessorMetricsContext.unregister();if (tpe != null) {if (!tpe.isShutdown()) {ExecutorUtil.shutdownAndAwaitTermination(tpe);diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.javaindex c811bf81020..5b18ab07f07 100644--- a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java@@ -63,6 +63,7 @@ import org.apache.solr.core.CoreContainer;import org.apache.solr.handler.component.HttpShardHandler;import org.apache.solr.handler.component.HttpShardHandlerFactory;import org.apache.solr.handler.component.ShardRequest;+import org.apache.solr.metrics.SolrMetricsContext;import org.apache.solr.update.UpdateShardHandler;import org.apache.solr.util.TimeOut;import org.apache.zookeeper.CreateMode;@@ -121,6 +122,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {private static CoreContainer coreContainerMock;private static UpdateShardHandler updateShardHandlerMock;private static HttpClient httpClientMock;+ private static SolrMetricsContext solrMetricsContextMock;private static ObjectCache objectCache;private Map<String, byte[]> zkClientData = new HashMap<>();@@ -144,8 +146,9 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {OverseerTaskQueue workQueue, DistributedMap runningMap,Overseer overseer,DistributedMap completedMap,- DistributedMap failureMap) {- super(zkStateReader, myId, shardHandlerFactory, adminPath, new Stats(), overseer, new OverseerNodePrioritizer(zkStateReader, overseer.getStateUpdateQueue(), adminPath, shardHandlerFactory), workQueue, runningMap, completedMap, failureMap);+ DistributedMap failureMap,+ SolrMetricsContext solrMetricsContext) {+ super(zkStateReader, myId, shardHandlerFactory, adminPath, new Stats(), overseer, new OverseerNodePrioritizer(zkStateReader, overseer.getStateUpdateQueue(), adminPath, shardHandlerFactory), workQueue, runningMap, completedMap, failureMap, solrMetricsContext);}@Override@@ -180,6 +183,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {coreContainerMock = mock(CoreContainer.class);updateShardHandlerMock = mock(UpdateShardHandler.class);httpClientMock = mock(HttpClient.class);+ solrMetricsContextMock = mock(SolrMetricsContext.class);}@AfterClass@@ -204,6 +208,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {coreContainerMock = null;updateShardHandlerMock = null;httpClientMock = null;+ solrMetricsContextMock = null;}@Before@@ -233,6 +238,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {reset(coreContainerMock);reset(updateShardHandlerMock);reset(httpClientMock);+ reset(solrMetricsContextMock);zkClientData.clear();collectionsSet.clear();@@ -493,7 +499,9 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {}}).when(distribStateManagerMock).makePath(anyString());zkClientData.put("/configs/myconfig", new byte[1]);-++ when(solrMetricsContextMock.getChildContext(any(Object.class))).thenReturn(solrMetricsContextMock);+return liveNodes;}@@ -728,7 +736,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {underTest = new OverseerCollectionConfigSetProcessorToBeTested(zkStateReaderMock,"1234", shardHandlerFactoryMock, ADMIN_PATH, workQueueMock, runningMapMock,- overseerMock, completedMapMock, failureMapMock);+ overseerMock, completedMapMock, failureMapMock, solrMetricsContextMock);if (log.isInfoEnabled()) {diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.javaindex 0fff3395e5a..91d0f95c51a 100644--- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java@@ -73,6 +73,7 @@ import org.apache.solr.core.PluginInfo;import org.apache.solr.core.SolrResourceLoader;import org.apache.solr.handler.component.HttpShardHandler;import org.apache.solr.handler.component.HttpShardHandlerFactory;+import org.apache.solr.metrics.SolrMetricManager;import org.apache.solr.update.UpdateShardHandler;import org.apache.solr.update.UpdateShardHandlerConfig;import org.apache.solr.util.TimeOut;@@ -1410,6 +1411,8 @@ public class OverseerTest extends SolrTestCaseJ4 {CoreContainer mockAlwaysUpCoreContainer = mock(CoreContainer.class,Mockito.withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS));+ SolrMetricManager mockMetricManager = mock(SolrMetricManager.class);+ when(mockAlwaysUpCoreContainer.getMetricManager()).thenReturn(mockMetricManager);when(mockAlwaysUpCoreContainer.isShutDown()).thenReturn(testDone); // Allow retry on session expirywhen(mockAlwaysUpCoreContainer.getResourceLoader()).thenReturn(new SolrResourceLoader(createTempDir()));ClusterSingletons singletons = new ClusterSingletons(() -> true, r -> r.run());diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.javaindex 603c4143544..f92039afb73 100644--- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java@@ -37,6 +37,7 @@ import org.apache.solr.common.util.Utils;import org.apache.solr.core.*;import org.apache.solr.handler.admin.CoreAdminHandler;import org.apache.solr.handler.component.HttpShardHandlerFactory;+import org.apache.solr.metrics.SolrMetricManager;import org.apache.solr.update.UpdateShardHandler;import org.apache.solr.update.UpdateShardHandlerConfig;import org.apache.solr.util.LogLevel;@@ -48,6 +49,7 @@ import org.junit.Test;import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;+import static org.mockito.Mockito.mock;@Slow@SolrTestCaseJ4.SuppressSSL@@ -333,6 +335,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 {private static class MockCoreContainer extends CoreContainer {UpdateShardHandler updateShardHandler = new UpdateShardHandler(UpdateShardHandlerConfig.DEFAULT);+ SolrMetricManager metricManager;public MockCoreContainer() {super(SolrXmlConfig.fromString(TEST_PATH(), "<solr/>"));@@ -340,6 +343,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 {httpShardHandlerFactory.init(new PluginInfo("shardHandlerFactory", Collections.emptyMap()));this.shardHandlerFactory = httpShardHandlerFactory;this.coreAdminHandler = new CoreAdminHandler();+ this.metricManager = mock(SolrMetricManager.class);}@Override@@ -356,5 +360,10 @@ public class ZkControllerTest extends SolrTestCaseJ4 {updateShardHandler.close();super.shutdown();}- }++ @Override+ public SolrMetricManager getMetricManager() {+ return metricManager;+ }+ }}diff --git a/solr/solr-ref-guide/src/metrics-reporting.adoc b/solr/solr-ref-guide/src/metrics-reporting.adocindex 96241658489..ccc033fce87 100644--- a/solr/solr-ref-guide/src/metrics-reporting.adoc+++ b/solr/solr-ref-guide/src/metrics-reporting.adoc@@ -57,6 +57,12 @@ This registry is returned at `solr.jvm` and includes the following information.* System properties such as Java information, various installation directory paths, ports, and similar information. You can control what appears here by modifying `solr.xml`.// TODO for 7.0 fix this+=== Overseer Registry++This registry is returned at `solr.overseer` when run in SolrCloud mode and includes the following information. When making requests with the <<Metrics API>>, you can specify `&group=overseer` to limit to only these metrics.++* size of the Overseer queues (collection work queue and cluster state update queue)+=== Node / CoreContainer RegistryThis registry is returned at `solr.node` and includes the following information. When making requests with the <<Metrics API>>, you can specify `&group=node` to limit to only these metrics.