Apache Solr open-source search software
SOLR-17853: Migrate Plugins / Stats UI tab with Prometheus metrics (#3623)
Matthew Biscocho94e9dfe356ca
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.javadeleted file mode 100644index d795d3d2f2f..00000000000--- a/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java+++ /dev/null@@ -1,85 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.handler.admin;--import static org.apache.solr.common.params.CommonParams.NAME;--import java.util.Map;-import org.apache.solr.common.params.SolrParams;-import org.apache.solr.common.util.SimpleOrderedMap;-import org.apache.solr.core.SolrCore;-import org.apache.solr.core.SolrInfoBean;-import org.apache.solr.handler.RequestHandlerBase;-import org.apache.solr.request.SolrQueryRequest;-import org.apache.solr.response.SolrQueryResponse;-import org.apache.solr.security.AuthorizationContext;--/**- * @since solr 1.2- */-public class PluginInfoHandler extends RequestHandlerBase {- @Override- public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {- SolrParams params = req.getParams();-- boolean stats = params.getBool("stats", false);- rsp.add("plugins", getSolrInfoBeans(req.getCore(), stats));- rsp.setHttpCaching(false);- }-- private static SimpleOrderedMap<Object> getSolrInfoBeans(SolrCore core, boolean stats) {- SimpleOrderedMap<Object> list = new SimpleOrderedMap<>();- for (SolrInfoBean.Category cat : SolrInfoBean.Category.values()) {- SimpleOrderedMap<Object> category = new SimpleOrderedMap<>();- list.add(cat.name(), category);- Map<String, SolrInfoBean> reg = core.getInfoRegistry();- for (Map.Entry<String, SolrInfoBean> entry : reg.entrySet()) {- SolrInfoBean m = entry.getValue();- if (m.getCategory() != cat) continue;-- String na = "Not Declared";- SimpleOrderedMap<Object> info = new SimpleOrderedMap<>();- category.add(entry.getKey(), info);-- info.add(NAME, (m.getName() != null ? m.getName() : na));- info.add("description", (m.getDescription() != null ? m.getDescription() : na));-- if (stats && m.getSolrMetricsContext() != null) {- info.add("stats", m.getSolrMetricsContext().getMetricsSnapshot());- }- }- }- return list;- }-- //////////////////////// SolrInfoMBeans methods //////////////////////-- @Override- public String getDescription() {- return "Registry";- }-- @Override- public Category getCategory() {- return Category.ADMIN;- }-- @Override- public Name getPermissionName(AuthorizationContext request) {- return Name.METRICS_READ_PERM;- }-}diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.javadeleted file mode 100644index a9ba9d0bea2..00000000000--- a/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java+++ /dev/null@@ -1,292 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.handler.admin;--import java.io.StringReader;-import java.text.NumberFormat;-import java.util.HashSet;-import java.util.Locale;-import java.util.Map;-import java.util.Map.Entry;-import java.util.Set;-import org.apache.solr.client.solrj.impl.XMLResponseParser;-import org.apache.solr.common.SolrException;-import org.apache.solr.common.SolrException.ErrorCode;-import org.apache.solr.common.util.ContentStream;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.common.util.SimpleOrderedMap;-import org.apache.solr.common.util.StrUtils;-import org.apache.solr.core.SolrInfoBean;-import org.apache.solr.handler.RequestHandlerBase;-import org.apache.solr.request.SolrQueryRequest;-import org.apache.solr.response.JavaBinResponseWriter;-import org.apache.solr.response.SolrQueryResponse;-import org.apache.solr.security.AuthorizationContext;--/** A request handler that provides info about all registered SolrInfoMBeans. */-public class SolrInfoMBeanHandler extends RequestHandlerBase {-- /**- * Take an array of any type and generate a Set containing the toString. Set is guarantee to never- * be null (but may be empty)- */- private Set<String> arrayToSet(Object[] arr) {- HashSet<String> r = new HashSet<>();- if (null == arr) return r;- for (Object o : arr) {- if (null != o) r.add(o.toString());- }- return r;- }-- @Override- @SuppressWarnings("unchecked")- public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {- NamedList<NamedList<NamedList<Object>>> cats = getMBeanInfo(req);- if (req.getParams().getBool("diff", false)) {- ContentStream body = null;- try {- body = req.getContentStreams().iterator().next();- } catch (Exception ex) {- throw new SolrException(ErrorCode.BAD_REQUEST, "missing content-stream for diff");- }- final String content = StrUtils.stringFromReader(body.getReader());-- NamedList<NamedList<NamedList<Object>>> ref = fromXML(content);-- // Normalize the output- SolrQueryResponse wrap = new SolrQueryResponse();- wrap.add("solr-mbeans", cats);- cats =- (NamedList<NamedList<NamedList<Object>>>)- JavaBinResponseWriter.getParsedResponse(req, wrap).get("solr-mbeans");-- // Get rid of irrelevant things- normalize(ref);- normalize(cats);-- // Only the changes- boolean showAll = req.getParams().getBool("all", false);- rsp.add("solr-mbeans", getDiff(ref, cats, showAll));- } else {- rsp.add("solr-mbeans", cats);- }- rsp.setHttpCaching(false); // never cache, no matter what init config looks like- }-- @SuppressWarnings("unchecked")- static NamedList<NamedList<NamedList<Object>>> fromXML(String content) {- int idx = content.indexOf("<response>");- if (idx < 0) {- throw new SolrException(ErrorCode.BAD_REQUEST, "Body does not appear to be an XML response");- }-- try {- XMLResponseParser parser = new XMLResponseParser();- return (NamedList<NamedList<NamedList<Object>>>)- parser.processResponse(new StringReader(content)).get("solr-mbeans");- } catch (Exception ex) {- throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to read original XML", ex);- }- }-- protected NamedList<NamedList<NamedList<Object>>> getMBeanInfo(SolrQueryRequest req) {-- NamedList<NamedList<NamedList<Object>>> cats = new NamedList<>();-- String[] requestedCats = req.getParams().getParams("cat");- if (null == requestedCats || 0 == requestedCats.length) {- for (SolrInfoBean.Category cat : SolrInfoBean.Category.values()) {- cats.add(cat.name(), new SimpleOrderedMap<>());- }- } else {- for (String catName : requestedCats) {- cats.add(catName, new SimpleOrderedMap<>());- }- }-- Set<String> requestedKeys = arrayToSet(req.getParams().getParams("key"));-- Map<String, SolrInfoBean> reg = req.getCore().getInfoRegistry();- for (Map.Entry<String, SolrInfoBean> entry : reg.entrySet()) {- addMBean(req, cats, requestedKeys, entry.getKey(), entry.getValue());- }-- for (SolrInfoBean infoMBean : req.getCoreContainer().getResourceLoader().getInfoMBeans()) {- addMBean(req, cats, requestedKeys, infoMBean.getName(), infoMBean);- }- return cats;- }-- private void addMBean(- SolrQueryRequest req,- NamedList<NamedList<NamedList<Object>>> cats,- Set<String> requestedKeys,- String key,- SolrInfoBean m) {- if (!(requestedKeys.isEmpty() || requestedKeys.contains(key))) return;- NamedList<NamedList<Object>> catInfo = cats.get(m.getCategory().name());- if (null == catInfo) return;- NamedList<Object> mBeanInfo = new SimpleOrderedMap<>();- mBeanInfo.add("class", m.getName());- mBeanInfo.add("description", m.getDescription());-- if (req.getParams().getFieldBool(key, "stats", false) && m.getSolrMetricsContext() != null)- mBeanInfo.add("stats", m.getSolrMetricsContext().getMetricsSnapshot());-- catInfo.add(key, mBeanInfo);- }-- protected NamedList<NamedList<NamedList<Object>>> getDiff(- NamedList<NamedList<NamedList<Object>>> ref,- NamedList<NamedList<NamedList<Object>>> now,- boolean includeAll) {-- NamedList<NamedList<NamedList<Object>>> changed = new NamedList<>();-- // Cycle through each category- for (Entry<String, NamedList<NamedList<Object>>> catEntry : ref) {- String category = catEntry.getKey();- NamedList<NamedList<Object>> ref_cat = catEntry.getValue();- NamedList<NamedList<Object>> now_cat = now.get(category);- if (now_cat != null) {- String ref_txt = ref_cat + "";- String now_txt = now_cat + "";- if (!ref_txt.equals(now_txt)) {- // Something in the category changed- // Now iterate the real beans-- NamedList<NamedList<Object>> cat = new SimpleOrderedMap<>();- for (Entry<String, NamedList<Object>> beanEntry : ref_cat) {- String name = beanEntry.getKey();- NamedList<Object> ref_bean = beanEntry.getValue();- NamedList<Object> now_bean = now_cat.get(name);-- ref_txt = ref_bean + "";- now_txt = now_bean + "";- if (!ref_txt.equals(now_txt)) {- // System.out.println( "----" );- // System.out.println( category +" : " + name );- // System.out.println( "REF: " + ref_txt );- // System.out.println( "NOW: " + now_txt );-- // Calculate the differences- NamedList<Object> diff = diffNamedList(ref_bean, now_bean);- diff.add("_changed_", true); // flag the changed thing- cat.add(name, diff);- } else if (includeAll) {- cat.add(name, ref_bean);- }- }- if (cat.size() > 0) {- changed.add(category, cat);- }- } else if (includeAll) {- changed.add(category, ref_cat);- }- }- }- return changed;- }-- public NamedList<Object> diffNamedList(NamedList<?> ref, NamedList<?> now) {- NamedList<Object> out = new SimpleOrderedMap<>();- ref.forEach(- (name, r) -> {- Object n = now.get(name);- if (n == null) {- if (r != null) {- out.add("REMOVE " + name, r);- now.remove(name);- }- } else if (r != null) {- out.add(name, diffObject(r, n));- now.remove(name);- }- });-- now.forEach(- (name, v) -> {- if (v != null) {- out.add("ADD " + name, v);- }- });-- return out;- }-- @SuppressWarnings("unchecked")- public Object diffObject(Object ref, Object now) {- if (now instanceof Map) {- now = new NamedList<>((Map<String, ?>) now);- }- if (ref instanceof NamedList) {- return diffNamedList((NamedList<?>) ref, (NamedList<?>) now);- }- if (ref.equals(now)) {- return ref;- }- StringBuilder str = new StringBuilder();- str.append("Was: ").append(ref).append(", Now: ").append(now);-- if (ref instanceof Number) {- NumberFormat nf = NumberFormat.getIntegerInstance(Locale.ROOT);- if ((ref instanceof Double) || (ref instanceof Float)) {- nf = NumberFormat.getInstance(Locale.ROOT);- }- double dref = ((Number) ref).doubleValue();- double dnow = ((Number) now).doubleValue();- double diff = Double.NaN;- if (Double.isNaN(dref)) {- diff = dnow;- } else if (Double.isNaN(dnow)) {- diff = dref;- } else {- diff = dnow - dref;- }- str.append(", Delta: ").append(nf.format(diff));- }- return str.toString();- }-- /** The 'avgRequestsPerSecond' field will make everything look like it changed */- public void normalize(NamedList<?> input) {- input.remove("avgRequestsPerSecond");- for (Entry<String, ?> entry : input) {- Object v = entry.getValue();- if (v instanceof NamedList) {- // edit in place so we don't need to return it- normalize((NamedList<?>) v);- }- }- }-- @Override- public String getDescription() {- return "Get Info (and statistics) for registered SolrInfoMBeans";- }-- @Override- public Category getCategory() {- return Category.ADMIN;- }-- @Override- public Name getPermissionName(AuthorizationContext request) {- return Name.METRICS_READ_PERM;- }-}diff --git a/solr/core/src/resources/ImplicitPlugins.json b/solr/core/src/resources/ImplicitPlugins.jsonindex c4da631c17a..4154e70ded9 100644--- a/solr/core/src/resources/ImplicitPlugins.json+++ b/solr/core/src/resources/ImplicitPlugins.json@@ -77,13 +77,6 @@"class": "solr.SystemInfoHandler","useParams":"_ADMIN_SYSTEM"},- "/admin/mbeans": {- "class": "solr.SolrInfoMBeanHandler",- "useParams":"_ADMIN_MBEANS"- },- "/admin/plugins": {- "class": "solr.PluginInfoHandler"- },"/admin/file": {"class": "solr.ShowFileRequestHandler","useParams":"_ADMIN_FILE"diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.javaindex a5642b81942..09c700cfee1 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java@@ -44,7 +44,6 @@ import org.apache.solr.client.solrj.impl.CloudLegacySolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.client.solrj.response.CollectionAdminResponse;-import org.apache.solr.client.solrj.response.QueryResponse;import org.apache.solr.common.SolrDocument;import org.apache.solr.common.SolrDocumentList;import org.apache.solr.common.SolrException;@@ -55,6 +54,7 @@ import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.TimeSource;+import org.apache.solr.core.CoreContainer;import org.apache.solr.core.CoreDescriptor;import org.apache.solr.core.SolrCore;import org.apache.solr.embedded.JettySolrRunner;@@ -310,17 +310,26 @@ public class TestPullReplica extends SolrCloudTestCase {: s.getReplicas(EnumSet.of(Replica.Type.PULL));waitForNumDocsInAllReplicas(numDocs, pullReplicas);- for (Replica r : pullReplicas) {- try (SolrClient pullReplicaClient = getHttpSolrClient(r)) {- SolrQuery req = new SolrQuery("qt", "/admin/plugins", "stats", "true");- QueryResponse statsResponse = pullReplicaClient.query(req);- // The adds gauge metric should be null for pull replicas since they don't process adds- assertNull(- "Replicas shouldn't process the add document request: " + statsResponse,- ((Map<String, Object>)- (statsResponse.getResponse())- ._get(List.of("plugins", "UPDATE", "updateHandler", "stats"), null))- .get("UPDATE.updateHandler.adds"));+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {+ CoreContainer cc = jetty.getCoreContainer();+ for (String coreName : cc.getAllCoreNames()) {+ try (SolrCore core = cc.getCore(coreName)) {+ var addOpsDatapoint =+ org.apache.solr.util.SolrMetricTestUtils.getCounterDatapoint(+ core,+ "solr_core_update_committed_ops",+ org.apache.solr.util.SolrMetricTestUtils.newCloudLabelsBuilder(core)+ .label("category", "UPDATE")+ .label("ops", "adds")+ .build());+ // The adds gauge metric should be null for pull replicas since they don't process adds+ if ((core.getCoreDescriptor().getCloudDescriptor().getReplicaType()+ == Replica.Type.PULL)) {+ assertNull(addOpsDatapoint);+ } else {+ assertNotNull(addOpsDatapoint);+ }+ }}}diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaWithAuth.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaWithAuth.javaindex 5e2b2500382..d0e1bb78f20 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaWithAuth.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaWithAuth.java@@ -24,7 +24,6 @@ import static org.apache.solr.cloud.TestPullReplica.waitForNumDocsInAllReplicas;import java.io.IOException;import java.util.EnumSet;import java.util.List;-import java.util.Map;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrQuery;import org.apache.solr.client.solrj.SolrRequest;@@ -40,8 +39,11 @@ import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;-import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.SolrCore;+import org.apache.solr.embedded.JettySolrRunner;import org.apache.solr.util.SecurityJson;+import org.apache.solr.util.SolrMetricTestUtils;import org.junit.BeforeClass;import org.junit.Test;@@ -68,9 +70,6 @@ public class TestPullReplicaWithAuth extends SolrCloudTestCase {}@Test- // NOCOMMIT: This test is broken from OTEL migration and the /admin/plugins endpoint. Placing- // BadApple test but this must be fixed before this feature gets merged to a release branch- @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458")public void testPKIAuthWorksForPullReplication() throws Exception {int numPullReplicas = 2;withBasicAuth(@@ -104,19 +103,36 @@ public class TestPullReplicaWithAuth extends SolrCloudTestCase {waitForNumDocsInAllReplicas(numDocs, pullReplicas, "*:*", SecurityJson.USER, SecurityJson.PASS);- for (Replica r : pullReplicas) {- try (SolrClient pullReplicaClient = getHttpSolrClient(r)) {- QueryResponse statsResponse =- queryWithBasicAuth(- pullReplicaClient, new SolrQuery("qt", "/admin/plugins", "stats", "true"));- // the 'adds' metric is a gauge, which is null for PULL replicas- assertNull(- "Replicas shouldn't process the add document request: " + statsResponse,- getUpdateHandlerMetric(statsResponse, "UPDATE.updateHandler.adds"));- assertEquals(- "Replicas shouldn't process the add document request: " + statsResponse,- 0L,- getUpdateHandlerMetric(statsResponse, "UPDATE.updateHandler.cumulativeAdds.count"));+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {+ CoreContainer cc = jetty.getCoreContainer();+ for (String coreName : cc.getAllCoreNames()) {+ try (SolrCore core = cc.getCore(coreName)) {+ var addOpsDatapoint =+ org.apache.solr.util.SolrMetricTestUtils.getCounterDatapoint(+ core,+ "solr_core_update_committed_ops",+ org.apache.solr.util.SolrMetricTestUtils.newCloudLabelsBuilder(core)+ .label("category", "UPDATE")+ .label("ops", "adds")+ .build());+ var cumulativeAddsDatapoint =+ SolrMetricTestUtils.getGaugeDatapoint(+ core,+ "solr_core_update_cumulative_ops",+ SolrMetricTestUtils.newCloudLabelsBuilder(core)+ .label("category", "UPDATE")+ .label("ops", "adds")+ .build());+ // The adds gauge metric should be null for pull replicas since they don't process adds+ if ((core.getCoreDescriptor().getCloudDescriptor().getReplicaType()+ == Replica.Type.PULL)) {+ assertNull(addOpsDatapoint);+ assertNull(cumulativeAddsDatapoint);+ } else {+ assertNotNull(addOpsDatapoint);+ assertNotNull(cumulativeAddsDatapoint);+ }+ }}}}@@ -154,12 +170,4 @@ public class TestPullReplicaWithAuth extends SolrCloudTestCase {.process(cluster.getSolrClient());waitForDeletion(collectionName);}-- @SuppressWarnings("unchecked")- private Object getUpdateHandlerMetric(QueryResponse statsResponse, String metric) {- NamedList<Object> entries = statsResponse.getResponse();- return ((Map<String, Object>)- entries._get(List.of("plugins", "UPDATE", "updateHandler", "stats"), null))- .get(metric);- }}diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.javaindex f89aa37f90b..eea30196557 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java@@ -257,9 +257,6 @@ public class TestTlogReplica extends SolrCloudTestCase {}@SuppressWarnings("unchecked")- // NOCOMMIT: This test is broken from OTEL migration and the /admin/plugins endpoint. Placing- // BadApple test but this must be fixed before this feature gets merged to a release branch- @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458")public void testAddDocs() throws Exception {int numTlogReplicas = 1 + random().nextInt(3);DocCollection docCollection = createAndWaitForCollection(1, 0, numTlogReplicas, 0);@@ -283,22 +280,29 @@ public class TestTlogReplica extends SolrCloudTestCase {"Replica " + r.getName() + " not up to date after 10 seconds",1,tlogReplicaClient.query(new SolrQuery("*:*")).getResults().getNumFound());- // Append replicas process all updates- SolrQuery req =- new SolrQuery(- "qt", "/admin/plugins",- "stats", "true");- QueryResponse statsResponse = tlogReplicaClient.query(req);- NamedList<Object> entries = (statsResponse.getResponse());- assertEquals(- "Append replicas should recive all updates. Replica: "- + r- + ", response: "- + statsResponse,- 1L,- ((Map<String, Object>)- entries._get(List.of("plugins", "UPDATE", "updateHandler", "stats"), null))- .get("UPDATE.updateHandler.cumulativeAdds.count"));+ JettySolrRunner jetty =+ cluster.getJettySolrRunners().stream()+ .filter(j -> j.getBaseUrl().toString().equals(r.getBaseUrl()))+ .findFirst()+ .orElse(null);+ assertNotNull("Could not find jetty for replica " + r, jetty);++ try (SolrCore core = jetty.getCoreContainer().getCore(r.getCoreName())) {+ var cumulativeAddsDatapoint =+ SolrMetricTestUtils.getGaugeDatapoint(+ core,+ "solr_core_update_cumulative_ops",+ SolrMetricTestUtils.newCloudLabelsBuilder(core)+ .label("category", "UPDATE")+ .label("ops", "adds")+ .build());+ assertNotNull("Could not find cumulative adds metric", cumulativeAddsDatapoint);+ assertEquals(+ "Append replicas should receive all updates. Replica: " + r,+ 1.0,+ cumulativeAddsDatapoint.getValue(),+ 0.0);+ }break;} catch (AssertionError e) {if (t.hasTimedOut()) {diff --git a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.javaindex 5af9bfe0436..3e2d3589958 100644--- a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java+++ b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java@@ -97,12 +97,8 @@ public class SolrCoreTest extends SolrTestCaseJ4 {++ihCount;assertEquals(pathToClassMap.get("/admin/luke"), "solr.LukeRequestHandler");++ihCount;- assertEquals(pathToClassMap.get("/admin/mbeans"), "solr.SolrInfoMBeanHandler");- ++ihCount;assertEquals(pathToClassMap.get("/admin/ping"), "solr.PingRequestHandler");++ihCount;- assertEquals(pathToClassMap.get("/admin/plugins"), "solr.PluginInfoHandler");- ++ihCount;assertEquals(pathToClassMap.get("/admin/segments"), "solr.SegmentsInfoRequestHandler");++ihCount;assertEquals(pathToClassMap.get("/admin/system"), "solr.SystemInfoHandler");diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.javaindex 0eeb3b463db..c7b8219bb33 100644--- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java+++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java@@ -153,8 +153,6 @@ public class TestSolrConfigHandler extends RestTestBase {MapWriter confMap = getRespMap("/config", harness);assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/luke"), null));assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/system"), null));- assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/mbeans"), null));- assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/plugins"), null));assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/file"), null));assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/ping"), null));diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.javadeleted file mode 100644index be55062f79f..00000000000--- a/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java+++ /dev/null@@ -1,240 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.handler.admin;--import io.opentelemetry.api.common.Attributes;-import java.lang.invoke.MethodHandles;-import java.util.ArrayList;-import java.util.Collections;-import java.util.List;-import java.util.concurrent.CountDownLatch;-import java.util.concurrent.TimeUnit;-import java.util.regex.Matcher;-import java.util.regex.Pattern;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.solr.SolrTestCaseJ4;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.util.ContentStream;-import org.apache.solr.common.util.ContentStreamBase;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.core.SolrInfoBean;-import org.apache.solr.metrics.SolrMetricsContext;-import org.apache.solr.request.LocalSolrQueryRequest;-import org.junit.BeforeClass;-import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--// NOCOMMIT: Test fails because of the /admin/mbeans endpoint. Need to create a bridge or shim after-// migrating all metrics to support this.[email protected](bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458")-public class MBeansHandlerTest extends SolrTestCaseJ4 {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- @BeforeClass- public static void beforeClass() throws Exception {- initCore("solrconfig.xml", "schema.xml");- }-- @Test- public void testDiff() throws Exception {- String xml =- h.query(req(CommonParams.QT, "/admin/mbeans", "stats", "true", CommonParams.WT, "xml"));- List<ContentStream> streams = new ArrayList<>();- streams.add(new ContentStreamBase.StringStream(xml));-- LocalSolrQueryRequest req =- lrf.makeRequest(- CommonParams.QT,- "/admin/mbeans",- "stats",- "true",- CommonParams.WT,- "xml",- "diff",- "true");- req.setContentStreams(streams);-- xml = h.query(req);- NamedList<NamedList<NamedList<Object>>> diff = SolrInfoMBeanHandler.fromXML(xml);-- // The stats bean for SolrInfoMBeanHandler- NamedList<?> stats = (NamedList<?>) diff.get("ADMIN").get("/admin/mbeans").get("stats");-- // System.out.println("stats:"+stats);- Pattern p =- Pattern.compile("Was: (?<was>[0-9]+), Now: (?<now>[0-9]+), Delta: (?<delta>[0-9]+)");- String response = stats.get("ADMIN./admin/mbeans.requests").toString();- Matcher m = p.matcher(response);- if (!m.matches()) {- fail("Response did not match pattern: " + response);- }-- assertEquals(1, Integer.parseInt(m.group("delta")));- int was = Integer.parseInt(m.group("was"));- int now = Integer.parseInt(m.group("now"));- assertEquals(1, now - was);-- xml =- h.query(- req(- CommonParams.QT,- "/admin/mbeans",- "stats",- "true",- "key",- "org.apache.solr.handler.admin.CollectionsHandler"));- NamedList<NamedList<NamedList<Object>>> nl = SolrInfoMBeanHandler.fromXML(xml);- assertNotNull(nl.get("ADMIN").get("org.apache.solr.handler.admin.CollectionsHandler"));- }-- @Test- public void testAddedMBeanDiff() throws Exception {- String xml =- h.query(req(CommonParams.QT, "/admin/mbeans", "stats", "true", CommonParams.WT, "xml"));-- // Artificially convert a long value to a null, to trigger the ADD case in- // SolrInfoMBeanHandler.diffObject()- xml =- xml.replaceFirst(- "<long\\s+(name\\s*=\\s*\"ADMIN./admin/mbeans.totalTime\"\\s*)>[^<]*</long>",- "<null $1/>");-- LocalSolrQueryRequest req =- lrf.makeRequest(- CommonParams.QT,- "/admin/mbeans",- "stats",- "true",- CommonParams.WT,- "xml",- "diff",- "true");- req.setContentStreams(Collections.singletonList(new ContentStreamBase.StringStream(xml)));- xml = h.query(req);-- NamedList<NamedList<NamedList<Object>>> nl = SolrInfoMBeanHandler.fromXML(xml);- assertNotNull(- ((NamedList<?>) nl.get("ADMIN").get("/admin/mbeans").get("stats"))- .get("ADD ADMIN./admin/mbeans.totalTime"));- }-- @Test- public void testXMLDiffWithExternalEntity() {- String file = getFile("mailing_lists.pdf").toUri().toASCIIString();- String xml =- "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"- + "<!DOCTYPE foo [<!ENTITY bar SYSTEM \""- + file- + "\">]>\n"- + "<response>\n"- + "&bar;"- + "<lst name=\"responseHeader\"><int name=\"status\">0</int><int name=\"QTime\">31</int></lst><lst name=\"solr-mbeans\"></lst>\n"- + "</response>";-- NamedList<NamedList<NamedList<Object>>> nl = SolrInfoMBeanHandler.fromXML(xml);-- assertTrue("external entity ignored properly", true);- }-- boolean runSnapshots;-- @Test- public void testMetricsSnapshot() throws Exception {- final CountDownLatch counter = new CountDownLatch(500);- SolrInfoBean bean =- new SolrInfoBean() {- SolrMetricsContext solrMetricsContext;-- @Override- public String getName() {- return "foo";- }-- @Override- public String getDescription() {- return "foo";- }-- @Override- public Category getCategory() {- return Category.ADMIN;- }-- @Override- public void initializeMetrics(- SolrMetricsContext parentContext, Attributes attributes, String scope) {- this.solrMetricsContext = parentContext.getChildContext(this);- }-- @Override- public SolrMetricsContext getSolrMetricsContext() {- return solrMetricsContext;- }- };- bean.initializeMetrics(- new SolrMetricsContext(- h.getCoreContainer().getMetricManager(), "testMetricsSnapshot", "foobar"),- Attributes.empty(),- "foo");- runSnapshots = true;- Thread modifier =- new Thread(- () -> {- int i = 0;- while (runSnapshots) {- bean.getSolrMetricsContext().registerMetricName("name-" + i++);- try {- Thread.sleep(31);- } catch (InterruptedException e) {- runSnapshots = false;- break;- }- }- });- Thread reader =- new Thread(- () -> {- while (runSnapshots) {- try {- bean.getSolrMetricsContext().getMetricsSnapshot();- } catch (Exception e) {- runSnapshots = false;- log.error("Exception getting metrics snapshot", e);- fail("Exception getting metrics snapshot: " + e);- }- try {- Thread.sleep(53);- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- log.error("interrupted", e);- runSnapshots = false;- break;- }- counter.countDown();- }- });- modifier.start();- reader.start();- counter.await(30, TimeUnit.SECONDS);- runSnapshots = false;- bean.close();-- reader.join();- modifier.join();- }-}diff --git a/solr/server/etc/security.policy b/solr/server/etc/security.policyindex 44514befe56..a779dfbc7a2 100644--- a/solr/server/etc/security.policy+++ b/solr/server/etc/security.policy@@ -132,7 +132,6 @@ grant {permission javax.management.MBeanServerPermission "releaseMBeanServer";permission javax.management.MBeanTrustPermission "register";-// needed by crossdcpermission javax.security.auth.AuthPermission "getLoginConfiguration";permission javax.security.auth.AuthPermission "setLoginConfiguration";@@ -219,6 +218,7 @@ grant {permission java.net.SocketPermission "${solr.internal.network.permission}", "accept,listen,connect,resolve";};+// Permissions for OTEL Runtime Java 17 telemetry and metricsgrant {- permission jdk.jfr.FlightRecorderPermission "accessFlightRecorder";+ permission jdk.jfr.FlightRecorderPermission "accessFlightRecorder";};diff --git a/solr/webapp/web/css/angular/plugins.css b/solr/webapp/web/css/angular/plugins.cssindex e4398bda20c..775287184da 100644--- a/solr/webapp/web/css/angular/plugins.css+++ b/solr/webapp/web/css/angular/plugins.css@@ -218,3 +218,18 @@ limitations under the License.{background-image: url( ../../img/ico/new-text.png );}++.prometheus-metric {+ font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;+ font-size: 11px;+ color: #0066cc;+ background-color: #f5f5f5;+ padding: 2px 4px;+ border-radius: 3px;+}++#content #plugins .stats dl dd.prometheus-metric {+ word-break: break-all;+ white-space: pre-wrap;+}+diff --git a/solr/webapp/web/js/angular/controllers/plugins.js b/solr/webapp/web/js/angular/controllers/plugins.jsindex 5bf7ab5edaa..d3abd35a903 100644--- a/solr/webapp/web/js/angular/controllers/plugins.js+++ b/solr/webapp/web/js/angular/controllers/plugins.js@@ -15,10 +15,8 @@limitations under the License.*/-//NOCOMMIT: This plugin seems tied to the Admin UIs plugin management but is tied to dropwizard metrics failing some tests.-// This needs to change how it gets these metrics or we need to make a shim to the /admin/plugins handler for this to support itsolrAdminApp.controller('PluginsController',- function($scope, $rootScope, $routeParams, $location, Mbeans, Constants) {+ function($scope, $rootScope, $routeParams, $location, Metrics, Constants) {$scope.resetMenu("plugins", Constants.IS_CORE_PAGE);if ($routeParams.legacytype) {@@ -29,9 +27,15 @@ solrAdminApp.controller('PluginsController',}$scope.refresh = function() {- Mbeans.stats({core: $routeParams.core}, function (data) {- var type = $location.search().type;- $scope.types = getPluginTypes(data, type);+ var params = {};+ if ($routeParams.core) {+ params.core = $routeParams.core;+ }++ var type = $location.search().type;++ Metrics.prometheus(params, function (response) {+ $scope.types = getPluginTypesFromMetrics(response.data, type);$scope.type = getSelectedType($scope.types, type);if ($scope.type && $routeParams.entry) {@@ -64,98 +68,153 @@ solrAdminApp.controller('PluginsController',}}- $scope.startRecording = function() {- $scope.isRecording = true;- Mbeans.reference({core: $routeParams.core}, function(data) {- $scope.reference = data.reference;- console.log($scope.reference);- })+ $scope.refresh();+ });++var getPluginTypesFromMetrics = function(metricsText, selected) {+ var keys = [];++ // Parse Prometheus format metrics+ var lines = metricsText.split('\n');+ var categoriesMap = {};+ var metricMetadata = {}; // Store HELP and TYPE info for each metric++ for (var i = 0; i < lines.length; i++) {+ var line = lines[i].trim();++ // Skip empty lines+ if (line === '') {+ continue;}- $scope.stopRecording = function() {- $scope.isRecording = false;- console.log($scope.reference);- Mbeans.delta({core: $routeParams.core}, $scope.reference, function(data) {- parseDelta($scope.types, data);- });+ // Parse HELP comments - format: # HELP metric_name description+ if (line.startsWith('# HELP ')) {+ var helpMatch = line.match(/^# HELP\s+([^\s]+)\s+(.*)$/);+ if (helpMatch) {+ var metricName = helpMatch[1];+ var description = helpMatch[2];+ if (!metricMetadata[metricName]) {+ metricMetadata[metricName] = {};+ }+ metricMetadata[metricName].description = description;+ }+ continue;}- $scope.refresh();- });+ // Parse TYPE comments - format: # TYPE metric_name type+ if (line.startsWith('# TYPE ')) {+ var typeMatch = line.match(/^# TYPE\s+([^\s]+)\s+(.*)$/);+ if (typeMatch) {+ var metricName = typeMatch[1];+ var type = typeMatch[2];+ if (!metricMetadata[metricName]) {+ metricMetadata[metricName] = {};+ }+ metricMetadata[metricName].type = type;+ }+ continue;+ }-var getPluginTypes = function(data, selected) {- var keys = [];- var mbeans = data["solr-mbeans"];- for (var i=0; i<mbeans.length; i+=2) {- var key = mbeans[i];- var lower = key.toLowerCase();- var plugins = getPlugins(mbeans[i+1]);- if (plugins.length == 0) continue;- keys.push({name: key,- selected: lower == selected,- changes: 0,- lower: lower,- plugins: plugins- });- }- keys.sort(function(a,b) {return a.name > b.name});- return keys;-};+ // Skip other comments+ if (line.startsWith('#')) {+ continue;+ }-var getPlugins = function(data) {- var plugins = [];- for (var key in data) {- var pluginProperties = data[key];- var stats = pluginProperties.stats;- delete pluginProperties.stats;- for (var stat in stats) {- // add breaking space after a bracket or @ to handle wrap long lines:- stats[stat] = new String(stats[stat]).replace( /([\(@])/g, '$1​');+ // Parse metric line - format: metric_name{labels} value timestamp+ var metricMatch = line.match(/^([a-zA-Z_:][a-zA-Z0-9_:]*)\{([^}]*)\}\s+([^\s]+)(\s+[^\s]+)?$/);+ if (!metricMatch) {+ // Try without labels - format: metric_name value timestamp+ metricMatch = line.match(/^([a-zA-Z_:][a-zA-Z0-9_:]*)\s+([^\s]+)(\s+[^\s]+)?$/);+ if (metricMatch) {+ // Skip metrics without category labels for prometheus format+ continue;+ }+ continue;}- plugin = {name: key, changed: false, stats: stats, open:false};- plugin.properties = pluginProperties;- plugins.push(plugin);- }- plugins.sort(function(a,b) {return a.name > b.name});- return plugins;-};-var getSelectedType = function(types, selected) {- if (selected) {- for (var i in types) {- if (types[i].lower == selected) {- return types[i];+ var metricName = metricMatch[1];+ var labelsStr = metricMatch[2];+ var value = metricMatch[3];++ // Parse labels+ var labels = {};+ if (labelsStr) {+ var labelPairs = labelsStr.split(',');+ for (var j = 0; j < labelPairs.length; j++) {+ var labelMatch = labelPairs[j].trim().match(/^([^=]+)="([^"]*)"$/);+ if (labelMatch) {+ labels[labelMatch[1]] = labelMatch[2];+ }}}- }-};-var parseDelta = function(types, data) {+ // Use category from labels only - don't fall back to metric name parsing+ var category = labels.category;++ // Skip metrics that don't have a category label+ if (!category) {+ continue;+ }++ if (!categoriesMap[category]) {+ categoriesMap[category] = {};+ }- var getByName = function(list, name) {- for (var i in list) {- if (list[i].name == name) return list[i];+ if (!categoriesMap[category][metricName]) {+ categoriesMap[category][metricName] = {};}++ // Create a descriptive key for the metric variant+ var labelParts = [];+ for (var labelKey in labels) {+ if (labelKey !== 'category') {+ labelParts.push(labelKey + '=' + labels[labelKey]);+ }+ }+ var variantKey = labelParts.length > 0 ? labelParts.join(', ') : 'default';++ categoriesMap[category][metricName][variantKey] = value;}- var mbeans = data["solr-mbeans"]- for (var i=0; i<mbeans.length; i+=2) {- var typeName = mbeans[i];- var type = getByName(types, typeName);- var plugins = mbeans[i+1];- for (var key in plugins) {- var changedPlugin = plugins[key];- if (changedPlugin._changed_) {- var plugin = getByName(type.plugins, key);- var stats = changedPlugin.stats;- delete changedPlugin.stats;- plugin.properties = changedPlugin;- for (var stat in stats) {- // add breaking space after a bracket or @ to handle wrap long lines:- plugin.stats[stat] = new String(stats[stat]).replace( /([\(@])/g, '$1​');+ // Convert to the expected format+ for (var categoryName in categoriesMap) {+ var lower = categoryName.toLowerCase();+ var metrics = [];++ for (var metricName in categoriesMap[categoryName]) {+ var metricData = categoriesMap[categoryName][metricName];+ var metadata = metricMetadata[metricName] || {};+ metrics.push({+ name: metricName,+ changed: false,+ stats: metricData,+ open: false,+ properties: {+ description: metadata.description,+ type: metadata.type}- plugin.changed = true;- type.changes++;+ });+ }++ if (metrics.length > 0) {+ keys.push({+ name: categoryName,+ selected: lower == selected,+ changes: 0,+ lower: lower,+ plugins: metrics+ });+ }+ }++ return keys;+};++var getSelectedType = function(types, selected) {+ if (selected) {+ for (var i in types) {+ if (types[i].lower == selected) {+ return types[i];}}}diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.jsindex 0a2f5d95266..265873ac9fb 100644--- a/solr/webapp/web/js/angular/services.js+++ b/solr/webapp/web/js/angular/services.js@@ -23,7 +23,15 @@ solrAdminServices.factory('System',}]).factory('Metrics',['$resource', function($resource) {- return $resource('admin/metrics', {"wt":"json", "nodes": "@nodes", "prefix":"@prefix", "_":Date.now()});+ return $resource('admin/metrics', {"wt":"json", "nodes": "@nodes", "prefix":"@prefix", "core":"@core", "_":Date.now()}, {+ "prometheus": {+ method: 'GET',+ params: {wt: 'prometheus', core: '@core'},+ transformResponse: function(data) {+ return {data: data};+ }+ }+ });}]).factory('CollectionsV2',function() {@@ -200,25 +208,6 @@ solrAdminServices.factory('System',"status": {params:{action:"status"}, headers: {doNotIntercept: "true"}}});}])-.factory('Mbeans',- ['$resource', function($resource) {- return $resource(':core/admin/mbeans', {'wt':'json', core: '@core', '_':Date.now()}, {- stats: {params: {stats: true}},- info: {},- reference: {- params: {wt: "xml", stats: true}, transformResponse: function (data) {- return {reference: data}- }- },- delta: {method: "POST",- params: {stats: true, diff:true},- headers: {'Content-type': 'application/x-www-form-urlencoded'},- transformRequest: function(data) {- return "stream.body=" + encodeURIComponent(data);- }- }- });- }]).factory('Files',['$resource', function($resource) {return $resource(':core/admin/file', {'wt':'json', core: '@core', '_':Date.now()}, {@@ -424,6 +413,6 @@ solrAdminServices.factory('System',}return params;};-+return service;}]);diff --git a/solr/webapp/web/partials/plugins.html b/solr/webapp/web/partials/plugins.htmlindex aaa424d3b81..1e80b6f9b6e 100644--- a/solr/webapp/web/partials/plugins.html+++ b/solr/webapp/web/partials/plugins.html@@ -31,12 +31,12 @@ limitations under the License.</dl></li><li class="stats clearfix" ng-show="plugin.stats">- <span>stats:</span>+ <span>metrics:</span><ul><li ng-repeat="(key, value) in plugin.stats" ng-class="{odd: $odd}"><dl class="clearfix"><dt>{{key}}:</dt>- <dd>{{value}}</dd>+ <dd class="prometheus-metric">{{value}}</dd></dl></li></ul>@@ -53,20 +53,9 @@ limitations under the License.<span ng-show="typeObj.changes">{{typeObj.changes}}</span></a></li>- <li class="PLUGINCHANGES"><a ng-click="startRecording()">Watch Changes</a></li><li class="RELOAD"><a ng-click="refresh()">Refresh Values</a></li>- <li class="NOTE"><p>NOTE: Only selected metrics are shown here. Full metrics can be accessed via /admin/metrics handler.</p></li>+ <li class="NOTE"><p>NOTE: Metrics are displayed in Prometheus format. Full metrics can be accessed via /admin/metrics handler.</p></li></ul></div>- <div id="recording" ng-show="isRecording">- <div class="wrapper clearfix">-- <p class="loader">Watching for Changes</p>- <button class="primary" ng-click="stopRecording()">Stop & Show Changes</button>-- </div>- <div id="blockUI"></div>- </div>-</div>