Apache Solr open-source search software
SOLR-15564: Improve filtering expressions in /admin/metrics.
Andrzej Bialecki56cc23021df9
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 4519edb8a8f..84502c89fa8 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -385,6 +385,8 @@ Improvements* SOLR-15570: Include fields declared in the schema in table metadata (SQL) even if they are empty (Timothy Potter)+* SOLR-15564: Improve filtering expressions in /admin/metrics. (ab)+Optimizations---------------------* SOLR-15433: Replace transient core cache LRU by Caffeine cache. (Bruno Roustant)diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.javaindex 0aa917a95bd..52d90f96f16 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java@@ -21,9 +21,11 @@ import java.util.ArrayList;import java.util.Collections;import java.util.EnumSet;import java.util.HashSet;+import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.Set;+import java.util.TreeSet;import java.util.function.BiConsumer;import java.util.function.Predicate;import java.util.regex.Pattern;@@ -66,11 +68,12 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamepublic static final String REGISTRY_PARAM = "registry";public static final String GROUP_PARAM = "group";public static final String KEY_PARAM = "key";+ public static final String EXPR_PARAM = "expr";public static final String TYPE_PARAM = "type";public static final String ALL = "all";- private static final Pattern KEY_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));+ private static final Pattern KEY_SPLIT_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));private final CoreContainer cc;private final Map<String, String> injectedSysProps = CommonTestInjection.injectAdditionalProps();private final boolean enabled;@@ -109,7 +112,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamehandleRequest(req.getParams(), (k, v) -> rsp.add(k, v));}- public void handleRequest(SolrParams params, BiConsumer<String, Object> consumer) throws Exception {+ private void handleRequest(SolrParams params, BiConsumer<String, Object> consumer) throws Exception {if (!enabled) {consumer.accept("error", "metrics collection is disabled");return;@@ -120,6 +123,11 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamehandleKeyRequest(keys, consumer);return;}+ String[] exprs = params.getParams(EXPR_PARAM);+ if (exprs != null && exprs.length > 0) {+ handleExprRequest(exprs, consumer);+ return;+ }MetricFilter mustMatchFilter = parseMustMatchFilter(params);Predicate<CharSequence> propertyFilter = parsePropertyFilter(params);List<MetricType> metricTypes = parseMetricTypes(params);@@ -139,14 +147,95 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNameconsumer.accept("metrics", response);}- public void handleKeyRequest(String[] keys, BiConsumer<String, Object> consumer) {+ private static class MetricsExpr {+ Pattern registryRegex;+ MetricFilter metricFilter;+ Predicate<CharSequence> propertyFilter;+ }++ private void handleExprRequest(String[] exprs, BiConsumer<String, Object> consumer) {+ SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();+ SimpleOrderedMap<Object> errors = new SimpleOrderedMap<>();+ List<MetricsExpr> metricsExprs = new ArrayList<>();++ for (String key : exprs) {+ if (key == null || key.isEmpty()) {+ continue;+ }+ String[] parts = KEY_SPLIT_REGEX.split(key);+ if (parts.length < 2 || parts.length > 3) {+ errors.add(key, "at least two and at most three colon-separated parts must be provided");+ continue;+ }+ MetricsExpr me = new MetricsExpr();+ me.registryRegex = Pattern.compile(unescape(parts[0]));+ me.metricFilter = new SolrMetricManager.RegexFilter(unescape(parts[1]));+ String propertyPart = parts.length > 2 ? unescape(parts[2]) : null;+ if (propertyPart == null) {+ me.propertyFilter = name -> true;+ } else {+ me.propertyFilter = new Predicate<>() {+ final Pattern pattern = Pattern.compile(propertyPart);+ @Override+ public boolean test(CharSequence charSequence) {+ return pattern.matcher(charSequence).matches();+ }+ };+ }+ metricsExprs.add(me);+ }+ // find matching registries first, to avoid scanning non-matching registries+ Set<String> matchingRegistries = new TreeSet<>();+ metricsExprs.forEach(me -> {+ metricManager.registryNames().forEach(name -> {+ if (me.registryRegex.matcher(name).matches()) {+ matchingRegistries.add(name);+ }+ });+ });+ for (String registryName : matchingRegistries) {+ MetricRegistry registry = metricManager.registry(registryName);+ for (MetricsExpr me : metricsExprs) {+ @SuppressWarnings("unchecked")+ SimpleOrderedMap<Object> perRegistryResult = (SimpleOrderedMap<Object>) result.get(registryName);+ final SimpleOrderedMap<Object> perRegistryTemp = new SimpleOrderedMap<>();+ // skip processing if not a matching registry+ if (!me.registryRegex.matcher(registryName).matches()) {+ continue;+ }+ MetricUtils.toMaps(registry, Collections.singletonList(MetricFilter.ALL), me.metricFilter,+ me.propertyFilter, false, false, true, false, (k, v) -> perRegistryTemp.add(k, v));+ // extracted some metrics and there's no entry for this registry yet+ if (perRegistryTemp.size() > 0) {+ if (perRegistryResult == null) { // new results for this registry+ result.add(registryName, perRegistryTemp);+ } else {+ // merge if needed+ for (Iterator<Map.Entry<String, Object>> it = perRegistryTemp.iterator(); it.hasNext(); ) {+ Map.Entry<String, Object> entry = it.next();+ Object existing = perRegistryResult.get(entry.getKey());+ if (existing == null) {+ perRegistryResult.add(entry.getKey(), entry.getValue());+ }+ }+ }+ }+ }+ }+ consumer.accept("metrics", result);+ if (errors.size() > 0) {+ consumer.accept("errors", errors);+ }+ }++ private void handleKeyRequest(String[] keys, BiConsumer<String, Object> consumer) {SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();SimpleOrderedMap<Object> errors = new SimpleOrderedMap<>();for (String key : keys) {if (key == null || key.isEmpty()) {continue;}- String[] parts = KEY_REGEX.split(key);+ String[] parts = KEY_SPLIT_REGEX.split(key);if (parts.length < 2 || parts.length > 3) {errors.add(key, "at least two and at most three colon-separated parts must be provided");continue;@@ -200,7 +289,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamefor (int i = 0; i < s.length(); i++) {char c = s.charAt(i);if (c == '\\') {- continue;+ if (i < s.length() - 1 && s.charAt(i + 1) == ':') {+ continue;+ }}sb.append(c);}diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.javaindex 338c38fe091..adcc3054d24 100644--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java@@ -19,6 +19,7 @@ package org.apache.solr.handler.admin;import java.util.Arrays;import java.util.HashMap;+import java.util.List;import java.util.Map;import com.codahale.metrics.Counter;@@ -349,6 +350,78 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {handler.close();}+ @Test+ @SuppressWarnings("unchecked")+ public void testExprMetrics() throws Exception {+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());++ String key1 = "solr\\.core\\..*:.*/select\\.request.*:.*Rate";+ SolrQueryResponse resp = new SolrQueryResponse();+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",+ MetricsHandler.EXPR_PARAM, key1), resp);+ // response structure is like in the case of non-key params+ Object val = resp.getValues().findRecursive( "metrics", "solr.core.collection1", "QUERY./select.requestTimes");+ assertNotNull(val);+ assertTrue(val instanceof MapWriter);+ Map<String, Object> map = new HashMap<>();+ ((MapWriter) val).toMap(map);+ assertEquals(map.toString(), 4, map.size()); // mean, 1, 5, 15+ assertNotNull(map.toString(), map.get("meanRate"));+ assertNotNull(map.toString(), map.get("1minRate"));+ assertNotNull(map.toString(), map.get("5minRate"));+ assertNotNull(map.toString(), map.get("15minRate"));+ assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(), 0.0, 0.0);+ map.clear();++ String key2 = "solr\\.core\\..*:.*/select\\.request.*";+ resp = new SolrQueryResponse();+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",+ MetricsHandler.EXPR_PARAM, key2), resp);+ // response structure is like in the case of non-key params+ val = resp.getValues().findRecursive( "metrics", "solr.core.collection1");+ assertNotNull(val);+ Object v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requestTimes");+ assertNotNull(v);+ assertTrue(v instanceof MapWriter);+ ((MapWriter) v).toMap(map);+ assertEquals(map.toString(), 14, map.size());+ assertNotNull(map.toString(), map.get("1minRate"));+ assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(), 0.0, 0.0);+ map.clear();+ // select requests counter+ v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requests");+ assertNotNull(v);+ assertTrue(v instanceof Number);++ // test multiple expressions producing overlapping metrics - should be no dupes++ // this key matches also sub-metrics of /select, eg. /select.distrib, /select.local, ...+ String key3 = "solr\\.core\\..*:.*/select.*\\.requestTimes:count";+ resp = new SolrQueryResponse();+ // ORDER OF PARAMS MATTERS HERE! see the refguide+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",+ MetricsHandler.EXPR_PARAM, key2, MetricsHandler.EXPR_PARAM, key1, MetricsHandler.EXPR_PARAM, key3), resp);+ val = resp.getValues().findRecursive( "metrics", "solr.core.collection1");+ assertNotNull(val);+ // for requestTimes only the full set of values from the first expr should be present+ assertNotNull(val);+ SimpleOrderedMap<Object> values = (SimpleOrderedMap<Object>) val;+ assertEquals(values.jsonStr(), 4, values.size());+ List<Object> multipleVals = values.getAll("QUERY./select.requestTimes");+ assertEquals(multipleVals.toString(), 1, multipleVals.size());+ v = values.get("QUERY./select.local.requestTimes");+ assertTrue(v instanceof MapWriter);+ ((MapWriter) v).toMap(map);+ assertEquals(map.toString(), 1, map.size());+ assertTrue(map.toString(), map.containsKey("count"));+ map.clear();+ v = values.get("QUERY./select.distrib.requestTimes");+ assertTrue(v instanceof MapWriter);+ ((MapWriter) v).toMap(map);+ assertEquals(map.toString(), 1, map.size());+ assertTrue(map.toString(), map.containsKey("count"));+ }+@Testpublic void testMetricsUnload() throws Exception {diff --git a/solr/solr-ref-guide/src/metrics-reporting.adoc b/solr/solr-ref-guide/src/metrics-reporting.adocindex 8854be7ad7b..59326fa1e6f 100644--- a/solr/solr-ref-guide/src/metrics-reporting.adoc+++ b/solr/solr-ref-guide/src/metrics-reporting.adoc@@ -763,7 +763,33 @@ Examples:* `key=solr.jvm:system.properties:user.name`+-*NOTE: when this parameter is used, other selection methods listed above are ignored.*+*NOTE: when this parameter is used, any other selection methods are ignored.*++`expr`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Extended notation of the `key` selection criteria, which supports regular expressions for each of the+parts supported by the `key` selector. This parameter can be specified multiple times to retrieve metrics that match+any expression. The API guarantees that the output will consist only of unique metric names even if+multiple expressions match the same metric name. Note: order of multiple `expr` parameters matters here+- only the first value of the first matching expression will be recorded, subsequent values for the same metric name+produced by matching other expressions will be skipped.+++Fully-qualified expression consists of at least two and at most three regex patterns separated by+colons: a registry pattern, colon, a metric pattern, and then an optional colon and metric property pattern.+Colons and other regex meta-characters in names and in regular expressions MUST be escaped using backslash (`\`) character.++Examples:++* `expr=solr\.core\..*:QUERY\..*\.requestTimes:max_ms`+* `expr=solr\.jvm:system\.properties:user\..*`++++*NOTE: when this parameter is used, any other selection methods are ignored.*`compact`::+@@ -845,3 +871,8 @@ Request only "user.name" property of "system.properties" metric from registry "s[source,text]http://localhost:8983/solr/admin/metrics?wt=xml&key=solr.jvm:system.properties:user.name++Request query rates (but not histograms) from any core in any collection in any QUERY handler:++[source,text]+http://localhost:8983/solr/admin/metrics?expr=solr\.core\..*:QUERY\..*\.requestTimes:.*Rate