Apache Solr open-source search software
SOLR-14691: Metrics reporting should avoid creating objects.
Andrzej Bialecki43b1a2fdc7a4
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 6e3e6e72804..0780d848207 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -195,6 +195,8 @@ Improvements* SOLR-14905: Update commons-io version to 2.8.0 due to security vulnerability. (Nazerke Seidan via Bruno Roustant)+* SOLR-14691: Metrics reporting should avoid creating objects. (ab, Noble Paul)+Optimizations---------------------diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.javaindex 0b07af8323d..97bed5c0de1 100644--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java@@ -68,6 +68,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.RateLimiter;+import org.apache.solr.common.MapWriter;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.params.CommonParams;@@ -894,7 +895,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAwtrue, "isLeader", getCategory().toString(), scope);solrMetricsContext.gauge(() -> isFollower,true, "isFollower", getCategory().toString(), scope);- final MetricsMap fetcherMap = new MetricsMap((detailed, map) -> {+ final MetricsMap fetcherMap = new MetricsMap(map -> {IndexFetcher fetcher = currentIndexFetcher;if (fetcher != null) {map.put(LEADER_URL, fetcher.getLeaderUrl());@@ -1110,10 +1111,10 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw}}- private void addVal(Map<String, Object> map, String key, Properties props, @SuppressWarnings({"rawtypes"})Class clzz) {+ private void addVal(MapWriter.EntryWriter ew, String key, Properties props, @SuppressWarnings({"rawtypes"})Class clzz) {Object val = formatVal(key, props, clzz);if (val != null) {- map.put(key, val);+ ew.putNoEx(key, val);}}diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.javaindex 1fcc183b6ce..d41ad54c758 100644--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java@@ -157,8 +157,8 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfonumClientErrors = solrMetricsContext.meter("clientErrors", getCategory().toString(), scope);numTimeouts = solrMetricsContext.meter("timeouts", getCategory().toString(), scope);requests = solrMetricsContext.counter("requests", getCategory().toString(), scope);- MetricsMap metricsMap = new MetricsMap((detail, map) ->- shardPurposes.forEach((k, v) -> map.put(k, v.getCount())));+ MetricsMap metricsMap = new MetricsMap(map ->+ shardPurposes.forEach((k, v) -> map.putNoEx(k, v.getCount())));solrMetricsContext.gauge(metricsMap, true, "shardRequests", getCategory().toString(), scope);requestTimes = solrMetricsContext.timer("requestTimes", getCategory().toString(), scope);distribRequestTimes = solrMetricsContext.timer("requestTimes", getCategory().toString(), scope, "distrib");diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.javaindex e6d80174b4f..76bb04ddcf9 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java@@ -25,6 +25,7 @@ import java.util.List;import java.util.Map;import java.util.Set;import java.util.function.BiConsumer;+import java.util.function.Predicate;import java.util.regex.Pattern;import java.util.stream.Collectors;@@ -36,6 +37,7 @@ import com.codahale.metrics.Metric;import com.codahale.metrics.MetricFilter;import com.codahale.metrics.MetricRegistry;import com.codahale.metrics.Timer;+import org.apache.solr.common.MapWriter;import org.apache.solr.common.SolrException;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.CommonTestInjection;@@ -112,17 +114,15 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamereturn;}MetricFilter mustMatchFilter = parseMustMatchFilter(params);- MetricUtils.PropertyFilter propertyFilter = parsePropertyFilter(params);+ Predicate<CharSequence> propertyFilter = parsePropertyFilter(params);List<MetricType> metricTypes = parseMetricTypes(params);List<MetricFilter> metricFilters = metricTypes.stream().map(MetricType::asMetricFilter).collect(Collectors.toList());Set<String> requestedRegistries = parseRegistries(params);- @SuppressWarnings({"rawtypes"})- NamedList response = new SimpleOrderedMap();+ NamedList<Object> response = new SimpleOrderedMap<>();for (String registryName : requestedRegistries) {MetricRegistry registry = metricManager.registry(registryName);- @SuppressWarnings({"rawtypes"})- SimpleOrderedMap result = new SimpleOrderedMap();+ SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();MetricUtils.toMaps(registry, metricFilters, mustMatchFilter, propertyFilter, false,false, compact, false, (k, v) -> result.add(k, v));if (result.size() > 0) {@@ -134,8 +134,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName@SuppressWarnings({"unchecked", "rawtypes"})public void handleKeyRequest(String[] keys, BiConsumer<String, Object> consumer) throws Exception {- SimpleOrderedMap result = new SimpleOrderedMap();- SimpleOrderedMap errors = new SimpleOrderedMap();+ SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();+ SimpleOrderedMap<Object> errors = new SimpleOrderedMap<>();for (String key : keys) {if (key == null || key.isEmpty()) {continue;@@ -158,7 +158,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNameerrors.add(key, "metric '" + metricName + "' not found");continue;}- MetricUtils.PropertyFilter propertyFilter = MetricUtils.PropertyFilter.ALL;+ Predicate<CharSequence> propertyFilter = MetricUtils.ALL_PROPERTIES;if (propertyName != null) {propertyFilter = (name) -> name.equals(propertyName);// use escaped versions@@ -173,6 +173,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNameMetricUtils.convertMetric(key, m, propertyFilter, false, true, true, false, ":", (k, v) -> {if ((v instanceof Map) && propertyName != null) {((Map)v).forEach((k1, v1) -> result.add(k + ":" + k1, v1));+ } else if ((v instanceof MapWriter) && propertyName != null) {+ ((MapWriter) v)._forEachEntry((k1, v1) -> result.add(k + ":" + k1, v1));} else {result.add(k, v);}@@ -229,10 +231,10 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamereturn mustMatchFilter;}- private MetricUtils.PropertyFilter parsePropertyFilter(SolrParams params) {+ private Predicate<CharSequence> parsePropertyFilter(SolrParams params) {String[] props = params.getParams(PROPERTY_PARAM);if (props == null || props.length == 0) {- return MetricUtils.PropertyFilter.ALL;+ return MetricUtils.ALL_PROPERTIES;}final Set<String> filter = new HashSet<>();for (String prop : props) {@@ -241,7 +243,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName}}if (filter.isEmpty()) {- return MetricUtils.PropertyFilter.ALL;+ return MetricUtils.ALL_PROPERTIES;} else {return (name) -> filter.contains(name);}diff --git a/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.javaindex 59a95717e93..adfcb15554b 100644--- a/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java+++ b/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java@@ -357,10 +357,10 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,super.initializeMetrics(parentContext, scope);this.solrMetricsContext.gauge(() -> ramBytesUsed(), true, "totalSizeInBytes", getCategory().toString());- MetricsMap suggestersMap = new MetricsMap((detailed, map) -> {+ MetricsMap suggestersMap = new MetricsMap(map -> {for (Map.Entry<String, SolrSuggester> entry : suggesters.entrySet()) {SolrSuggester suggester = entry.getValue();- map.put(entry.getKey(), suggester.toString());+ map.putNoEx(entry.getKey(), suggester.toString());}});this.solrMetricsContext.gauge(suggestersMap, true, "suggesters", getCategory().toString(), scope);diff --git a/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java b/solr/core/src/java/org/apache/solr/metrics/MetricsMap.javaindex bd9abaf3985..8faccff4bd3 100644--- a/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java+++ b/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java@@ -28,6 +28,7 @@ import javax.management.ReflectionException;import javax.management.openmbean.OpenMBeanAttributeInfoSupport;import javax.management.openmbean.OpenType;import javax.management.openmbean.SimpleType;+import java.io.IOException;import java.lang.invoke.MethodHandles;import java.lang.reflect.Field;import java.util.ArrayList;@@ -38,6 +39,7 @@ import java.util.function.BiConsumer;import com.codahale.metrics.Gauge;import com.codahale.metrics.Metric;import org.apache.lucene.store.AlreadyClosedException;+import org.apache.solr.common.MapWriter;import org.apache.solr.common.SolrException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -51,18 +53,33 @@ import org.slf4j.LoggerFactory;* {@link javax.management.openmbean.OpenType#ALLOWED_CLASSNAMES_LIST}, otherwise only their toString()* representation will be shown in JConsole.</p>*/-public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {+public class MetricsMap implements Gauge<Map<String,Object>>, MapWriter, DynamicMBean {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());// set to true to use cached statistics between getMBeanInfo calls to work// around over calling getStatistics on MBeanInfos when iterating over all attributes (SOLR-6586)private final boolean useCachedStatsBetweenGetMBeanInfoCalls = Boolean.getBoolean("useCachedStatsBetweenGetMBeanInfoCalls");- private BiConsumer<Boolean, Map<String, Object>> initializer;- private Map<String, String> jmxAttributes = new HashMap<>();+ private BiConsumer<Boolean, Map<String, Object>> mapInitializer;+ private MapWriter initializer;+ private Map<String, String> jmxAttributes;private volatile Map<String,Object> cachedValue;- public MetricsMap(BiConsumer<Boolean, Map<String,Object>> initializer) {+ /**+ * Create an instance that reports values to a Map.+ * @param mapInitializer function to populate the Map result.+ * @deprecated use {@link #MetricsMap(MapWriter)} instead.+ */+ @Deprecated(since = "8.7")+ public MetricsMap(BiConsumer<Boolean, Map<String,Object>> mapInitializer) {+ this.mapInitializer = mapInitializer;+ }++ /**+ * Create an instance that reports values to a MapWriter.+ * @param initializer function to populate the MapWriter result.+ */+ public MetricsMap(MapWriter initializer) {this.initializer = initializer;}@@ -73,7 +90,11 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {public Map<String,Object> getValue(boolean detailed) {Map<String,Object> map = new HashMap<>();- initializer.accept(detailed, map);+ if (mapInitializer != null) {+ mapInitializer.accept(detailed, map);+ } else {+ initializer.toMap(map);+ }return map;}@@ -81,13 +102,22 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {return getValue().toString();}+ // lazy init+ private synchronized void initJmxAttributes() {+ if (jmxAttributes == null) {+ jmxAttributes = new HashMap<>();+ }+ }+@Overridepublic Object getAttribute(String attribute) throws AttributeNotFoundException, MBeanException, ReflectionException {Object val;// jmxAttributes override any real values- val = jmxAttributes.get(attribute);- if (val != null) {- return val;+ if (jmxAttributes != null) {+ val = jmxAttributes.get(attribute);+ if (val != null) {+ return val;+ }}Map<String,Object> stats = null;if (useCachedStatsBetweenGetMBeanInfoCalls) {@@ -117,6 +147,7 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {@Overridepublic void setAttribute(Attribute attribute) throws AttributeNotFoundException, InvalidAttributeValueException, MBeanException, ReflectionException {+ initJmxAttributes();jmxAttributes.put(attribute.getName(), String.valueOf(attribute.getValue()));}@@ -150,13 +181,15 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {if (useCachedStatsBetweenGetMBeanInfoCalls) {cachedValue = stats;}- jmxAttributes.forEach((k, v) -> {- attrInfoList.add(new MBeanAttributeInfo(k, String.class.getName(),- null, true, false, false));- });+ if (jmxAttributes != null) {+ jmxAttributes.forEach((k, v) -> {+ attrInfoList.add(new MBeanAttributeInfo(k, String.class.getName(),+ null, true, false, false));+ });+ }try {stats.forEach((k, v) -> {- if (jmxAttributes.containsKey(k)) {+ if (jmxAttributes != null && jmxAttributes.containsKey(k)) {return;}@SuppressWarnings({"rawtypes"})@@ -197,4 +230,14 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {}return null;}++ @Override+ public void writeMap(EntryWriter ew) throws IOException {+ if (mapInitializer != null) {+ Map<String, Object> value = getValue();+ value.forEach((k, v) -> ew.putNoEx(k, v));+ } else {+ initializer.writeMap(ew);+ }+ }}\ No newline at end of filediff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.javaindex 929aa93108e..9606ef5a340 100644--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java@@ -453,7 +453,7 @@ public class SolrReporter extends ScheduledReporter {}final String effectiveGroup = group;MetricUtils.toSolrInputDocuments(metricManager.registry(registryName), Collections.singletonList(report.filter), MetricFilter.ALL,- MetricUtils.PropertyFilter.ALL, skipHistograms, skipAggregateValues, compact, metadata, doc -> {+ MetricUtils.ALL_PROPERTIES, skipHistograms, skipAggregateValues, compact, metadata, doc -> {doc.setField(REGISTRY_ID, registryName);doc.setField(GROUP_ID, effectiveGroup);if (effectiveLabel != null) {diff --git a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java b/solr/core/src/java/org/apache/solr/search/CaffeineCache.javaindex 8da244b972e..756718c6175 100644--- a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java+++ b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java@@ -364,7 +364,7 @@ public class CaffeineCache<K, V> extends SolrCacheBase implements SolrCache<K, V@Overridepublic void initializeMetrics(SolrMetricsContext parentContext, String scope) {solrMetricsContext = parentContext.getChildContext(this);- cacheMap = new MetricsMap((detailed, map) -> {+ cacheMap = new MetricsMap(map -> {if (cache != null) {CacheStats stats = cache.stats();long insertCount = inserts.sum();diff --git a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.javaindex 50056272ee8..193abbfd27c 100644--- a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java+++ b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java@@ -49,8 +49,8 @@ public class SolrFieldCacheBean implements SolrInfoBean {@Overridepublic void initializeMetrics(SolrMetricsContext parentContext, String scope) {this.solrMetricsContext = parentContext;- MetricsMap metricsMap = new MetricsMap((detailed, map) -> {- if (detailed && !disableEntryList && !disableJmxEntryList) {+ MetricsMap metricsMap = new MetricsMap(map -> {+ if (!disableEntryList && !disableJmxEntryList) {UninvertingReader.FieldCacheStats fieldCacheStats = UninvertingReader.getUninvertedStats();String[] entries = fieldCacheStats.info;map.put("entries_count", entries.length);diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.javaindex 3ebd43c845e..5d1af6d1865 100644--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java@@ -2298,8 +2298,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI}, true, "indexCommitSize", Category.SEARCHER.toString(), scope);// statsCache metricsparentContext.gauge(- new MetricsMap((detailed, map) -> {- statsCache.getCacheMetrics().getSnapshot(map::put);+ new MetricsMap(map -> {+ statsCache.getCacheMetrics().getSnapshot(map::putNoEx);map.put("statsCacheImpl", statsCache.getClass().getSimpleName());}), true, "statsCache", Category.CACHE.toString(), scope);}diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.javaindex 01630a8d156..95afda4b82c 100644--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java@@ -235,10 +235,10 @@ public class SolrDispatchFilter extends BaseSolrFilter {metricManager.registerAll(registryName, new GarbageCollectorMetricSet(), SolrMetricManager.ResolutionStrategy.IGNORE, "gc");metricManager.registerAll(registryName, new MemoryUsageGaugeSet(), SolrMetricManager.ResolutionStrategy.IGNORE, "memory");metricManager.registerAll(registryName, new ThreadStatesGaugeSet(), SolrMetricManager.ResolutionStrategy.IGNORE, "threads"); // todo should we use CachedThreadStatesGaugeSet instead?- MetricsMap sysprops = new MetricsMap((detailed, map) -> {+ MetricsMap sysprops = new MetricsMap(map -> {System.getProperties().forEach((k, v) -> {if (!hiddenSysProps.contains(k)) {- map.put(String.valueOf(k), v);+ map.putNoEx(String.valueOf(k), v);}});});diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java b/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.javaindex 37d735e1ec5..76bb265109a 100644--- a/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java+++ b/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java@@ -59,7 +59,7 @@ public class Metrics extends SolrCacheBase implements SolrInfoBean {@Overridepublic void initializeMetrics(SolrMetricsContext parentContext, String scope) {solrMetricsContext = parentContext.getChildContext(this);- metricsMap = new MetricsMap((detailed, map) -> {+ metricsMap = new MetricsMap(map -> {long now = System.nanoTime();long delta = Math.max(now - previous, 1);double seconds = delta / 1000000000.0;diff --git a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.javaindex 6ff88f273bd..fe19d120a55 100644--- a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java+++ b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java@@ -93,7 +93,7 @@ public class HdfsLocalityReporter implements SolrInfoBean {@Overridepublic void initializeMetrics(SolrMetricsContext parentContext, String scope) {solrMetricsContext = parentContext.getChildContext(this);- MetricsMap metricsMap = new MetricsMap((detailed, map) -> {+ MetricsMap metricsMap = new MetricsMap(map -> {long totalBytes = 0;long localBytes = 0;int totalCount = 0;diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.javaindex c4fdd515d56..1b7b13c2b5e 100644--- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java+++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java@@ -20,14 +20,13 @@ import java.beans.BeanInfo;import java.beans.IntrospectionException;import java.beans.Introspector;import java.beans.PropertyDescriptor;+import java.io.IOException;import java.lang.invoke.MethodHandles;import java.lang.management.OperatingSystemMXBean;import java.lang.management.PlatformManagedObject;import java.lang.reflect.InvocationTargetException;import java.util.Collection;-import java.util.Collections;import java.util.HashMap;-import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.SortedSet;@@ -35,6 +34,7 @@ import java.util.concurrent.ExecutorService;import java.util.concurrent.TimeUnit;import java.util.function.BiConsumer;import java.util.function.Consumer;+import java.util.function.Predicate;import com.codahale.metrics.Counter;import com.codahale.metrics.Gauge;@@ -46,10 +46,14 @@ import com.codahale.metrics.MetricFilter;import com.codahale.metrics.MetricRegistry;import com.codahale.metrics.Snapshot;import com.codahale.metrics.Timer;+import org.apache.solr.common.ConditionalKeyMapWriter;+import org.apache.solr.common.IteratorWriter;+import org.apache.solr.common.MapWriter;import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.SolrInfoBean;import org.apache.solr.metrics.AggregateMetric;+import org.apache.solr.metrics.SolrMetricManager;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -87,7 +91,9 @@ public class MetricUtils {/*** This filter can limit what properties of a metric are returned.+ * @deprecated use {@link Predicate} instead.*/+ @Deprecated(since = "8.7")public interface PropertyFilter {PropertyFilter ALL = (name) -> true;@@ -97,8 +103,14 @@ public class MetricUtils {* @return true if this property should be returned, false otherwise.*/boolean accept(String name);++ static Predicate<CharSequence> toPredicate(PropertyFilter filter) {+ return (name) -> filter.accept(name.toString());+ }}+ public static final Predicate<CharSequence> ALL_PROPERTIES = (name) -> true;+/*** Adds metrics from a Timer to a NamedList, using well-known back-compat names.* @param lst The NamedList to add the metrics data to@@ -145,11 +157,40 @@ public class MetricUtils {* @param metadata optional metadata. If not null and not empty then this map will be added under a* {@code _metadata_} key.* @param consumer consumer that accepts produced {@link SolrInputDocument}-s+ * @deprecated use {@link #toSolrInputDocuments(MetricRegistry, List, MetricFilter, Predicate, boolean, boolean, boolean, Map, Consumer)} instead.*/+ @Deprecated(since = "8.7")public static void toSolrInputDocuments(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,MetricFilter mustMatchFilter, PropertyFilter propertyFilter, boolean skipHistograms,boolean skipAggregateValues, boolean compact,Map<String, Object> metadata, Consumer<SolrInputDocument> consumer) {+ toSolrInputDocuments(registry, shouldMatchFilters, mustMatchFilter,+ PropertyFilter.toPredicate(propertyFilter), skipHistograms,+ skipAggregateValues, compact, metadata, consumer);+ }+ /**+ * Provides a representation of the given metric registry as {@link SolrInputDocument}-s.+ Only those metrics+ * are converted which match at least one of the given MetricFilter instances.+ *+ * @param registry the {@link MetricRegistry} to be converted+ * @param shouldMatchFilters a list of {@link MetricFilter} instances.+ * A metric must match <em>any one</em> of the filters from this list to be+ * included in the output+ * @param mustMatchFilter a {@link MetricFilter}.+ * A metric <em>must</em> match this filter to be included in the output.+ * @param propertyFilter limit what properties of a metric are returned+ * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.+ * @param skipAggregateValues discard internal values of {@link AggregateMetric}-s.+ * @param compact use compact representation for counters and gauges.+ * @param metadata optional metadata. If not null and not empty then this map will be added under a+ * {@code _metadata_} key.+ * @param consumer consumer that accepts produced {@link SolrInputDocument}-s+ */+ public static void toSolrInputDocuments(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,+ MetricFilter mustMatchFilter, Predicate<CharSequence> propertyFilter, boolean skipHistograms,+ boolean skipAggregateValues, boolean compact,+ Map<String, Object> metadata, Consumer<SolrInputDocument> consumer) {boolean addMetadata = metadata != null && !metadata.isEmpty();toMaps(registry, shouldMatchFilters, mustMatchFilter, propertyFilter, skipHistograms, skipAggregateValues, compact, false, (k, v) -> {SolrInputDocument doc = new SolrInputDocument();@@ -169,23 +210,69 @@ public class MetricUtils {* @param o an instance of converted metric, either a Map or a flat Object*/static void toSolrInputDocument(String prefix, SolrInputDocument doc, Object o) {- if (!(o instanceof Map)) {- String key = prefix != null ? prefix : VALUE;- doc.addField(key, o);- return;- }- @SuppressWarnings({"unchecked"})- Map<String, Object> map = (Map<String, Object>)o;- for (Map.Entry<String, Object> entry : map.entrySet()) {- if (entry.getValue() instanceof Map) { // flatten recursively- toSolrInputDocument(entry.getKey(), doc, entry.getValue());+ final BiConsumer<Object, Object> consumer = (k, v) -> {+ if ((v instanceof Map) || (v instanceof MapWriter) || (v instanceof IteratorWriter)) {+ toSolrInputDocument(k.toString(), doc, v);} else {- String key = prefix != null ? prefix + "." + entry.getKey() : entry.getKey();- doc.addField(key, entry.getValue());+ String key = prefix != null ? prefix + "." + k : k.toString();+ doc.addField(key, v);+ }+ };+ if (o instanceof MapWriter) {+ @SuppressWarnings({"unchecked"})+ MapWriter writer = (MapWriter) o;+ writer._forEachEntry(consumer);+ } else if (o instanceof Map) {+ @SuppressWarnings({"unchecked"})+ Map<String, Object> map = (Map<String, Object>) o;+ for (Map.Entry<String, Object> entry : map.entrySet()) {+ consumer.accept(entry.getKey(), entry.getValue());}+ } else if (o instanceof IteratorWriter) {+ @SuppressWarnings({"unchecked"})+ IteratorWriter writer = (IteratorWriter) o;+ final String name = prefix != null ? prefix : "value";+ try {+ writer.writeIter(new IteratorWriter.ItemWriter() {+ @Override+ public IteratorWriter.ItemWriter add(Object o) throws IOException {+ consumer.accept(name, o);+ return this;+ }+ });+ } catch (IOException e) {+ throw new RuntimeException("this should never happen", e);+ }+ } else {+ String key = prefix != null ? prefix : VALUE;+ doc.addField(key, o);}}+ /**+ * Convert selected metrics to maps or to flattened objects.+ * @param registry source of metrics+ * @param shouldMatchFilters metrics must match any of these filters+ * @param mustMatchFilter metrics must match this filter+ * @param propertyFilter limit what properties of a metric are returned+ * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.+ * @param skipAggregateValues discard internal values of {@link AggregateMetric}-s.+ * @param compact use compact representation for counters and gauges.+ * @param simple use simplified representation for complex metrics - instead of a (name, map)+ * only the selected (name "." key, value) pairs will be produced.+ * @param consumer consumer that accepts produced objects+ * @deprecated use {@link #toMaps(MetricRegistry, List, MetricFilter, Predicate, boolean, boolean, boolean, boolean, BiConsumer)} instead.+ */+ @Deprecated(since = "8.7")+ public static void toMaps(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,+ MetricFilter mustMatchFilter, PropertyFilter propertyFilter,+ boolean skipHistograms, boolean skipAggregateValues,+ boolean compact, boolean simple,+ BiConsumer<String, Object> consumer) {+ toMaps(registry, shouldMatchFilters, mustMatchFilter,+ PropertyFilter.toPredicate(propertyFilter), skipHistograms,+ skipAggregateValues, compact, simple, consumer);+ }/*** Convert selected metrics to maps or to flattened objects.* @param registry source of metrics@@ -200,7 +287,7 @@ public class MetricUtils {* @param consumer consumer that accepts produced objects*/public static void toMaps(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,- MetricFilter mustMatchFilter, PropertyFilter propertyFilter,+ MetricFilter mustMatchFilter, Predicate<CharSequence> propertyFilter,boolean skipHistograms, boolean skipAggregateValues,boolean compact, boolean simple,BiConsumer<String, Object> consumer) {@@ -263,8 +350,27 @@ public class MetricUtils {* @param simple use simplified representation for complex metrics - instead of a (name, map)* only the selected (name "." key, value) pairs will be produced.* @param consumer consumer that accepts produced objects+ * @deprecated use {@link #convertMetric(String, Metric, Predicate, boolean, boolean, boolean, boolean, String, BiConsumer)} instead.*/+ @Deprecated(since = "8.7")public static void convertMetric(String n, Metric metric, PropertyFilter propertyFilter, boolean skipHistograms, boolean skipAggregateValues,+ boolean compact, boolean simple, String separator, BiConsumer<String, Object> consumer) {+ convertMetric(n, metric, PropertyFilter.toPredicate(propertyFilter),+ skipHistograms, skipAggregateValues, compact, simple, separator, consumer);+ }+ /**+ * Convert a single instance of metric into a map or flattened object.+ * @param n metric name+ * @param metric metric instance+ * @param propertyFilter limit what properties of a metric are returned+ * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.+ * @param skipAggregateValues discard internal values of {@link AggregateMetric}-s.+ * @param compact use compact representation for counters and gauges.+ * @param simple use simplified representation for complex metrics - instead of a (name, map)+ * only the selected (name "." key, value) pairs will be produced.+ * @param consumer consumer that accepts produced objects+ */+ public static void convertMetric(String n, Metric metric, Predicate<CharSequence> propertyFilter, boolean skipHistograms, boolean skipAggregateValues,boolean compact, boolean simple, String separator, BiConsumer<String, Object> consumer) {if (metric instanceof Counter) {Counter counter = (Counter) metric;@@ -272,8 +378,16 @@ public class MetricUtils {} else if (metric instanceof Gauge) {@SuppressWarnings({"rawtypes"})Gauge gauge = (Gauge) metric;+ // unwrap if needed+ if (gauge instanceof SolrMetricManager.GaugeWrapper) {+ gauge = ((SolrMetricManager.GaugeWrapper) gauge).getGauge();+ }try {- convertGauge(n, gauge, propertyFilter, simple, compact, separator, consumer);+ if (gauge instanceof MapWriter) {+ convertMapWriter(n, (MapWriter) gauge, propertyFilter, simple, compact, separator, consumer);+ } else {+ convertGauge(n, gauge, propertyFilter, simple, compact, separator, consumer);+ }} catch (InternalError ie) {if (n.startsWith("memory.") && ie.getMessage().contains("Memory Pool not found")) {log.warn("Error converting gauge '{}', possible JDK bug: SOLR-10362", n, ie);@@ -309,37 +423,38 @@ public class MetricUtils {* @param consumer consumer that accepts produced objects*/static void convertAggregateMetric(String name, AggregateMetric metric,- PropertyFilter propertyFilter,+ Predicate<CharSequence> propertyFilter,boolean skipAggregateValues, boolean simple, String separator, BiConsumer<String, Object> consumer) {if (simple) {- if (propertyFilter.accept(MEAN)) {+ if (propertyFilter.test(MEAN)) {consumer.accept(name + separator + MEAN, metric.getMean());}} else {- Map<String, Object> response = new LinkedHashMap<>();- BiConsumer<String, Object> filter = (k, v) -> {- if (propertyFilter.accept(k)) {- response.put(k, v);+ MapWriter writer = ew -> {+ BiConsumer<String, Object> filter = (k, v) -> {+ if (propertyFilter.test(k)) {+ ew.putNoEx(k, v);+ }+ };+ filter.accept("count", metric.size());+ filter.accept(MAX, metric.getMax());+ filter.accept(MIN, metric.getMin());+ filter.accept(MEAN, metric.getMean());+ filter.accept(STDDEV, metric.getStdDev());+ filter.accept(SUM, metric.getSum());+ if (!(metric.isEmpty() || skipAggregateValues)) {+ ew.putNoEx(VALUES, (MapWriter) ew1 -> {+ metric.getValues().forEach((k, v) -> {+ ew1.putNoEx(k, (MapWriter) ew2 -> {+ ew2.putNoEx("value", v.value);+ ew2.putNoEx("updateCount", v.updateCount.get());+ });+ });+ });}};- filter.accept("count", metric.size());- filter.accept(MAX, metric.getMax());- filter.accept(MIN, metric.getMin());- filter.accept(MEAN, metric.getMean());- filter.accept(STDDEV, metric.getStdDev());- filter.accept(SUM, metric.getSum());- if (!(metric.isEmpty() || skipAggregateValues)) {- Map<String, Object> values = new LinkedHashMap<>();- response.put(VALUES, values);- metric.getValues().forEach((k, v) -> {- Map<String, Object> map = new LinkedHashMap<>();- map.put("value", v.value);- map.put("updateCount", v.updateCount.get());- values.put(k, map);- });- }- if (!response.isEmpty()) {- consumer.accept(name, response);+ if (writer._size() > 0) {+ consumer.accept(name, writer);}}}@@ -354,24 +469,23 @@ public class MetricUtils {* only the selected (name "." key, value) pairs will be produced.* @param consumer consumer that accepts produced objects*/- static void convertHistogram(String name, Histogram histogram, PropertyFilter propertyFilter,+ static void convertHistogram(String name, Histogram histogram, Predicate<CharSequence> propertyFilter,boolean simple, String separator, BiConsumer<String, Object> consumer) {Snapshot snapshot = histogram.getSnapshot();if (simple) {- if (propertyFilter.accept(MEAN)) {+ if (propertyFilter.test(MEAN)) {consumer.accept(name + separator + MEAN, snapshot.getMean());}} else {- Map<String, Object> response = new LinkedHashMap<>();- String prop = "count";- if (propertyFilter.accept(prop)) {- response.put(prop, histogram.getCount());- }- // non-time based values- addSnapshot(response, snapshot, propertyFilter, false);- if (!response.isEmpty()) {- consumer.accept(name, response);- }+ MapWriter writer = ew -> {+ String prop = "count";+ if (propertyFilter.test(prop)) {+ ew.putNoEx(prop, histogram.getCount());+ }+ // non-time based values+ addSnapshot(ew, snapshot, propertyFilter, false);+ };+ consumer.accept(name, writer);}}@@ -385,10 +499,10 @@ public class MetricUtils {}// some snapshots represent time in ns, other snapshots represent raw values (eg. chunk size)- static void addSnapshot(Map<String, Object> response, Snapshot snapshot, PropertyFilter propertyFilter, boolean ms) {+ static void addSnapshot(MapWriter.EntryWriter ew, Snapshot snapshot, Predicate<CharSequence> propertyFilter, boolean ms) {BiConsumer<String, Object> filter = (k, v) -> {- if (propertyFilter.accept(k)) {- response.put(k, v);+ if (propertyFilter.test(k)) {+ ew.putNoEx(k, v);}};filter.accept((ms ? MIN_MS: MIN), nsToMs(ms, snapshot.getMin()));@@ -411,32 +525,50 @@ public class MetricUtils {* @param simple use simplified representation for complex metrics - instead of a (name, map)* only the selected (name "." key, value) pairs will be produced.* @param consumer consumer that accepts produced objects+ * @deprecated use {@link #convertTimer(String, Timer, Predicate, boolean, boolean, String, BiConsumer)} instead.*/+ @Deprecated(since = "8.7")public static void convertTimer(String name, Timer timer, PropertyFilter propertyFilter, boolean skipHistograms,+ boolean simple, String separator, BiConsumer<String, Object> consumer) {+ convertTimer(name, timer, PropertyFilter.toPredicate(propertyFilter),+ skipHistograms, simple, separator, consumer);+ }+ /**+ * Convert a {@link Timer} to a map.+ * @param name metric name+ * @param timer timer instance+ * @param propertyFilter limit what properties of a metric are returned+ * @param skipHistograms if true then discard the histogram part of the timer.+ * @param simple use simplified representation for complex metrics - instead of a (name, map)+ * only the selected (name "." key, value) pairs will be produced.+ * @param consumer consumer that accepts produced objects+ */+ public static void convertTimer(String name, Timer timer, Predicate<CharSequence> propertyFilter, boolean skipHistograms,boolean simple, String separator, BiConsumer<String, Object> consumer) {if (simple) {String prop = "meanRate";- if (propertyFilter.accept(prop)) {+ if (propertyFilter.test(prop)) {consumer.accept(name + separator + prop, timer.getMeanRate());}} else {- Map<String, Object> response = new LinkedHashMap<>();- BiConsumer<String,Object> filter = (k, v) -> {- if (propertyFilter.accept(k)) {- response.put(k, v);+ MapWriter writer = ew -> {+ BiConsumer<String,Object> filter = (k, v) -> {+ if (propertyFilter.test(k)) {+ ew.putNoEx(k, v);+ }+ };+ filter.accept("count", timer.getCount());+ filter.accept("meanRate", timer.getMeanRate());+ filter.accept("1minRate", timer.getOneMinuteRate());+ filter.accept("5minRate", timer.getFiveMinuteRate());+ filter.accept("15minRate", timer.getFifteenMinuteRate());+ if (!skipHistograms) {+ // time-based values in nanoseconds+ addSnapshot(ew, timer.getSnapshot(), propertyFilter, true);}};- filter.accept("count", timer.getCount());- filter.accept("meanRate", timer.getMeanRate());- filter.accept("1minRate", timer.getOneMinuteRate());- filter.accept("5minRate", timer.getFiveMinuteRate());- filter.accept("15minRate", timer.getFifteenMinuteRate());- if (!skipHistograms) {- // time-based values in nanoseconds- addSnapshot(response, timer.getSnapshot(), propertyFilter, true);- }- if (!response.isEmpty()) {- consumer.accept(name, response);+ if (writer._size() > 0) {+ consumer.accept(name, writer);}}}@@ -450,29 +582,49 @@ public class MetricUtils {* only the selected (name "." key, value) pairs will be produced.* @param consumer consumer that accepts produced objects*/- static void convertMeter(String name, Meter meter, PropertyFilter propertyFilter, boolean simple, String separator, BiConsumer<String, Object> consumer) {+ static void convertMeter(String name, Meter meter, Predicate<CharSequence> propertyFilter, boolean simple, String separator, BiConsumer<String, Object> consumer) {if (simple) {- if (propertyFilter.accept("count")) {+ if (propertyFilter.test("count")) {consumer.accept(name + separator + "count", meter.getCount());}} else {- Map<String, Object> response = new LinkedHashMap<>();- BiConsumer<String, Object> filter = (k, v) -> {- if (propertyFilter.accept(k)) {- response.put(k, v);- }+ MapWriter writer = ew -> {+ BiConsumer<String, Object> filter = (k, v) -> {+ if (propertyFilter.test(k)) {+ ew.putNoEx(k, v);+ }+ };+ filter.accept("count", meter.getCount());+ filter.accept("meanRate", meter.getMeanRate());+ filter.accept("1minRate", meter.getOneMinuteRate());+ filter.accept("5minRate", meter.getFiveMinuteRate());+ filter.accept("15minRate", meter.getFifteenMinuteRate());};- filter.accept("count", meter.getCount());- filter.accept("meanRate", meter.getMeanRate());- filter.accept("1minRate", meter.getOneMinuteRate());- filter.accept("5minRate", meter.getFiveMinuteRate());- filter.accept("15minRate", meter.getFifteenMinuteRate());- if (!response.isEmpty()) {- consumer.accept(name, response);+ if (writer._size() > 0) {+ consumer.accept(name, writer);}}}+ static void convertMapWriter(String name, MapWriter metric,+ Predicate<CharSequence> propertyFilter, boolean simple, boolean compact,+ String separator, BiConsumer<String, Object> consumer) {+ ConditionalKeyMapWriter filteredMetric = new ConditionalKeyMapWriter(metric, propertyFilter);+ if (compact || simple) {+ if (simple) {+ filteredMetric._forEachEntry((k, v) ->+ consumer.accept(name + separator + k, v));+ } else {+ if (filteredMetric._size() > 0) {+ consumer.accept(name, filteredMetric);+ }+ }+ } else {+ if (filteredMetric._size() > 0) {+ consumer.accept(name, (MapWriter) ew -> ew.putNoEx("value", filteredMetric));+ }+ }+ }/*** Convert a {@link Gauge}.* @param name metric name@@ -486,7 +638,7 @@ public class MetricUtils {*/static void convertGauge(String name,@SuppressWarnings({"rawtypes"})Gauge gauge,- PropertyFilter propertyFilter, boolean simple, boolean compact,+ Predicate<CharSequence> propertyFilter, boolean simple, boolean compact,String separator, BiConsumer<String, Object> consumer) {if (compact || simple) {Object o = gauge.getValue();@@ -494,20 +646,23 @@ public class MetricUtils {if (simple) {for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {String prop = entry.getKey().toString();- if (propertyFilter.accept(prop)) {+ if (propertyFilter.test(prop)) {consumer.accept(name + separator + prop, entry.getValue());}}} else {- Map<String, Object> val = new HashMap<>();- for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {- String prop = entry.getKey().toString();- if (propertyFilter.accept(prop)) {- val.put(prop, entry.getValue());+ boolean notEmpty = ((Map<?, ?>)o).entrySet().stream()+ .anyMatch(entry -> propertyFilter.test(entry.getKey().toString()));+ MapWriter writer = ew -> {+ for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {+ String prop = entry.getKey().toString();+ if (propertyFilter.test(prop)) {+ ew.putNoEx(prop, entry.getValue());+ }}- }- if (!val.isEmpty()) {- consumer.accept(name, val);+ };+ if (notEmpty) {+ consumer.accept(name, writer);}}} else {@@ -515,21 +670,24 @@ public class MetricUtils {}} else {Object o = gauge.getValue();- Map<String, Object> response = new LinkedHashMap<>();if (o instanceof Map) {- for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {- String prop = entry.getKey().toString();- if (propertyFilter.accept(prop)) {- response.put(prop, entry.getValue());- }- }- if (!response.isEmpty()) {- consumer.accept(name, Collections.singletonMap("value", response));+ boolean notEmpty = ((Map<?, ?>)o).entrySet().stream()+ .anyMatch(entry -> propertyFilter.test(entry.getKey().toString()));+ if (notEmpty) {+ consumer.accept(name, (MapWriter) ew -> {+ ew.putNoEx("value", (MapWriter) ew1 -> {+ for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {+ String prop = entry.getKey().toString();+ if (propertyFilter.test(prop)) {+ ew1.put(prop, entry.getValue());+ }+ }+ });+ });}} else {- if (propertyFilter.accept("value")) {- response.put("value", o);- consumer.accept(name, response);+ if (propertyFilter.test("value")) {+ consumer.accept(name, (MapWriter) ew -> ew.putNoEx("value", o));}}}@@ -542,14 +700,12 @@ public class MetricUtils {* @param compact if true then only return {@link Counter#getCount()}. If false* then return a map with a "count" field.*/- static void convertCounter(String name, Counter counter, PropertyFilter propertyFilter, boolean compact, BiConsumer<String, Object> consumer) {+ static void convertCounter(String name, Counter counter, Predicate<CharSequence> propertyFilter, boolean compact, BiConsumer<String, Object> consumer) {if (compact) {consumer.accept(name, counter.getCount());} else {- if (propertyFilter.accept("count")) {- Map<String, Object> response = new LinkedHashMap<>();- response.put("count", counter.getCount());- consumer.accept(name, response);+ if (propertyFilter.test("count")) {+ consumer.accept(name, (MapWriter) ew -> ew.putNoEx("count", counter.getCount()));}}}diff --git a/solr/core/src/test/org/apache/solr/core/MockInfoBean.java b/solr/core/src/test/org/apache/solr/core/MockInfoBean.javaindex ce73a02c2c7..1217283acbf 100644--- a/solr/core/src/test/org/apache/solr/core/MockInfoBean.java+++ b/solr/core/src/test/org/apache/solr/core/MockInfoBean.java@@ -50,7 +50,7 @@ class MockInfoBean implements SolrInfoBean, SolrMetricProducer {@Overridepublic void initializeMetrics(SolrMetricsContext parentContext, String scope) {solrMetricsContext = parentContext.getChildContext(this);- MetricsMap metricsMap = new MetricsMap((detailed, map) -> {+ MetricsMap metricsMap = new MetricsMap(map -> {map.put("Integer", 123);map.put("Double",567.534);map.put("Long", 32352463l);diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.javaindex 4ab33df5504..8f2bdbab253 100644--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java@@ -18,10 +18,12 @@package org.apache.solr.handler.admin;import java.util.Arrays;+import java.util.HashMap;import java.util.Map;import com.codahale.metrics.Counter;import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.common.MapWriter;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;@@ -84,15 +86,17 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {assertNotNull(nl);Object o = nl.get("SEARCHER.new.errors");assertNotNull(o); // counter type- assertTrue(o instanceof Map);+ assertTrue(o instanceof MapWriter);// response wasn't serialized so we get here whatever MetricUtils produced instead of NamedList- assertNotNull(((Map) o).get("count"));- assertEquals(0L, ((Map) nl.get("SEARCHER.new.errors")).get("count"));+ assertNotNull(((MapWriter) o)._get("count", null));+ assertEquals(0L, ((MapWriter) nl.get("SEARCHER.new.errors"))._get("count", null));nl = (NamedList) values.get("solr.node");assertNotNull(nl.get("CONTAINER.cores.loaded")); // int gauge- assertEquals(1, ((Map) nl.get("CONTAINER.cores.loaded")).get("value"));+ assertEquals(1, ((MapWriter) nl.get("CONTAINER.cores.loaded"))._get("value", null));assertNotNull(nl.get("ADMIN./admin/authorization.clientErrors")); // timer type- assertEquals(5, ((Map) nl.get("ADMIN./admin/authorization.clientErrors")).size());+ Map<String, Object> map = new HashMap<>();+ ((MapWriter) nl.get("ADMIN./admin/authorization.clientErrors")).toMap(map);+ assertEquals(5, map.size());resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", MetricsHandler.COMPACT_PARAM, "false", CommonParams.WT, "json", "group", "jvm,jetty"), resp);@@ -184,9 +188,9 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {values = (NamedList) values.get("solr.core.collection1");assertEquals(1, values.size());@SuppressWarnings({"rawtypes"})- Map m = (Map) values.get("CACHE.core.fieldCache");- assertNotNull(m);- assertNotNull(m.get("entries_count"));+ MapWriter writer = (MapWriter) values.get("CACHE.core.fieldCache");+ assertNotNull(writer);+ assertNotNull(writer._get("entries_count", null));resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", MetricsHandler.COMPACT_PARAM, "false", CommonParams.WT, "json", "group", "jvm", "prefix", "CONTAINER.cores"), resp);@@ -200,8 +204,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {values = resp.getValues();assertNotNull(values.get("metrics"));@SuppressWarnings({"rawtypes"})- SimpleOrderedMap map = (SimpleOrderedMap) values.get("metrics");- assertEquals(0, map.size());+ SimpleOrderedMap map1 = (SimpleOrderedMap) values.get("metrics");+ assertEquals(0, map1.size());handler.close();}@@ -243,9 +247,9 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {assertNotNull(nl);assertTrue(nl.size() > 0);nl.forEach((k, v) -> {- assertTrue(v instanceof Map);- @SuppressWarnings({"rawtypes"})- Map map = (Map) v;+ assertTrue(v instanceof MapWriter);+ Map<String, Object> map = new HashMap<>();+ ((MapWriter) v).toMap(map);assertTrue(map.size() > 2);});@@ -259,10 +263,10 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {assertNotNull(nl);assertTrue(nl.size() > 0);nl.forEach((k, v) -> {- assertTrue(v instanceof Map);- @SuppressWarnings({"rawtypes"})- Map map = (Map) v;- assertEquals(2, map.size());+ assertTrue(v instanceof MapWriter);+ Map<String, Object> map = new HashMap<>();+ ((MapWriter) v).toMap(map);+ assertEquals("k=" + k + ", v=" + map, 2, map.size());assertNotNull(map.get("inserts"));assertNotNull(map.get("size"));});@@ -281,15 +285,14 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {NamedList values = resp.getValues();Object val = values.findRecursive("metrics", key1);assertNotNull(val);- assertTrue(val instanceof Map);- assertTrue(((Map) val).size() >= 2);+ assertTrue(val instanceof MapWriter);+ assertTrue(((MapWriter)val)._size() >= 2);String key2 = "solr.core.collection1:CACHE.core.fieldCache:entries_count";resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",MetricsHandler.KEY_PARAM, key2), resp);- values = resp.getValues();- val = values.findRecursive("metrics", key2);+ val = resp.getValues()._get("metrics/" + key2, null);assertNotNull(val);assertTrue(val instanceof Number);@@ -297,8 +300,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",MetricsHandler.KEY_PARAM, key3), resp);- values = resp.getValues();- val = values.findRecursive("metrics", key3);++ val = resp.getValues()._get( "metrics/" + key3, null);assertNotNull(val);assertTrue(val instanceof Number);assertEquals(3, ((Number) val).intValue());@@ -307,20 +310,20 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",MetricsHandler.KEY_PARAM, key1, MetricsHandler.KEY_PARAM, key2, MetricsHandler.KEY_PARAM, key3), resp);- values = resp.getValues();- val = values.findRecursive("metrics", key1);++ val = resp.getValues()._get( "metrics/" + key1, null);assertNotNull(val);- val = values.findRecursive("metrics", key2);+ val = resp.getValues()._get( "metrics/" + key2, null);assertNotNull(val);- val = values.findRecursive("metrics", key3);+ val = resp.getValues()._get( "metrics/" + key3, null);assertNotNull(val);String key4 = "solr.core.collection1:QUERY./select.requestTimes:1minRate";resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",MetricsHandler.KEY_PARAM, key4), resp);- values = resp.getValues();- val = values.findRecursive("metrics", key4);+ // the key contains a slash, need explicit list of path elements+ val = resp.getValues()._get(Arrays.asList("metrics", key4), null);assertNotNull(val);assertTrue(val instanceof Number);@@ -460,7 +463,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {@Overridepublic void initializeMetrics(SolrMetricsContext parentContext, String scope) {super.initializeMetrics(parentContext, scope);- MetricsMap metrics = new MetricsMap((detailed, map) -> map.putAll(gaugevals));+ MetricsMap metrics = new MetricsMap(map -> gaugevals.forEach((k, v) -> map.putNoEx(k, v)));solrMetricsContext.gauge(metrics, true, "dumphandlergauge", getCategory().toString(), scope);diff --git a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.javaindex f9595e1e576..e95bea46831 100644--- a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java+++ b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java@@ -31,8 +31,11 @@ import com.codahale.metrics.MetricRegistry;import com.codahale.metrics.Snapshot;import com.codahale.metrics.Timer;import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.common.MapWriter;import org.apache.solr.common.util.NamedList;import org.apache.solr.metrics.AggregateMetric;+import org.apache.solr.metrics.MetricsMap;+import org.apache.solr.metrics.SolrMetricManager;import org.junit.Test;public class MetricUtilsTest extends SolrTestCaseJ4 {@@ -48,8 +51,8 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {}// obtain timer metricsMap<String,Object> map = new HashMap<>();- MetricUtils.convertTimer("", timer, MetricUtils.PropertyFilter.ALL, false, false, ".", (k, v) -> {- map.putAll((Map<String,Object>)v);+ MetricUtils.convertTimer("", timer, MetricUtils.ALL_PROPERTIES, false, false, ".", (k, v) -> {+ ((MapWriter) v).toMap(map);});@SuppressWarnings({"rawtypes"})NamedList lst = new NamedList(map);@@ -103,10 +106,26 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {registry.register("gauge", gauge);Gauge<Long> error = () -> {throw new InternalError("Memory Pool not found error");};registry.register("memory.expected.error", error);++ MetricsMap metricsMapWithMap = new MetricsMap((detailed, map) -> {+ map.put("foo", "bar");+ });+ registry.register("mapWithMap", metricsMapWithMap);+ MetricsMap metricsMap = new MetricsMap(map -> {+ map.putNoEx("foo", "bar");+ });+ registry.register("map", metricsMap);++ SolrMetricManager.GaugeWrapper<Map<String,Object>> gaugeWrapper = new SolrMetricManager.GaugeWrapper<>(metricsMap, "foo-tag");+ registry.register("wrappedGauge", gaugeWrapper);+MetricUtils.toMaps(registry, Collections.singletonList(MetricFilter.ALL), MetricFilter.ALL,- MetricUtils.PropertyFilter.ALL, false, false, false, false, (k, o) -> {+ MetricUtils.ALL_PROPERTIES, false, false, false, false, (k, o) -> {@SuppressWarnings({"rawtypes"})- Map v = (Map)o;+ Map<String, Object> v = new HashMap<>();+ if (o != null) {+ ((MapWriter) o).toMap(v);+ }if (k.startsWith("counter")) {assertEquals(1L, v.get("count"));} else if (k.startsWith("gauge")) {@@ -139,12 +158,16 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {assertEquals(0D, v.get("max"));assertEquals(0D, v.get("mean"));} else if (k.startsWith("memory.expected.error")) {- assertNull(v);+ assertTrue(v.isEmpty());+ } else if (k.startsWith("map") || k.startsWith("wrapped")) {+ assertNotNull(v.toString(), v.get("value"));+ assertTrue(v.toString(), v.get("value") instanceof Map);+ assertEquals(v.toString(), "bar", ((Map) v.get("value")).get("foo"));}});// test compact formatMetricUtils.toMaps(registry, Collections.singletonList(MetricFilter.ALL), MetricFilter.ALL,- MetricUtils.PropertyFilter.ALL, false, false, true, false, (k, o) -> {+ MetricUtils.ALL_PROPERTIES, false, false, true, false, (k, o) -> {if (k.startsWith("counter")) {assertTrue(o instanceof Long);assertEquals(1L, o);@@ -152,25 +175,25 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {assertTrue(o instanceof String);assertEquals("foobar", o);} else if (k.startsWith("timer")) {- assertTrue(o instanceof Map);- @SuppressWarnings({"rawtypes"})- Map v = (Map)o;+ assertTrue(o instanceof MapWriter);+ Map<String, Object> v = new HashMap<>();+ ((MapWriter) o).toMap(v);assertEquals(1L, v.get("count"));assertTrue(((Number)v.get("min_ms")).intValue() > 100);} else if (k.startsWith("meter")) {- assertTrue(o instanceof Map);- @SuppressWarnings({"rawtypes"})- Map v = (Map)o;+ assertTrue(o instanceof MapWriter);+ Map<String, Object> v = new HashMap<>();+ ((MapWriter) o).toMap(v);assertEquals(1L, v.get("count"));} else if (k.startsWith("histogram")) {- assertTrue(o instanceof Map);- @SuppressWarnings({"rawtypes"})- Map v = (Map)o;+ assertTrue(o instanceof MapWriter);+ Map<String, Object> v = new HashMap<>();+ ((MapWriter) o).toMap(v);assertEquals(1L, v.get("count"));} else if (k.startsWith("aggregate1")) {- assertTrue(o instanceof Map);- @SuppressWarnings({"rawtypes"})- Map v = (Map)o;+ assertTrue(o instanceof MapWriter);+ Map<String, Object> v = new HashMap<>();+ ((MapWriter) o).toMap(v);assertEquals(4, v.get("count"));Map<String, Object> values = (Map<String, Object>)v.get("values");assertNotNull(values);@@ -182,9 +205,9 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {assertEquals(-2, update.get("value"));assertEquals(2, update.get("updateCount"));} else if (k.startsWith("aggregate2")) {- assertTrue(o instanceof Map);- @SuppressWarnings({"rawtypes"})- Map v = (Map)o;+ assertTrue(o instanceof MapWriter);+ Map<String, Object> v = new HashMap<>();+ ((MapWriter) o).toMap(v);assertEquals(2, v.get("count"));Map<String, Object> values = (Map<String, Object>)v.get("values");assertNotNull(values);@@ -197,9 +220,15 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {assertEquals(1, update.get("updateCount"));} else if (k.startsWith("memory.expected.error")) {assertNull(o);+ } else if (k.startsWith("map") || k.startsWith("wrapped")) {+ assertTrue(o instanceof MapWriter);+ MapWriter writer = (MapWriter) o;+ assertEquals(1, writer._size());+ assertEquals("bar", writer._get("foo", null));} else {- @SuppressWarnings({"rawtypes"})- Map v = (Map)o;+ assertTrue(o instanceof MapWriter);+ Map<String, Object> v = new HashMap<>();+ ((MapWriter) o).toMap(v);assertEquals(1L, v.get("count"));}});diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java b/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.javaindex c5f61648f5a..5e6dfeb2e25 100644--- a/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java@@ -50,6 +50,11 @@ public class MapWriterMap implements MapWriter {}+ @Override+ public int _size() {+ return delegate.size();+ }+@Override@SuppressWarnings("unchecked")public Map<String,Object> toMap(Map<String, Object> map) {diff --git a/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java b/solr/solrj/src/java/org/apache/solr/common/NavigableObject.javaindex 22dca2bbe40..4f88de5c1d5 100644--- a/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java+++ b/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java@@ -86,4 +86,10 @@ public interface NavigableObject {Object v = Utils.getObjectByPath(this, false, path);return v == null ? def : String.valueOf(v);}++ default int _size() {+ int[] size = new int[1];+ _forEachEntry((k, v) -> size[0]++);+ return size[0];+ }}diff --git a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.javaindex 9156619cde8..e5ae8e23b20 100644--- a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java+++ b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java@@ -889,6 +889,11 @@ public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry}}+ @Override+ public int _size() {+ return size();+ }+@Overridepublic void forEachEntry(BiConsumer<String, ? super T> fun) {forEach(fun);diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.javaindex 3cb817b9d62..c05ce233f0b 100644--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java@@ -227,9 +227,48 @@ public class Utils {@Override@SuppressWarnings({"rawtypes"})public void handleUnknownClass(Object o) {+ // avoid materializing MapWriter / IteratorWriter to Map / List+ // instead serialize them directlyif (o instanceof MapWriter) {- Map m = ((MapWriter) o).toMap(new LinkedHashMap<>());- write(m);+ MapWriter mapWriter = (MapWriter) o;+ startObject();+ final boolean[] first = new boolean[1];+ first[0] = true;+ mapWriter._forEachEntry((k, v) -> {+ if (first[0]) {+ first[0] = false;+ } else {+ writeValueSeparator();+ }+ indent();+ writeString(k.toString());+ writeNameSeparator();+ write(v);+ });+ endObject();+ } else if (o instanceof IteratorWriter) {+ IteratorWriter iteratorWriter = (IteratorWriter) o;+ startArray();+ final boolean[] first = new boolean[1];+ first[0] = true;+ try {+ iteratorWriter.writeIter(new IteratorWriter.ItemWriter() {+ @Override+ public IteratorWriter.ItemWriter add(Object o) throws IOException {+ if (first[0]) {+ first[0] = false;+ } else {+ writeValueSeparator();+ }+ indent();+ write(o);+ return this;+ }+ });+ } catch (IOException e) {+ throw new RuntimeException("this should never happen", e);+ }+ endArray();} else {super.handleUnknownClass(o);}@@ -239,13 +278,13 @@ public class Utils {public static byte[] toJSON(Object o) {if (o == null) return new byte[0];CharArr out = new CharArr();- if (!(o instanceof List) && !(o instanceof Map)) {- if (o instanceof MapWriter) {- o = ((MapWriter) o).toMap(new LinkedHashMap<>());- } else if (o instanceof IteratorWriter) {- o = ((IteratorWriter) o).toList(new ArrayList<>());- }- }+// if (!(o instanceof List) && !(o instanceof Map)) {+// if (o instanceof MapWriter) {+// o = ((MapWriter) o).toMap(new LinkedHashMap<>());+// } else if (o instanceof IteratorWriter) {+// o = ((IteratorWriter) o).toList(new ArrayList<>());+// }+// }new MapWriterJSONWriter(out, 2).write(o); // indentation by defaultreturn toUTF8(out);}