Apache Solr open-source search software
SOLR-15416: Remove metrics history collection (and MetricsHistoryHandler).
Andrzej Bialeckia4618acbe8cf
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex a19822b1127..d33b280ef41 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -299,6 +299,8 @@ Other Changes* SOLR-15409: Upgrade to Zookeeper 3.7.0 (Mike Drob)+* SOLR-15416: Remove metrics history collection (and MetricsHistoryHandler). (ab)+Bug Fixes---------------------* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order executiondiff --git a/solr/core/build.gradle b/solr/core/build.gradleindex 6eba46c710e..7cafbadfd13 100644--- a/solr/core/build.gradle+++ b/solr/core/build.gradle@@ -95,8 +95,6 @@ dependencies {implementation 'org.codehaus.janino:commons-compiler'implementation 'org.codehaus.janino:janino'- implementation 'org.rrd4j:rrd4j'-// For faster XML processing than the JDKimplementation ('org.codehaus.woodstox:woodstox-core-asl', {exclude group: "javax.xml.stream", module: "stax-api"diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.javaindex df7895c06e7..6f077025f8e 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java@@ -43,11 +43,8 @@ import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;-import org.apache.solr.core.SolrInfoBean;import org.apache.solr.core.snapshots.SolrSnapshotManager;import org.apache.solr.handler.admin.ConfigSetsHandler;-import org.apache.solr.handler.admin.MetricsHistoryHandler;-import org.apache.solr.metrics.SolrMetricManager;import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -100,8 +97,6 @@ public class DeleteCollectionCmd implements CollApiCmds.CollectionApiCommand {assignStrategy.verifyDeleteCollection(ccc.getSolrCloudManager(), coll);}- final boolean deleteHistory = message.getBool(CoreAdminParams.DELETE_METRICS_HISTORY, true);-boolean removeCounterNode = true;try {// Remove the snapshots meta-data for this collection in ZK. Deleting actual index files@@ -117,19 +112,10 @@ public class DeleteCollectionCmd implements CollApiCmds.CollectionApiCommand {return;}}- // remove collection-level metrics history- if (deleteHistory) {- MetricsHistoryHandler historyHandler = ccc.getCoreContainer().getMetricsHistoryHandler();- if (historyHandler != null) {- String registry = SolrMetricManager.getRegistryName(SolrInfoBean.Group.collection, collection);- historyHandler.removeHistory(registry);- }- }ModifiableSolrParams params = new ModifiableSolrParams();params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.UNLOAD.toString());params.set(CoreAdminParams.DELETE_INSTANCE_DIR, true);params.set(CoreAdminParams.DELETE_DATA_DIR, true);- params.set(CoreAdminParams.DELETE_METRICS_HISTORY, deleteHistory);String asyncId = message.getStr(ASYNC);diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.javaindex 4e25dbd6e76..b9b43b181fb 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java@@ -268,7 +268,6 @@ public class DeleteReplicaCmd implements CollectionApiCommand {params.set(CoreAdminParams.DELETE_INDEX, message.getBool(CoreAdminParams.DELETE_INDEX, true));params.set(CoreAdminParams.DELETE_INSTANCE_DIR, message.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, true));params.set(CoreAdminParams.DELETE_DATA_DIR, message.getBool(CoreAdminParams.DELETE_DATA_DIR, true));- params.set(CoreAdminParams.DELETE_METRICS_HISTORY, message.getBool(CoreAdminParams.DELETE_METRICS_HISTORY, true));boolean isLive = ccc.getZkStateReader().getClusterState().getLiveNodes().contains(replica.getNodeName());final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc);diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.javaindex 0c9a03c0b2d..50fe79cf011 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java@@ -53,7 +53,6 @@ import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.params.CollectionParams;import org.apache.solr.common.params.CommonAdminParams;import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;@@ -283,8 +282,7 @@ public class ReindexCollectionCmd implements CollApiCmds.CollectionApiCommand {// delete the checkpoint collectioncmd = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower(),- CommonParams.NAME, chkCollection,- CoreAdminParams.DELETE_METRICS_HISTORY, "true"+ CommonParams.NAME, chkCollection);new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults);CollectionHandlingUtils.checkResults("deleting old checkpoint collection " + chkCollection, cmdResults, true);@@ -458,8 +456,7 @@ public class ReindexCollectionCmd implements CollApiCmds.CollectionApiCommand {log.debug("- deleting {}", chkCollection);cmd = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower(),- CommonParams.NAME, chkCollection,- CoreAdminParams.DELETE_METRICS_HISTORY, "true"+ CommonParams.NAME, chkCollection);cmdResults = new NamedList<>();new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults);@@ -471,8 +468,7 @@ public class ReindexCollectionCmd implements CollApiCmds.CollectionApiCommand {cmd = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower(),CommonParams.NAME, collection,- FOLLOW_ALIASES, "false",- CoreAdminParams.DELETE_METRICS_HISTORY, "true"+ FOLLOW_ALIASES, "false");cmdResults = new NamedList<>();new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults);@@ -795,8 +791,7 @@ public class ReindexCollectionCmd implements CollApiCmds.CollectionApiCommand {ZkNodeProps cmd = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower(),CommonParams.NAME, targetCollection,- FOLLOW_ALIASES, "false",- CoreAdminParams.DELETE_METRICS_HISTORY, "true"+ FOLLOW_ALIASES, "false");new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults);CollectionHandlingUtils.checkResults("CLEANUP: deleting target collection " + targetCollection, cmdResults, false);@@ -808,8 +803,7 @@ public class ReindexCollectionCmd implements CollApiCmds.CollectionApiCommand {ZkNodeProps cmd = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower(),CommonParams.NAME, chkCollection,- FOLLOW_ALIASES, "false",- CoreAdminParams.DELETE_METRICS_HISTORY, "true"+ FOLLOW_ALIASES, "false");cmdResults = new NamedList<>();new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults);diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex 998a8229afc..997cec9fb33 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -28,7 +28,6 @@ import java.util.ArrayList;import java.util.Arrays;import java.util.Collections;import java.util.Date;-import java.util.HashMap;import java.util.LinkedHashSet;import java.util.List;import java.util.Locale;@@ -59,10 +58,6 @@ import org.apache.lucene.index.IndexWriter;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.store.Directory;import org.apache.solr.api.ContainerPluginsRegistry;-import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.cloud.SolrCloudManager;-import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;-import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.impl.HttpClientUtil;import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;import org.apache.solr.client.solrj.impl.SolrHttpClientContextBuilder;@@ -109,9 +104,7 @@ import org.apache.solr.handler.admin.ContainerPluginsApi;import org.apache.solr.handler.admin.CoreAdminHandler;import org.apache.solr.handler.admin.HealthCheckHandler;import org.apache.solr.handler.admin.InfoHandler;-import org.apache.solr.handler.admin.MetricsCollectorHandler;import org.apache.solr.handler.admin.MetricsHandler;-import org.apache.solr.handler.admin.MetricsHistoryHandler;import org.apache.solr.handler.admin.SecurityConfHandler;import org.apache.solr.handler.admin.SecurityConfHandlerLocal;import org.apache.solr.handler.admin.SecurityConfHandlerZk;@@ -154,7 +147,6 @@ import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATimport static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH;import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;-import static org.apache.solr.common.params.CommonParams.METRICS_HISTORY_PATH;import static org.apache.solr.common.params.CommonParams.METRICS_PATH;import static org.apache.solr.common.params.CommonParams.ZK_PATH;import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH;@@ -258,10 +250,6 @@ public class CoreContainer {protected MetricsHandler metricsHandler;- protected volatile MetricsHistoryHandler metricsHistoryHandler;-- protected volatile MetricsCollectorHandler metricsCollectorHandler;-private volatile SolrClientCache solrClientCache;private final ObjectCache objectCache = new ObjectCache();@@ -646,10 +634,6 @@ public class CoreContainer {return metricsHandler;}- public MetricsHistoryHandler getMetricsHistoryHandler() {- return metricsHistoryHandler;- }-/** Never null but may implement {@link NoopTracer}. */public Tracer getTracer() {return tracer;@@ -806,19 +790,10 @@ public class CoreContainer {infoHandler = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class);coreAdminHandler = createHandler(CORES_HANDLER_PATH, cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);- // metricsHistoryHandler uses metricsHandler, so create it firstmetricsHandler = new MetricsHandler(this);containerHandlers.put(METRICS_PATH, metricsHandler);metricsHandler.initializeMetrics(solrMetricsContext, METRICS_PATH);- createMetricsHistoryHandler();-- if (cfg.getMetricsConfig().isEnabled()) {- metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(), MetricsCollectorHandler.class);- // may want to add some configuration here in the future- metricsCollectorHandler.init(null);- }-containerHandlers.put(AUTHZ_PATH, securityConfHandler);securityConfHandler.initializeMetrics(solrMetricsContext, AUTHZ_PATH);containerHandlers.put(AUTHC_PATH, securityConfHandler);@@ -988,47 +963,6 @@ public class CoreContainer {}}- // MetricsHistoryHandler supports both cloud and standalone configs- @SuppressWarnings({"unchecked"})- private void createMetricsHistoryHandler() {- PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();- if (plugin != null && MetricsConfig.NOOP_IMPL_CLASS.equals(plugin.className)) {- // still create the handler but it will be disabled- plugin = null;- }- Map<String, Object> initArgs;- if (plugin != null && plugin.initArgs != null) {- initArgs = plugin.initArgs.asMap(5);- initArgs.putIfAbsent(MetricsHistoryHandler.ENABLE_PROP, plugin.isEnabled());- } else {- initArgs = new HashMap<>();- }- String name;- SolrCloudManager cloudManager;- SolrClient client;- if (isZooKeeperAware()) {- name = getZkController().getNodeName();- cloudManager = getZkController().getSolrCloudManager();- client = new CloudSolrClient.Builder(Collections.singletonList(getZkController().getZkServerAddress()), Optional.empty())- .withSocketTimeout(30000).withConnectionTimeout(15000)- .withHttpClient(updateShardHandler.getDefaultHttpClient()).build();- } else {- name = getNodeConfig().getNodeName();- if (name == null || name.isEmpty()) {- name = "localhost";- }- cloudManager = null;- client = new EmbeddedSolrServer(this, null);- // enable local metrics unless specifically set otherwise- initArgs.putIfAbsent(MetricsHistoryHandler.ENABLE_NODES_PROP, true);- initArgs.putIfAbsent(MetricsHistoryHandler.ENABLE_REPLICAS_PROP, true);- }- metricsHistoryHandler = new MetricsHistoryHandler(name, metricsHandler,- client, cloudManager, initArgs);- containerHandlers.put(METRICS_HISTORY_PATH, metricsHistoryHandler);- metricsHistoryHandler.initializeMetrics(solrMetricsContext, METRICS_HISTORY_PATH);- }-public void securityNodeChanged() {log.info("Security node changed, reloading security.json");reloadSecurityProperties();@@ -1165,11 +1099,6 @@ public class CoreContainer {replayUpdatesExecutor.shutdownAndAwaitTermination();});- if (metricsHistoryHandler != null) {- metricsHistoryHandler.close();- IOUtils.closeQuietly(metricsHistoryHandler.getSolrClient());- }-if (metricManager != null) {metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.jvm));diff --git a/solr/core/src/java/org/apache/solr/core/MetricsConfig.java b/solr/core/src/java/org/apache/solr/core/MetricsConfig.javaindex 17449fd1636..1e17c6a35aa 100644--- a/solr/core/src/java/org/apache/solr/core/MetricsConfig.java+++ b/solr/core/src/java/org/apache/solr/core/MetricsConfig.java@@ -31,7 +31,6 @@ public class MetricsConfig {private final PluginInfo meterSupplier;private final PluginInfo timerSupplier;private final PluginInfo histogramSupplier;- private final PluginInfo historyHandler;private final Object nullNumber;private final Object notANumber;private final Object nullString;@@ -42,7 +41,6 @@ public class MetricsConfig {PluginInfo[] metricReporters, Set<String> hiddenSysProps,PluginInfo counterSupplier, PluginInfo meterSupplier,PluginInfo timerSupplier, PluginInfo histogramSupplier,- PluginInfo historyHandler,Object nullNumber, Object notANumber, Object nullString, Object nullObject) {this.enabled = enabled;this.metricReporters = metricReporters;@@ -51,7 +49,6 @@ public class MetricsConfig {this.meterSupplier = meterSupplier;this.timerSupplier = timerSupplier;this.histogramSupplier = histogramSupplier;- this.historyHandler = historyHandler;this.nullNumber = nullNumber;this.notANumber = notANumber;this.nullString = nullString;@@ -136,14 +133,6 @@ public class MetricsConfig {}}- public PluginInfo getHistoryHandler() {- if (enabled) {- return historyHandler;- } else {- return NO_OP_PLUGIN;- }- }-public static class MetricsConfigBuilder {private PluginInfo[] metricReporterPlugins = new PluginInfo[0];private Set<String> hiddenSysProps = new HashSet<>();@@ -151,7 +140,6 @@ public class MetricsConfig {private PluginInfo meterSupplier;private PluginInfo timerSupplier;private PluginInfo histogramSupplier;- private PluginInfo historyHandler;private Object nullNumber = null;private Object notANumber = null;private Object nullString = null;@@ -201,11 +189,6 @@ public class MetricsConfig {return this;}- public MetricsConfigBuilder setHistoryHandler(PluginInfo info) {- this.historyHandler = info;- return this;- }-public MetricsConfigBuilder setNullNumber(Object nullNumber) {this.nullNumber = nullNumber;return this;@@ -228,7 +211,7 @@ public class MetricsConfig {public MetricsConfig build() {return new MetricsConfig(enabled, metricReporterPlugins, hiddenSysProps, counterSupplier, meterSupplier,- timerSupplier, histogramSupplier, historyHandler,+ timerSupplier, histogramSupplier,nullNumber, notANumber, nullString, nullObject);}diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.javaindex 24354fb0d28..320c658a82a 100644--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java@@ -599,10 +599,6 @@ public class SolrXmlConfig {if (node != null) {builder = builder.setHistogramSupplier(new PluginInfo(node, "histogramSupplier", false, false));}- node = config.getNode("solr/metrics/history", false);- if (node != null) {- builder = builder.setHistoryHandler(new PluginInfo(node, "history", false, false));- }node = config.getNode("solr/metrics/missingValues", false);;if (node != null) {NamedList<Object> missingValues = DOMUtil.childNodesToNamedList(node);diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.javaindex 5c5499278ba..8a570569b74 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java@@ -733,7 +733,6 @@ public class CollectionsHandler extends RequestHandlerBase implements PermissionDELETE_INDEX,DELETE_DATA_DIR,DELETE_INSTANCE_DIR,- DELETE_METRICS_HISTORY,FOLLOW_ALIASES);return map;}),@@ -771,7 +770,6 @@ public class CollectionsHandler extends RequestHandlerBase implements PermissionDELETE_INDEX,DELETE_DATA_DIR,DELETE_INSTANCE_DIR,- DELETE_METRICS_HISTORY,COUNT_PROP, REPLICA_PROP,SHARD_ID_PROP,ONLY_IF_DOWN,diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.javaindex 4f09c8513da..6396024dc29 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java@@ -24,7 +24,6 @@ import java.util.Map;import java.util.Optional;import org.apache.commons.lang3.StringUtils;-import org.apache.solr.cloud.CloudDescriptor;import org.apache.solr.cloud.ZkController;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;@@ -33,16 +32,13 @@ import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.PropertiesUtil;import org.apache.solr.common.util.SimpleOrderedMap;-import org.apache.solr.common.util.Utils;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.CoreDescriptor;import org.apache.solr.core.SolrCore;-import org.apache.solr.core.SolrInfoBean;import org.apache.solr.core.snapshots.SolrSnapshotManager;import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;import org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminOp;-import org.apache.solr.metrics.SolrMetricManager;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.update.UpdateLog;import org.apache.solr.util.NumberUtils;@@ -103,27 +99,8 @@ enum CoreAdminOperation implements CoreAdminOp {boolean deleteIndexDir = params.getBool(CoreAdminParams.DELETE_INDEX, false);boolean deleteDataDir = params.getBool(CoreAdminParams.DELETE_DATA_DIR, false);boolean deleteInstanceDir = params.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, false);- boolean deleteMetricsHistory = params.getBool(CoreAdminParams.DELETE_METRICS_HISTORY, false);CoreDescriptor cdescr = it.handler.coreContainer.getCoreDescriptor(cname);it.handler.coreContainer.unload(cname, deleteIndexDir, deleteDataDir, deleteInstanceDir);- if (deleteMetricsHistory) {- MetricsHistoryHandler historyHandler = it.handler.coreContainer.getMetricsHistoryHandler();- if (historyHandler != null) {- CloudDescriptor cd = cdescr != null ? cdescr.getCloudDescriptor() : null;- String registry;- if (cd == null) {- registry = SolrMetricManager.getRegistryName(SolrInfoBean.Group.core, cname);- } else {- String replicaName = Utils.parseMetricsReplicaName(cd.getCollectionName(), cname);- registry = SolrMetricManager.getRegistryName(SolrInfoBean.Group.core,- cd.getCollectionName(),- cd.getShardId(),- replicaName);- }- historyHandler.checkSystemCollection();- historyHandler.removeHistory(registry);- }- }assert TestInjection.injectNonExistentCoreExceptionAfterUnload(cname);}),diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.javadeleted file mode 100644index cbec386424e..00000000000--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java+++ /dev/null@@ -1,995 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.handler.admin;--import javax.imageio.ImageIO;-import java.awt.Color;-import java.awt.image.BufferedImage;-import java.io.ByteArrayInputStream;-import java.io.ByteArrayOutputStream;-import java.io.Closeable;-import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.net.MalformedURLException;-import java.net.URI;-import java.net.URL;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Collection;-import java.util.Collections;-import java.util.HashMap;-import java.util.HashSet;-import java.util.Iterator;-import java.util.LinkedHashMap;-import java.util.List;-import java.util.Locale;-import java.util.Map;-import java.util.NoSuchElementException;-import java.util.Set;-import java.util.TimeZone;-import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.Executors;-import java.util.concurrent.ScheduledThreadPoolExecutor;-import java.util.concurrent.TimeUnit;-import java.util.concurrent.atomic.AtomicBoolean;-import java.util.concurrent.atomic.AtomicReference;-import java.util.concurrent.atomic.DoubleAdder;-import java.util.function.Function;-import java.util.stream.Collectors;-import java.util.stream.Stream;--import com.google.common.annotations.VisibleForTesting;-import org.apache.solr.api.Api;-import org.apache.solr.api.ApiBag;-import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.SolrQuery;-import org.apache.solr.client.solrj.SolrRequest;-import org.apache.solr.client.solrj.cloud.NodeStateProvider;-import org.apache.solr.client.solrj.cloud.SolrCloudManager;-import org.apache.solr.client.solrj.cloud.VersionedData;-import org.apache.solr.client.solrj.impl.HttpClientUtil;-import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.Variable;-import org.apache.solr.cloud.LeaderElector;-import org.apache.solr.cloud.Overseer;-import org.apache.solr.common.SolrException;-import org.apache.solr.common.cloud.ClusterState;-import org.apache.solr.common.cloud.DocCollection;-import org.apache.solr.common.cloud.Replica;-import org.apache.solr.common.cloud.Slice;-import org.apache.solr.common.cloud.ZkNodeProps;-import org.apache.solr.common.params.CollectionAdminParams;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.params.SolrParams;-import org.apache.solr.common.util.Base64;-import org.apache.solr.common.util.ExecutorUtil;-import org.apache.solr.common.util.JavaBinCodec;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.common.util.Pair;-import org.apache.solr.common.util.SimpleOrderedMap;-import org.apache.solr.common.util.TimeSource;-import org.apache.solr.common.util.Utils;-import org.apache.solr.handler.RequestHandlerBase;-import org.apache.solr.metrics.SolrMetricManager;-import org.apache.solr.metrics.rrd.SolrRrdBackendFactory;-import org.apache.solr.request.SolrQueryRequest;-import org.apache.solr.response.SolrQueryResponse;-import org.apache.solr.security.AuthorizationContext;-import org.apache.solr.security.PermissionNameProvider;-import org.apache.solr.common.util.SolrNamedThreadFactory;-import org.apache.zookeeper.KeeperException;-import org.rrd4j.ConsolFun;-import org.rrd4j.DsType;-import org.rrd4j.core.ArcDef;-import org.rrd4j.core.Archive;-import org.rrd4j.core.Datasource;-import org.rrd4j.core.DsDef;-import org.rrd4j.core.FetchData;-import org.rrd4j.core.FetchRequest;-import org.rrd4j.core.RrdDb;-import org.rrd4j.core.RrdDef;-import org.rrd4j.core.Sample;-import org.rrd4j.graph.RrdGraph;-import org.rrd4j.graph.RrdGraphDef;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import static java.util.stream.Collectors.toMap;-import static org.apache.solr.common.cloud.ZkStateReader.URL_SCHEME;-import static org.apache.solr.common.params.CommonParams.ID;--/**- * Collects metrics from all nodes in the system on a regular basis in a background thread.- * @since 7.4- */-public class MetricsHistoryHandler extends RequestHandlerBase implements PermissionNameProvider, Closeable {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- public static final List<String> DEFAULT_CORE_COUNTERS = new ArrayList<>();- public static final List<String> DEFAULT_CORE_GAUGES = new ArrayList<>();- public static final List<String> DEFAULT_NODE_GAUGES = new ArrayList<>();- public static final List<String> DEFAULT_JVM_GAUGES = new ArrayList<>();-- public static final String NUM_SHARDS_KEY = "numShards";- public static final String NUM_REPLICAS_KEY = "numReplicas";- public static final String NUM_NODES_KEY = "numNodes";-- public static final List<String> DEFAULT_COLLECTION_GAUGES = new ArrayList<>();-- static {- DEFAULT_JVM_GAUGES.add("memory.heap.used");- DEFAULT_JVM_GAUGES.add("os.processCpuLoad");- DEFAULT_JVM_GAUGES.add("os.systemLoadAverage");-- DEFAULT_NODE_GAUGES.add("CONTAINER.fs.coreRoot.usableSpace");-- DEFAULT_CORE_GAUGES.add(Variable.CORE_IDX.metricsAttribute);-- DEFAULT_CORE_COUNTERS.add("QUERY./select.requests");- DEFAULT_CORE_COUNTERS.add("UPDATE./update.requests");-- DEFAULT_COLLECTION_GAUGES.add(NUM_SHARDS_KEY);- DEFAULT_COLLECTION_GAUGES.add(NUM_REPLICAS_KEY);- }-- public static final String COLLECT_PERIOD_PROP = "collectPeriod";- public static final String SYNC_PERIOD_PROP = "syncPeriod";- public static final String ENABLE_PROP = "enable";- public static final String ENABLE_REPLICAS_PROP = "enableReplicas";- public static final String ENABLE_NODES_PROP = "enableNodes";-- public static final int DEFAULT_COLLECT_PERIOD = 60;- public static final String URI_PREFIX = "solr:";-- private final SolrRrdBackendFactory factory;- private final String nodeName;- private final SolrClient solrClient;- private final MetricsHandler metricsHandler;- private final SolrCloudManager cloudManager;- private final TimeSource timeSource;- private final int collectPeriod;- private final Map<String, List<String>> counters = new HashMap<>();- private final Map<String, List<String>> gauges = new HashMap<>();- private final String overseerUrlScheme;-- private final Map<String, RrdDb> knownDbs = new ConcurrentHashMap<>();-- private ScheduledThreadPoolExecutor collectService;- private boolean logMissingCollection = true;- private boolean enable;- private boolean enableReplicas;- private boolean enableNodes;- private String versionString;-- public MetricsHistoryHandler(String nodeName, MetricsHandler metricsHandler,- SolrClient solrClient, SolrCloudManager cloudManager, Map<String, Object> pluginArgs) {-- Map<String, Object> args = new HashMap<>();- // init from optional solr.xml config- if (pluginArgs != null) {- args.putAll(pluginArgs);- }- // override from ZK if available- if (cloudManager != null) {- @SuppressWarnings({"unchecked"})- Map<String, Object> props = (Map<String, Object>)cloudManager.getClusterStateProvider()- .getClusterProperty("metrics", Collections.emptyMap())- .getOrDefault("history", Collections.emptyMap());- args.putAll(props);-- overseerUrlScheme = cloudManager.getClusterStateProvider().getClusterProperty(URL_SCHEME, "http");- } else {- overseerUrlScheme = "http";- }-- this.nodeName = nodeName;- // disable when metrics reporting is disabled- this.enable = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_PROP, "true")))- && metricsHandler.isEnabled();- // default to false - don't collect local per-replica metrics- this.enableReplicas = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_REPLICAS_PROP, "false")));- this.enableNodes = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_NODES_PROP, "false")));- this.collectPeriod = Integer.parseInt(String.valueOf(args.getOrDefault(COLLECT_PERIOD_PROP, DEFAULT_COLLECT_PERIOD)));- int syncPeriod = Integer.parseInt(String.valueOf(args.getOrDefault(SYNC_PERIOD_PROP, SolrRrdBackendFactory.DEFAULT_SYNC_PERIOD)));-- this.solrClient = solrClient;- this.metricsHandler = metricsHandler;- this.cloudManager = cloudManager;- this.timeSource = cloudManager != null ? cloudManager.getTimeSource() : TimeSource.NANO_TIME;-- counters.put(Group.core.toString(), DEFAULT_CORE_COUNTERS);- counters.put(Group.node.toString(), Collections.emptyList());- counters.put(Group.jvm.toString(), Collections.emptyList());- counters.put(Group.collection.toString(), Collections.emptyList());- gauges.put(Group.core.toString(), DEFAULT_CORE_GAUGES);- gauges.put(Group.node.toString(), DEFAULT_NODE_GAUGES);- gauges.put(Group.jvm.toString(), DEFAULT_JVM_GAUGES);- gauges.put(Group.collection.toString(), DEFAULT_COLLECTION_GAUGES);-- versionString = this.getClass().getPackage().getImplementationVersion();- if (versionString == null) {- versionString = "?.?.?";- }- if (versionString.length() > 24) {- versionString = versionString.substring(0, 24) + "...";- }-- if (enable) {- factory = new SolrRrdBackendFactory(solrClient, CollectionAdminParams.SYSTEM_COLL,- syncPeriod, this.timeSource);-- collectService = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1,- new SolrNamedThreadFactory("MetricsHistoryHandler"));- collectService.setRemoveOnCancelPolicy(true);- collectService.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);- collectService.scheduleWithFixedDelay(() -> collectMetrics(),- timeSource.convertDelay(TimeUnit.SECONDS, collectPeriod, TimeUnit.MILLISECONDS),- timeSource.convertDelay(TimeUnit.SECONDS, collectPeriod, TimeUnit.MILLISECONDS),- TimeUnit.MILLISECONDS);- checkSystemCollection();- } else {- factory = null;- }- }-- // check that .system exists- public void checkSystemCollection() {- if (!enable) {- return;- }- if (cloudManager != null) {- try {- if (cloudManager.isClosed() || Thread.interrupted()) {- factory.setPersistent(false);- return;- }- ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();- DocCollection systemColl = clusterState.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL);- if (systemColl == null) {- if (logMissingCollection) {- log.info("No {} collection, keeping metrics history in memory.", CollectionAdminParams.SYSTEM_COLL);- logMissingCollection = false;- }- factory.setPersistent(false);- return;- } else {- boolean ready = false;- for (Replica r : systemColl.getReplicas()) {- if (r.isActive(clusterState.getLiveNodes())) {- ready = true;- break;- }- }- if (!ready) {- log.debug("{} collection not ready yet, keeping metrics history in memory", CollectionAdminParams.SYSTEM_COLL);- factory.setPersistent(false);- return;- }- }- } catch (Exception e) {- if (logMissingCollection) {- log.warn("Error getting cluster state, keeping metrics history in memory", e);- }- logMissingCollection = false;- factory.setPersistent(false);- return;- }- logMissingCollection = true;- factory.setPersistent(true);- } else {- try {- solrClient.query(CollectionAdminParams.SYSTEM_COLL, new SolrQuery(CommonParams.Q, "*:*", CommonParams.ROWS, "0"));- factory.setPersistent(true);- logMissingCollection = true;- } catch (Exception e) {- if (logMissingCollection) {- log.info("No {} collection, keeping metrics history in memory.", CollectionAdminParams.SYSTEM_COLL);- }- logMissingCollection = false;- factory.setPersistent(false);- }- }- }-- public SolrClient getSolrClient() {- return solrClient;- }-- public void removeHistory(String registry) throws IOException {- registry = SolrMetricManager.enforcePrefix(registry);- knownDbs.remove(registry);- if (factory != null) {- factory.remove(registry);- }- }-- @VisibleForTesting- public SolrRrdBackendFactory getFactory() {- return factory;- }-- private String getOverseerLeader() {- // non-ZK node has no Overseer- if (cloudManager == null) {- return null;- }- ZkNodeProps props = null;- try {- VersionedData data = cloudManager.getDistribStateManager().getData(- Overseer.OVERSEER_ELECT + "/leader");- if (data != null && data.getData() != null) {- props = ZkNodeProps.load(data.getData());- }- } catch (KeeperException | NoSuchElementException e) {- log.info("Could not obtain overseer's address, skipping.", e);- return null;- } catch (IOException e) {- log.warn("Could not obtain overseer's address, skipping.", e);- return null;- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- return null;- }- if (props == null) {- return null;- }- String oid = props.getStr(ID);- if (oid == null) {- return null;- }- String nodeName = null;- try {- nodeName = LeaderElector.getNodeName(oid);- } catch (Exception e) {- log.warn("Unknown format of leader id, skipping: {}", oid, e);- return null;- }- return nodeName;- }-- private boolean amIOverseerLeader() {- return amIOverseerLeader(null);- }-- private boolean amIOverseerLeader(String leader) {- if (leader == null) {- leader = getOverseerLeader();- }- if (leader == null) {- return false;- } else {- return nodeName.equals(leader);- }- }-- private void collectMetrics() {- log.debug("-- collectMetrics");- // Make sure we are a solr server thread, so we can use PKI auth, SOLR-12860- // This is a workaround since we could not instrument the ScheduledThreadPoolExecutor in ExecutorUtils- ExecutorUtil.setServerThreadFlag(true);- try {- checkSystemCollection();- } catch (Exception e) {- log.warn("Error checking for .system collection, keeping metrics history in memory", e);- factory.setPersistent(false);- }- // get metrics- collectLocalReplicaMetrics();- collectGlobalMetrics();- ExecutorUtil.setServerThreadFlag(false);- }-- @SuppressWarnings({"unchecked", "rawtypes"})- private void collectLocalReplicaMetrics() {- List<Group> groups = new ArrayList<>();- if (enableNodes) {- groups.add(Group.jvm);- groups.add(Group.node);- }- if (enableReplicas) {- groups.add(Group.core);- }- for (Group group : groups) {- if (Thread.interrupted()) {- return;- }- log.debug("-- collecting local {}...", group);- ModifiableSolrParams params = new ModifiableSolrParams();- params.add(MetricsHandler.GROUP_PARAM, group.toString());- params.add(MetricsHandler.COMPACT_PARAM, "true");- counters.get(group.toString()).forEach(c -> params.add(MetricsHandler.PREFIX_PARAM, c));- gauges.get(group.toString()).forEach(c -> params.add(MetricsHandler.PREFIX_PARAM, c));- AtomicReference<Object> result = new AtomicReference<>();- try {- metricsHandler.handleRequest(params, (k, v) -> {- if (k.equals("metrics")) {- result.set(v);- }- });- NamedList nl = (NamedList)result.get();- if (nl != null) {- for (Iterator<Map.Entry<String, Object>> it = nl.iterator(); it.hasNext(); ) {- Map.Entry<String, Object> entry = it.next();- String registry = entry.getKey();- if (group != Group.core) { // add nodeName suffix- registry = registry + "." + nodeName;- }-- RrdDb db = getOrCreateDb(registry, group);- if (db == null) {- continue;- }- // set the timestamp- Sample s = db.createSample(TimeUnit.SECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS));- NamedList<Object> values = (NamedList<Object>)entry.getValue();- AtomicBoolean dirty = new AtomicBoolean(false);- counters.get(group.toString()).forEach(c -> {- Number val = (Number)values.get(c);- if (val != null) {- dirty.set(true);- s.setValue(c, val.doubleValue());- }- });- gauges.get(group.toString()).forEach(c -> {- Number val = (Number)values.get(c);- if (val != null) {- dirty.set(true);- s.setValue(c, val.doubleValue());- }- });- if (dirty.get()) {- s.update();- }- }- }- } catch (Exception e) {- log.warn("Exception retrieving local metrics for group {}: {}", group, e);- }- }- }-- private void collectGlobalMetrics() {- if (!amIOverseerLeader()) {- return;- }- Set<String> nodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes());- NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();- Set<String> collTags = new HashSet<>();- collTags.addAll(counters.get(Group.core.toString()));- collTags.addAll(gauges.get(Group.core.toString()));-- Set<String> nodeTags = new HashSet<>();- String nodePrefix = "metrics:" + SolrMetricManager.getRegistryName(Group.node) + ":";- counters.get(Group.node.toString()).forEach(name -> {- nodeTags.add(nodePrefix + name);- });- gauges.get(Group.node.toString()).forEach(name -> {- nodeTags.add(nodePrefix + name);- });- String jvmPrefix = "metrics:" + SolrMetricManager.getRegistryName(Group.jvm) + ":";- counters.get(Group.jvm.toString()).forEach(name -> {- nodeTags.add(jvmPrefix + name);- });- gauges.get(Group.jvm.toString()).forEach(name -> {- nodeTags.add(jvmPrefix + name);- });-- // per-registry totals- // XXX at the moment the type of metrics that we collect allows- // adding all partial values. At some point it may be necessary to implement- // other aggregation functions.- // group : registry : name : value- Map<Group, Map<String, Map<String, Number>>> totals = new HashMap<>();-- // collect and aggregate per-collection totals- for (String node : nodes) {- if (cloudManager.isClosed() || Thread.interrupted()) {- return;- }- // add core-level stats- Map<String, Map<String, List<Replica>>> infos = nodeStateProvider.getReplicaInfo(node, collTags);- infos.forEach((coll, shards) -> {- shards.forEach((sh, replicas) -> {- String registry = SolrMetricManager.getRegistryName(Group.collection, coll);- Map<String, Number> perReg = totals- .computeIfAbsent(Group.collection, g -> new HashMap<>())- .computeIfAbsent(registry, r -> new HashMap<>());- replicas.forEach(ri -> {- collTags.forEach(tag -> {- double value = ((Number)ri.get(tag, 0.0)).doubleValue();- DoubleAdder adder = (DoubleAdder)perReg.computeIfAbsent(tag, t -> new DoubleAdder());- adder.add(value);- });- });- });- });- // add node-level stats- Map<String, Object> nodeValues = nodeStateProvider.getNodeValues(node, nodeTags);- for (Group g : Arrays.asList(Group.node, Group.jvm)) {- String registry = SolrMetricManager.getRegistryName(g);- Map<String, Number> perReg = totals- .computeIfAbsent(g, gr -> new HashMap<>())- .computeIfAbsent(registry, r -> new HashMap<>());- Set<String> names = new HashSet<>();- names.addAll(counters.get(g.toString()));- names.addAll(gauges.get(g.toString()));- names.forEach(name -> {- String tag = "metrics:" + registry + ":" + name;- double value = ((Number)nodeValues.getOrDefault(tag, 0.0)).doubleValue();- DoubleAdder adder = (DoubleAdder)perReg.computeIfAbsent(name, t -> new DoubleAdder());- adder.add(value);- });- }- }-- // add numNodes- String nodeReg = SolrMetricManager.getRegistryName(Group.node);- Map<String, Number> perNodeReg = totals- .computeIfAbsent(Group.node, gr -> new HashMap<>())- .computeIfAbsent(nodeReg, r -> new HashMap<>());- perNodeReg.put(NUM_NODES_KEY, nodes.size());-- // add some global collection-level stats- try {- ClusterState state = cloudManager.getClusterStateProvider().getClusterState();- state.forEachCollection(coll -> {- String registry = SolrMetricManager.getRegistryName(Group.collection, coll.getName());- Map<String, Number> perReg = totals- .computeIfAbsent(Group.collection, g -> new HashMap<>())- .computeIfAbsent(registry, r -> new HashMap<>());- Slice[] slices = coll.getActiveSlicesArr();- perReg.put(NUM_SHARDS_KEY, slices.length);- DoubleAdder numActiveReplicas = new DoubleAdder();- for (Slice s : slices) {- s.forEach(r -> {- if (r.isActive(state.getLiveNodes())) {- numActiveReplicas.add(1.0);- }- });- }- perReg.put(NUM_REPLICAS_KEY, numActiveReplicas);- });- } catch (IOException e) {- log.warn("Exception getting cluster state", e);- }-- // now update the db-s- totals.forEach((group, perGroup) -> {- perGroup.forEach((reg, perReg) -> {- RrdDb db = getOrCreateDb(reg, group);- if (db == null) {- return;- }- try {- // set the timestamp- Sample s = db.createSample(TimeUnit.SECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS));- AtomicBoolean dirty = new AtomicBoolean(false);- List<Group> groups = new ArrayList<>();- groups.add(group);- if (group == Group.collection) {- groups.add(Group.core);- }- for (Group g : groups) {- counters.get(g.toString()).forEach(c -> {- Number val = perReg.get(c);- if (val != null) {- dirty.set(true);- s.setValue(c, val.doubleValue());- }- });- gauges.get(g.toString()).forEach(c -> {- Number val = perReg.get(c);- if (val != null) {- dirty.set(true);- s.setValue(c, val.doubleValue());- }- });- }- if (dirty.get()) {- s.update();- }- } catch (Exception e) {- log.warn("Exception storing sample in RrdDb for group {}: {}", group, e);- }- });- });- }-- private RrdDef createDef(String registry, Group group) {- registry = SolrMetricManager.enforcePrefix(registry);-- // base sampling period is collectPeriod - samples more frequent than- // that will be dropped, samples less frequent will be interpolated- RrdDef def = new RrdDef(URI_PREFIX + registry, collectPeriod);- // set the start time early enough so that the first sample is always later- // than the start of the archive- def.setStartTime(TimeUnit.SECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS) - def.getStep());-- // add datasources- List<Group> groups = new ArrayList<>();- groups.add(group);- if (group == Group.collection) {- groups.add(Group.core);- }- for (Group g : groups) {- // use NaN when more than 1 sample is missing- counters.get(g.toString()).forEach(name ->- def.addDatasource(name, DsType.COUNTER, collectPeriod * 2, Double.NaN, Double.NaN));- gauges.get(g.toString()).forEach(name ->- def.addDatasource(name, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN));- }- if (groups.contains(Group.node)) {- // add nomNodes gauge- def.addDatasource(NUM_NODES_KEY, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN);- }-- // add archives-- // use AVERAGE consolidation,- // use NaN when >50% samples are missing- def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 240); // 4 hours- def.addArchive(ConsolFun.AVERAGE, 0.5, 10, 288); // 48 hours- def.addArchive(ConsolFun.AVERAGE, 0.5, 60, 336); // 2 weeks- def.addArchive(ConsolFun.AVERAGE, 0.5, 240, 180); // 2 months- def.addArchive(ConsolFun.AVERAGE, 0.5, 1440, 365); // 1 year- return def;- }-- private RrdDb getOrCreateDb(String registry, Group group) {- RrdDb db = knownDbs.computeIfAbsent(registry, r -> {- RrdDef def = createDef(r, group);- try {- RrdDb newDb = new RrdDb(def, factory);- return newDb;- } catch (IOException e) {- log.warn("Can't create RrdDb for registry {}, group {}: {}", registry, group, e);- return null;- }- });- return db;- }-- @Override- public void close() {- if (log.isDebugEnabled()) {- log.debug("Closing {}", hashCode());- }- ExecutorUtil.shutdownNowAndAwaitTermination(collectService);- if (factory != null) {- factory.close();- }- knownDbs.clear();- }-- public enum Cmd {- LIST, STATUS, GET, DELETE;-- static final Map<String, Cmd> actions = Collections.unmodifiableMap(- Stream.of(Cmd.values())- .collect(toMap(Cmd::toLower, Function.identity())));-- public static Cmd get(String p) {- return p == null ? null : actions.get(p.toLowerCase(Locale.ROOT));- }-- public String toLower() {- return toString().toLowerCase(Locale.ROOT);- }- }-- public enum Format {- LIST, STRING, GRAPH;-- static final Map<String, Format> formats = Collections.unmodifiableMap(- Stream.of(Format.values())- .collect(toMap(Format::toLower, Function.identity())));-- public static Format get(String p) {- return p == null ? null : formats.get(p.toLowerCase(Locale.ROOT));- }-- public String toLower() {- return toString().toLowerCase(Locale.ROOT);- }- }--- @Override- public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {- if (!enable) {- rsp.add("error", "metrics history collection is disabled");- return;- }- String actionStr = req.getParams().get(CommonParams.ACTION);- if (actionStr == null) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'action' is a required param");- }- Cmd cmd = Cmd.get(actionStr);- if (cmd == null) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown 'action' param '" + actionStr + "', supported actions: " + Cmd.actions);- }- final SimpleOrderedMap<Object> res = new SimpleOrderedMap<>();- rsp.add("metrics", res);- switch (cmd) {- case LIST:- int rows = req.getParams().getInt(CommonParams.ROWS, SolrRrdBackendFactory.DEFAULT_MAX_DBS);- List<Pair<String, Long>> lst = factory.list(rows);- lst.forEach(p -> {- SimpleOrderedMap<Object> data = new SimpleOrderedMap<>();- // RrdDb always uses seconds - convert here for compatibility- data.add("lastModified", TimeUnit.SECONDS.convert(p.second(), TimeUnit.MILLISECONDS));- data.add("node", nodeName);- res.add(p.first(), data);- });- break;- case GET:- String name = req.getParams().get(CommonParams.NAME);- if (name == null) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'name' is a required param");- }- String[] dsNames = req.getParams().getParams("ds");- String formatStr = req.getParams().get("format", Format.LIST.toString());- Format format = Format.get(formatStr);- if (format == null) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown 'format' param '" + formatStr + "', supported formats: " + Format.formats);- }- if (factory.exists(name)) {- // get a throwaway copy (safe to close and discard)- RrdDb db = new RrdDb(URI_PREFIX + name, true, factory);- SimpleOrderedMap<Object> data = new SimpleOrderedMap<>();- data.add("data", getDbData(db, dsNames, format, req.getParams()));- data.add("lastModified", db.getLastUpdateTime());- data.add("node", nodeName);- res.add(name, data);- db.close();- }- break;- case STATUS:- name = req.getParams().get(CommonParams.NAME);- if (name == null) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'name' is a required param");- }- if (factory.exists(name)) {- // get a throwaway copy (safe to close and discard)- RrdDb db = RrdDb.getBuilder().setBackendFactory(factory).setReadOnly(true).setPath(new URI(URI_PREFIX + name)).build();- SimpleOrderedMap<Object> status = new SimpleOrderedMap<>();- status.add("status", getDbStatus(db));- status.add("node", nodeName);- res.add(name, status);- db.close();- }- break;- case DELETE:- name = req.getParams().get(CommonParams.NAME);- if (name == null) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'name' is a required param");- }- if (name.equalsIgnoreCase("all") || name.equals("*")) {- factory.removeAll();- } else {- factory.remove(name);- }- rsp.add("success", "ok");- break;- }- // when using in-memory DBs non-overseer node has no access to overseer DBs - in this case- // forward the request to Overseer leader if available- if (!factory.isPersistent()) {- String leader = getOverseerLeader();- if (leader != null && !amIOverseerLeader(leader)) {- // get & merge remote response- NamedList<Object> remoteRes = handleRemoteRequest(leader, req);- mergeRemoteRes(rsp, remoteRes);- }- }- SimpleOrderedMap<Object> apiState = new SimpleOrderedMap<>();- apiState.add("enableReplicas", enableReplicas);- apiState.add("enableNodes", enableNodes);- apiState.add("mode", enable ? (factory.isPersistent() ? "index" : "memory") : "inactive");- if (!factory.isPersistent()) {- apiState.add("message", "WARNING: metrics history is not being persisted. Create .system collection to start persisting history.");- }- rsp.add("state", apiState);- rsp.getResponseHeader().add("zkConnected", cloudManager != null);- }-- @SuppressWarnings({"unchecked"})- private NamedList<Object> handleRemoteRequest(String nodeName, SolrQueryRequest req) {- String baseUrl = Utils.getBaseUrlForNodeName(nodeName, overseerUrlScheme);- String url;- try {- URL u = new URL(baseUrl);- u = new URL(u.getProtocol(), u.getHost(), u.getPort(), "/api/cluster/metrics/history");- url = u.toString();- } catch (MalformedURLException e) {- log.warn("Invalid Overseer url '{}', unable to fetch remote metrics history", baseUrl, e);- return null;- }- // always use javabin- ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());- params.set(CommonParams.WT, "javabin");- url = url + "?" + params.toString();- try {- byte[] data = cloudManager.httpRequest(url, SolrRequest.METHOD.GET, null, null, HttpClientUtil.DEFAULT_CONNECT_TIMEOUT, true);- // response is always a NamedList- try (JavaBinCodec codec = new JavaBinCodec()) {- return (NamedList<Object>)codec.unmarshal(new ByteArrayInputStream(data));- }- } catch (IOException e) {- log.warn("Exception forwarding request to Overseer at {}", url, e);- return null;- }- }-- @SuppressWarnings({"unchecked", "rawtypes"})- private void mergeRemoteRes(SolrQueryResponse rsp, NamedList<Object> remoteRes) {- if (remoteRes == null || remoteRes.get("metrics") == null) {- return;- }- NamedList<Object> remoteMetrics = (NamedList<Object>)remoteRes.get("metrics");- SimpleOrderedMap localMetrics = (SimpleOrderedMap) rsp.getValues().get("metrics");- remoteMetrics.forEach((k, v) -> localMetrics.add(k, v));- }-- private NamedList<Object> getDbStatus(RrdDb db) throws IOException {- NamedList<Object> res = new SimpleOrderedMap<>();- res.add("lastModified", db.getLastUpdateTime());- RrdDef def = db.getRrdDef();- res.add("step", def.getStep());- res.add("datasourceCount", db.getDsCount());- res.add("archiveCount", db.getArcCount());- res.add("datasourceNames", Arrays.asList(db.getDsNames()));- List<Object> dss = new ArrayList<>(db.getDsCount());- res.add("datasources", dss);- for (DsDef dsDef : def.getDsDefs()) {- Map<String, Object> map = new LinkedHashMap<>();- map.put("datasource", dsDef.dump());- Datasource ds = db.getDatasource(dsDef.getDsName());- map.put("lastValue", ds.getLastValue());- dss.add(map);- }- List<Object> archives = new ArrayList<>(db.getArcCount());- res.add("archives", archives);- ArcDef[] arcDefs = def.getArcDefs();- for (int i = 0; i < db.getArcCount(); i++) {- Archive a = db.getArchive(i);- Map<String, Object> map = new LinkedHashMap<>();- map.put("archive", arcDefs[i].dump());- map.put("steps", a.getSteps());- map.put("consolFun", a.getConsolFun().name());- map.put("xff", a.getXff());- map.put("startTime", a.getStartTime());- map.put("endTime", a.getEndTime());- map.put("rows", a.getRows());- archives.add(map);- }-- return res;- }-- private NamedList<Object> getDbData(RrdDb db, String[] dsNames, Format format, SolrParams params) throws IOException {- NamedList<Object> res = new SimpleOrderedMap<>();- if (dsNames == null || dsNames.length == 0) {- dsNames = db.getDsNames();- }- StringBuilder str = new StringBuilder();- RrdDef def = db.getRrdDef();- ArcDef[] arcDefs = def.getArcDefs();- for (ArcDef arcDef : arcDefs) {- SimpleOrderedMap<Object> map = new SimpleOrderedMap<>();- res.add(arcDef.dump(), map);- Archive a = db.getArchive(arcDef.getConsolFun(), arcDef.getSteps());- // startTime / endTime, arcStep are in seconds- FetchRequest fr = db.createFetchRequest(arcDef.getConsolFun(),- a.getStartTime() - a.getArcStep(),- a.getEndTime() + a.getArcStep());- FetchData fd = fr.fetchData();- if (format != Format.GRAPH) {- // add timestamps separately from values- long[] timestamps = fd.getTimestamps();- if (format == Format.LIST) {- // Arrays.asList works only on arrays of Objects- map.add("timestamps", Arrays.stream(timestamps).boxed().collect(Collectors.toList()));- } else {- str.setLength(0);- for (int i = 0; i < timestamps.length; i++) {- if (i > 0) {- str.append('\n');- }- str.append(String.valueOf(timestamps[i]));- }- map.add("timestamps", str.toString());- }- }- SimpleOrderedMap<Object> values = new SimpleOrderedMap<>();- map.add("values", values);- for (String name : dsNames) {- double[] vals = fd.getValues(name);- switch (format) {- case GRAPH:- RrdGraphDef graphDef = new RrdGraphDef();- graphDef.setTitle(name);- graphDef.datasource(name, fd);- graphDef.setStartTime(a.getStartTime() - a.getArcStep());- graphDef.setEndTime(a.getEndTime() + a.getArcStep());- graphDef.setPoolUsed(false);- graphDef.setAltAutoscale(true);- graphDef.setAltYGrid(true);- graphDef.setAltYMrtg(true);- graphDef.setSignature("Apache Solr " + versionString);- graphDef.setNoLegend(true);- graphDef.setAntiAliasing(true);- graphDef.setTextAntiAliasing(true);- graphDef.setWidth(500);- graphDef.setHeight(175);- graphDef.setTimeZone(TimeZone.getDefault());- graphDef.setLocale(Locale.ROOT);- // redraw immediately- graphDef.setLazy(false);- // area with a border- graphDef.area(name, new Color(0xffb860), null);- graphDef.line(name, Color.RED, null, 1.0f);- RrdGraph graph = new RrdGraph(graphDef);- BufferedImage bi = new BufferedImage(- graph.getRrdGraphInfo().getWidth(),- graph.getRrdGraphInfo().getHeight(),- BufferedImage.TYPE_INT_RGB);- graph.render(bi.getGraphics());- ByteArrayOutputStream baos = new ByteArrayOutputStream();- ImageIO.write(bi, "png", baos);- values.add(name, Base64.byteArrayToBase64(baos.toByteArray()));- break;- case STRING:- str.setLength(0);- for (int i = 0; i < vals.length; i++) {- if (i > 0) {- str.append('\n');- }- str.append(String.valueOf(vals[i]));- }- values.add(name, str.toString());- break;- case LIST:- values.add(name, Arrays.stream(vals).boxed().collect(Collectors.toList()));- break;- }- }- }- return res;- }-- @Override- public String getDescription() {- return "A handler for metrics history";- }-- @Override- public Name getPermissionName(AuthorizationContext request) {- return Name.METRICS_HISTORY_READ_PERM;- }-- @Override- public Boolean registerV2() {- return Boolean.TRUE;- }-- @Override- public Collection<Api> getApis() {- return ApiBag.wrapRequestHandlers(this, "metrics.history");- }--}diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.javadeleted file mode 100644index f65b8f0fd71..00000000000--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java+++ /dev/null@@ -1,295 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.metrics.reporters.solr;--import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.net.MalformedURLException;-import java.util.ArrayList;-import java.util.Collections;-import java.util.HashSet;-import java.util.List;-import java.util.Map;-import java.util.concurrent.TimeUnit;-import java.util.function.Supplier;--import org.apache.http.client.HttpClient;-import org.apache.solr.cloud.LeaderElector;-import org.apache.solr.cloud.Overseer;-import org.apache.solr.cloud.ZkController;-import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkNodeProps;-import org.apache.solr.core.CoreContainer;-import org.apache.solr.core.PluginInfo;-import org.apache.solr.core.SolrInfoBean;-import org.apache.solr.handler.admin.MetricsCollectorHandler;-import org.apache.solr.metrics.SolrCoreContainerReporter;-import org.apache.solr.metrics.SolrMetricManager;-import org.apache.zookeeper.KeeperException;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import static org.apache.solr.common.params.CommonParams.ID;--/**- * This reporter sends selected metrics from local registries to {@link Overseer}.- * <p>The following configuration properties are supported:</p>- * <ul>- * <li>handler - (optional str) handler path where reports are sent. Default is- * {@link MetricsCollectorHandler#HANDLER_PATH}.</li>- * <li>period - (optional int) how often reports are sent, in seconds. Default is 60. Setting this- * to 0 disables the reporter.</li>- * <li>report - (optional multiple lst) report configuration(s), see below.</li>- * </ul>- * Each report configuration consist of the following properties:- * <ul>- * <li>registry - (required str) regex pattern matching source registries (see {@link SolrMetricManager#registryNames(String...)}),- * may contain capture groups.</li>- * <li>group - (required str) target registry name where metrics will be grouped. This can be a regex pattern that- * contains back-references to capture groups collected by <code>registry</code> pattern</li>- * <li>label - (optional str) optional prefix to prepend to metric names, may contain back-references to- * capture groups collected by <code>registry</code> pattern</li>- * <li>filter - (optional multiple str) regex expression(s) matching selected metrics to be reported.</li>- * </ul>- * NOTE: this reporter uses predefined "cluster" group, and it's always created even if explicit configuration- * is missing. Default configuration uses report specifications from {@link #DEFAULT_REPORTS}.- * <p>Example configuration:</p>- * <pre>- * <reporter name="test" group="cluster" class="solr.SolrClusterReporter">- * <str name="handler">/admin/metrics/collector</str>- * <int name="period">11</int>- * <lst name="report">- * <str name="group">overseer</str>- * <str name="label">jvm</str>- * <str name="registry">solr\.jvm</str>- * <str name="filter">memory\.total\..*</str>- * <str name="filter">memory\.heap\..*</str>- * <str name="filter">os\.SystemLoadAverage</str>- * <str name="filter">threads\.count</str>- * </lst>- * <lst name="report">- * <str name="group">overseer</str>- * <str name="label">leader.$1</str>- * <str name="registry">solr\.core\.(.*)\.leader</str>- * <str name="filter">UPDATE\./update/.*</str>- * </lst>- * </reporter>- * </pre>- *- */-public class SolrClusterReporter extends SolrCoreContainerReporter {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- public static final String CLUSTER_GROUP = SolrMetricManager.enforcePrefix(SolrInfoBean.Group.cluster.toString());-- public static final List<SolrReporter.Report> DEFAULT_REPORTS = new ArrayList<SolrReporter.Report>() {{- add(new SolrReporter.Report(CLUSTER_GROUP, "jetty",- SolrMetricManager.enforcePrefix(SolrInfoBean.Group.jetty.toString()),- Collections.emptySet())); // all metrics- add(new SolrReporter.Report(CLUSTER_GROUP, "jvm",- SolrMetricManager.enforcePrefix(SolrInfoBean.Group.jvm.toString()),- new HashSet<String>() {{- add("memory\\.total\\..*");- add("memory\\.heap\\..*");- add("os\\.SystemLoadAverage");- add("os\\.FreePhysicalMemorySize");- add("os\\.FreeSwapSpaceSize");- add("os\\.OpenFileDescriptorCount");- add("threads\\.count");- }}));- add(new SolrReporter.Report(CLUSTER_GROUP, "node", SolrMetricManager.enforcePrefix(SolrInfoBean.Group.node.toString()),- new HashSet<String>() {{- add("CONTAINER\\.cores\\..*");- add("CONTAINER\\.fs\\..*");- }}));- add(new SolrReporter.Report(CLUSTER_GROUP, "leader.$1", "solr\\.core\\.(.*)\\.leader",- new HashSet<String>(){{- add("UPDATE\\./update/.*");- add("QUERY\\./select.*");- add("INDEX\\..*");- add("TLOG\\..*");- }}));- }};-- private String handler = MetricsCollectorHandler.HANDLER_PATH;- private List<SolrReporter.Report> reports = new ArrayList<>();-- private SolrReporter reporter;-- /**- * Create a reporter for metrics managed in a named registry.- *- * @param metricManager metric manager- * @param registryName this is ignored- */- public SolrClusterReporter(SolrMetricManager metricManager, String registryName) {- super(metricManager, registryName);- }-- public void setHandler(String handler) {- this.handler = handler;- }-- public void setReport(@SuppressWarnings({"rawtypes"})List<Map> reportConfig) {- if (reportConfig == null || reportConfig.isEmpty()) {- return;- }- reportConfig.forEach(map -> {- SolrReporter.Report r = SolrReporter.Report.fromMap(map);- if (r != null) {- reports.add(r);- }- });- }-- public void setReport(@SuppressWarnings({"rawtypes"})Map map) {- if (map == null || map.isEmpty()) {- return;- }- SolrReporter.Report r = SolrReporter.Report.fromMap(map);- if (r != null) {- reports.add(r);- }- }-- List<SolrReporter.Report> getReports() {- return reports;- }-- @Override- protected void doInit() {- if (reports.isEmpty()) { // set defaults- reports = DEFAULT_REPORTS;- }- }-- @Override- protected void validate() throws IllegalStateException {- // (period < 1) means "don't start reporter" and so no (period > 0) validation needed- }-- @Override- public void close() throws IOException {- if (reporter != null) {- reporter.close();;- }- }-- @Override- public void init(PluginInfo pluginInfo, CoreContainer cc) {- super.init(pluginInfo, cc);- if (reporter != null) {- reporter.close();;- }- if (!enabled) {- log.info("Reporter disabled for registry {}", registryName);- return;- }- // start reporter only in cloud mode- if (!cc.isZooKeeperAware()) {- log.warn("Not ZK-aware, not starting...");- return;- }- if (period < 1) { // don't start it- log.info("Turning off node reporter, period={}", period);- return;- }- HttpClient httpClient = cc.getUpdateShardHandler().getDefaultHttpClient();- ZkController zk = cc.getZkController();- String reporterId = zk.getNodeName();- reporter = SolrReporter.Builder.forReports(metricManager, reports)- .convertRatesTo(TimeUnit.SECONDS)- .convertDurationsTo(TimeUnit.MILLISECONDS)- .withHandler(handler)- .withReporterId(reporterId)- .setCompact(true)- .cloudClient(false) // we want to send reports specifically to a selected leader instance- .skipAggregateValues(true) // we don't want to transport details of aggregates- .skipHistograms(true) // we don't want to transport histograms- .build(httpClient, new OverseerUrlSupplier(zk));-- reporter.start(period, TimeUnit.SECONDS);- }-- // TODO: fix this when there is an elegant way to retrieve URL of a node that runs Overseer leader.- // package visibility for unit tests- static class OverseerUrlSupplier implements Supplier<String> {- private static final long DEFAULT_INTERVAL = 30000000; // 30s- private ZkController zk;- private String lastKnownUrl = null;- private long lastCheckTime = 0;- private long interval = DEFAULT_INTERVAL;-- OverseerUrlSupplier(ZkController zk) {- this.zk = zk;- }-- @Override- public String get() {- if (zk == null) {- return null;- }- // primitive caching for lastKnownUrl- long now = System.nanoTime();- if (lastKnownUrl != null && (now - lastCheckTime) < interval) {- return lastKnownUrl;- }- if (!zk.isConnected()) {- return lastKnownUrl;- }- lastCheckTime = now;- SolrZkClient zkClient = zk.getZkClient();- ZkNodeProps props;- try {- props = ZkNodeProps.load(zkClient.getData(- Overseer.OVERSEER_ELECT + "/leader", null, null, true));- } catch (KeeperException e) {- log.warn("Could not obtain overseer's address, skipping.", e);- return lastKnownUrl;- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- return lastKnownUrl;- }- if (props == null) {- return lastKnownUrl;- }- String oid = props.getStr(ID);- if (oid == null) {- return lastKnownUrl;- }- String nodeName = null;- try {- nodeName = LeaderElector.getNodeName(oid);- } catch (Exception e) {- log.warn("Unknown format of leader id, skipping: {}", oid, e);- return lastKnownUrl;- }- // convert nodeName back to URL- String url = zk.getZkStateReader().getBaseUrlForNodeName(nodeName);- // check that it's parseable- try {- new java.net.URL(url);- } catch (MalformedURLException mue) {- log.warn("Malformed Overseer's leader URL: url", mue);- return lastKnownUrl;- }- lastKnownUrl = url;- return url;- }- }--}diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.javadeleted file mode 100644index 8791160e966..00000000000--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java+++ /dev/null@@ -1,189 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.metrics.reporters.solr;--import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.util.ArrayList;-import java.util.Collections;-import java.util.List;-import java.util.concurrent.TimeUnit;-import java.util.function.Supplier;--import org.apache.solr.cloud.CloudDescriptor;-import org.apache.solr.common.cloud.ClusterState;-import org.apache.solr.common.cloud.DocCollection;-import org.apache.solr.common.cloud.Replica;-import org.apache.solr.core.PluginInfo;-import org.apache.solr.core.SolrCore;-import org.apache.solr.handler.admin.MetricsCollectorHandler;-import org.apache.solr.metrics.SolrCoreReporter;-import org.apache.solr.metrics.SolrMetricManager;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import com.codahale.metrics.MetricFilter;--/**- * This class reports selected metrics from replicas to shard leader.- * <p>The following configuration properties are supported:</p>- * <ul>- * <li>handler - (optional str) handler path where reports are sent. Default is- * {@link MetricsCollectorHandler#HANDLER_PATH}.</li>- * <li>period - (optional int) how often reports are sent, in seconds. Default is 60. Setting this- * to 0 disables the reporter.</li>- * <li>filter - (optional multiple str) regex expression(s) matching selected metrics to be reported.</li>- * </ul>- * NOTE: this reporter uses predefined "shard" group, and it's always created even if explicit configuration- * is missing. Default configuration uses filters defined in {@link #DEFAULT_FILTERS}.- * <p>Example configuration:</p>- * <pre>- * <reporter name="test" group="shard" class="solr.SolrShardReporter">- * <int name="period">11</int>- * <str name="filter">UPDATE\./update/.*requests</str>- * <str name="filter">QUERY\./select.*requests</str>- * </reporter>- * </pre>- */-public class SolrShardReporter extends SolrCoreReporter {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- public static final List<String> DEFAULT_FILTERS = new ArrayList<>(){{- add("TLOG.*");- add("CORE\\.fs.*");- add("REPLICATION.*");- add("INDEX\\.flush.*");- add("INDEX\\.merge\\.major.*");- add("UPDATE\\./update.*requests");- add("QUERY\\./select.*requests");- }};-- private String handler = MetricsCollectorHandler.HANDLER_PATH;-- private SolrReporter reporter;-- /**- * Create a reporter for metrics managed in a named registry.- *- * @param metricManager metric manager- * @param registryName registry to use, one of registries managed by- * {@link SolrMetricManager}- */- public SolrShardReporter(SolrMetricManager metricManager, String registryName) {- super(metricManager, registryName);- }-- public void setHandler(String handler) {- this.handler = handler;- }-- @Override- protected void doInit() {- if (filters.isEmpty()) {- filters = DEFAULT_FILTERS;- }- // start in setCore(SolrCore) when core is available- }-- @Override- protected MetricFilter newMetricFilter() {- // unsupported here since setCore(SolrCore) directly uses the this.filters- throw new UnsupportedOperationException(getClass().getCanonicalName()+".newMetricFilter() is not supported");- }-- @Override- protected void validate() throws IllegalStateException {- // (period < 1) means "don't start reporter" and so no (period > 0) validation needed- }-- @Override- public void close() throws IOException {- if (reporter != null) {- reporter.close();- }- }-- @Override- public void init(PluginInfo pluginInfo, SolrCore core) {- super.init(pluginInfo, core);- if (reporter != null) {- reporter.close();- }- if (!enabled) {- log.info("Reporter disabled for registry {}", registryName);- return;- }- if (core.getCoreDescriptor().getCloudDescriptor() == null) {- // not a cloud core- log.warn("Not initializing shard reporter for non-cloud core {}", core.getName());- return;- }- if (period < 1) { // don't start it- log.warn("period={}, not starting shard reporter ", period);- return;- }- // our id is coreNodeName- String id = core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();- // target registry is the leaderRegistryName- String groupId = core.getCoreMetricManager().getLeaderRegistryName();- if (groupId == null) {- log.warn("No leaderRegistryName for core {}, not starting the reporter...", core);- return;- }- SolrReporter.Report spec = new SolrReporter.Report(groupId, null, registryName, filters);- reporter = SolrReporter.Builder.forReports(metricManager, Collections.singletonList(spec))- .convertRatesTo(TimeUnit.SECONDS)- .convertDurationsTo(TimeUnit.MILLISECONDS)- .withHandler(handler)- .withReporterId(id)- .setCompact(true)- .cloudClient(false) // we want to send reports specifically to a selected leader instance- .skipAggregateValues(true) // we don't want to transport details of aggregates- .skipHistograms(true) // we don't want to transport histograms- .build(core.getCoreContainer().getSolrClientCache(), new LeaderUrlSupplier(core));-- reporter.start(period, TimeUnit.SECONDS);- }-- private static class LeaderUrlSupplier implements Supplier<String> {- private SolrCore core;-- LeaderUrlSupplier(SolrCore core) {- this.core = core;- }-- @Override- public String get() {- CloudDescriptor cd = core.getCoreDescriptor().getCloudDescriptor();- if (cd == null) {- return null;- }- ClusterState state = core.getCoreContainer().getZkController().getClusterState();- DocCollection collection = state.getCollection(core.getCoreDescriptor().getCollectionName());- Replica replica = collection.getLeader(core.getCoreDescriptor().getCloudDescriptor().getShardId());- if (replica == null) {- log.warn("No leader for {}/{}", collection.getName(), core.getCoreDescriptor().getCloudDescriptor().getShardId());- return null;- }- String baseUrl = replica.getStr("base_url");- if (baseUrl == null) {- log.warn("No base_url for replica {}", replica);- }- return baseUrl;- }- }-}diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.javadeleted file mode 100644index 14dbc557f8d..00000000000--- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java+++ /dev/null@@ -1,273 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.metrics.rrd;--import java.io.Closeable;-import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.util.concurrent.TimeUnit;-import java.util.concurrent.locks.ReentrantLock;--import org.rrd4j.core.RrdByteArrayBackend;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--/**- *- */-public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- private final SolrRrdBackendFactory factory;- private final boolean readOnly;- private final ReentrantLock lock = new ReentrantLock();- private volatile boolean dirty = false;- private volatile boolean closed = false;- private volatile long lastModifiedTime;-- public static final class SyncData {- public byte[] data;- public long timestamp;-- public SyncData(byte[] data, long timestamp) {- this.data = data;- this.timestamp = timestamp;- }- }-- public SolrRrdBackend(String path, boolean readOnly, SolrRrdBackendFactory factory) {- super(path);- this.factory = factory;- this.lastModifiedTime = TimeUnit.MILLISECONDS.convert(factory.getTimeSource().getEpochTimeNs(), TimeUnit.NANOSECONDS);- try {- SyncData syncData = factory.getData(path);- if (syncData != null) {- setBuffer(syncData.data);- this.lastModifiedTime = syncData.timestamp;- }- } catch (IOException e) {- log.warn("Exception retrieving data from {}, store will be readOnly", path, e);- readOnly = true;- }- this.readOnly = readOnly;- }-- /**- * Open an unregistered (throwaway) read-only clone of another backend.- * @param other other backend- */- public SolrRrdBackend(SolrRrdBackend other) {- super(other.getPath());- readOnly = true;- factory = null;- this.lastModifiedTime = other.lastModifiedTime;- byte[] otherBuffer = other.getBuffer();- byte[] newBuffer = new byte[otherBuffer.length];- System.arraycopy(otherBuffer, 0, newBuffer, 0, otherBuffer.length);- super.setBuffer(newBuffer);- }-- public boolean isReadOnly() {- return readOnly;- }-- public long getLastModifiedTime() {- return lastModifiedTime;- }-- private void markDirty() {- lastModifiedTime = TimeUnit.MILLISECONDS.convert(factory.getTimeSource().getEpochTimeNs(), TimeUnit.NANOSECONDS);- dirty = true;- }-- @Override- protected void write(long offset, byte[] bytes) throws IOException {- if (readOnly || closed) {- return;- }- lock.lock();- try {- super.write(offset, bytes);- markDirty();- } finally {- lock.unlock();- }- }-- @Override- protected void writeShort(long offset, short value) throws IOException {- if (readOnly || closed) {- return;- }- lock.lock();- try {- super.writeShort(offset, value);- markDirty();- } finally {- lock.unlock();- }- }-- @Override- protected void writeInt(long offset, int value) throws IOException {- if (readOnly || closed) {- return;- }- lock.lock();- try {- super.writeInt(offset, value);- markDirty();- } finally {- lock.unlock();- }- }-- @Override- protected void writeLong(long offset, long value) throws IOException {- if (readOnly || closed) {- return;- }- lock.lock();- try {- super.writeLong(offset, value);- markDirty();- } finally {- lock.unlock();- }- }-- @Override- protected void writeDouble(long offset, double value) throws IOException {- if (readOnly || closed) {- return;- }- lock.lock();- try {- super.writeDouble(offset, value);- markDirty();- } finally {- lock.unlock();- }- }-- @Override- protected void writeDouble(long offset, double value, int count) throws IOException {- if (readOnly || closed) {- return;- }- lock.lock();- try {- super.writeDouble(offset, value, count);- markDirty();- } finally {- lock.unlock();- }- }-- @Override- protected void writeDouble(long offset, double[] values) throws IOException {- if (readOnly || closed) {- return;- }- lock.lock();- try {- super.writeDouble(offset, values);- markDirty();- } finally {- lock.unlock();- }- }-- @Override- protected void writeString(long offset, String value, int length) throws IOException {- if (readOnly || closed) {- return;- }- lock.lock();- try {- super.writeString(offset, value, length);- markDirty();- } finally {- lock.unlock();- }- }-- @Override- protected boolean isDirty() {- return dirty;- }-- @Override- protected void setBuffer(byte[] buffer) {- if (readOnly || closed) {- return;- }- lock.lock();- try {- super.setBuffer(buffer);- markDirty();- } finally {- lock.unlock();- }- }-- @Override- protected void setLength(long length) throws IOException {- if (readOnly || closed) {- return;- }- lock.lock();- try {- super.setLength(length);- markDirty();- } finally {- lock.unlock();- }- }-- public SyncData getSyncDataAndMarkClean() {- if (readOnly || closed) {- return null;- }- if (!dirty) {- return null;- }- // hold a lock to block writes so that we get consistent data- lock.lock();- try {- byte[] oldBuffer = getBuffer();- byte[] bufferCopy = new byte[oldBuffer.length];- System.arraycopy(oldBuffer, 0, bufferCopy, 0, oldBuffer.length);- return new SyncData(bufferCopy, lastModifiedTime);- } finally {- // reset the dirty flag- dirty = false;- lock.unlock();- }- }-- @Override- public void close() throws IOException {- super.close();- closed = true;- if (factory != null) {- // unregister myself from the factory- factory.unregisterBackend(getPath());- }- // close- }-}diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.javadeleted file mode 100644index 105fd64ce06..00000000000--- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java+++ /dev/null@@ -1,468 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.metrics.rrd;--import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.net.URI;-import java.net.URISyntaxException;-import java.util.ArrayList;-import java.util.Collections;-import java.util.Comparator;-import java.util.HashMap;-import java.util.Iterator;-import java.util.List;-import java.util.Locale;-import java.util.Map;-import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.Executors;-import java.util.concurrent.ScheduledThreadPoolExecutor;-import java.util.concurrent.TimeUnit;--import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.response.QueryResponse;-import org.apache.solr.common.SolrCloseable;-import org.apache.solr.common.SolrDocument;-import org.apache.solr.common.SolrDocumentList;-import org.apache.solr.common.SolrException;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.params.CollectionAdminParams;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.util.ExecutorUtil;-import org.apache.solr.common.util.IOUtils;-import org.apache.solr.common.util.Pair;-import org.apache.solr.common.util.TimeSource;-import org.apache.solr.common.util.SolrNamedThreadFactory;-import org.rrd4j.core.RrdBackend;-import org.rrd4j.core.RrdBackendFactory;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--/**- * RRD backend factory using Solr documents as underlying storage.- * <p>RRD databases are identified by paths in the format <code>solr:dbName</code>.- * Typically the path will correspond to the name of metric or a group of metrics, eg:- * <code>solr:QUERY./select.requests</code></p>- * <p>NOTE: Solr doesn't register instances of this factory in the static- * registry {@link RrdBackendFactory#registerFactory(RrdBackendFactory)} because- * it's then impossible to manage its life-cycle.</p>- */-public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrCloseable {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- public static final int DEFAULT_SYNC_PERIOD = 60;- public static final int DEFAULT_MAX_DBS = 500;-- public static final String NAME = "SOLR";- public static final String URI_PREFIX = "solr:";- public static final String ID_SEP = "|";- public static final String ID_PREFIX = "rrd";- public static final String DOC_TYPE = "metrics_rrd";-- public static final String DATA_FIELD = "data_bin";-- private final SolrClient solrClient;- private final TimeSource timeSource;- private final String collection;- private final int syncPeriod;- private final int idPrefixLength;- private ScheduledThreadPoolExecutor syncService;- private volatile boolean closed = false;- private volatile boolean persistent = true;-- private final Map<String, SolrRrdBackend> backends = new ConcurrentHashMap<>();-- /**- * Create a factory.- * @param solrClient SolrClient to use- * @param collection collection name where documents are stored (typically this is- * {@link CollectionAdminParams#SYSTEM_COLL})- * @param syncPeriod synchronization period in seconds - how often modified- * databases are stored as updated Solr documents- * @param timeSource time source- */- public SolrRrdBackendFactory(SolrClient solrClient, String collection, int syncPeriod, TimeSource timeSource) {- this.solrClient = solrClient;- this.timeSource = timeSource;- this.collection = collection;- this.syncPeriod = syncPeriod;- if (log.isDebugEnabled()) {- log.debug("Created {}", hashCode());- }- this.idPrefixLength = ID_PREFIX.length() + ID_SEP.length();- syncService = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(2,- new SolrNamedThreadFactory("SolrRrdBackendFactory"));- syncService.setRemoveOnCancelPolicy(true);- syncService.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);- syncService.scheduleWithFixedDelay(() -> maybeSyncBackends(),- timeSource.convertDelay(TimeUnit.SECONDS, syncPeriod, TimeUnit.MILLISECONDS),- timeSource.convertDelay(TimeUnit.SECONDS, syncPeriod, TimeUnit.MILLISECONDS),- TimeUnit.MILLISECONDS);- }-- public TimeSource getTimeSource() {- return timeSource;- }-- private void ensureOpen() throws IOException {- if (closed) {- throw new IOException("Factory already closed");- }- }-- @Override- public boolean canStore(URI uri) {- if (uri == null) {- return false;- }- if (uri.getScheme().toUpperCase(Locale.ROOT).equals(getName())) {- return true;- } else {- return false;- }- }-- @Override- public String getPath(URI uri) {- return uri.getSchemeSpecificPart();- }-- @Override- public URI getUri(String path) {- if (!path.startsWith(URI_PREFIX)) {- path = URI_PREFIX + path;- }- try {- return new URI(path);- } catch (URISyntaxException e) {- throw new IllegalArgumentException("Invalid path: " + path);- }- }-- @Override- public URI getCanonicalUri(URI uri) {- return uri;- }--// @Override-// protected URI getRootUri() {-// try {-// return new URI("solr", null, null, null);-// } catch (URISyntaxException e) {-// throw new RuntimeException("Impossible error", e);-// }-// }-//- /**- * Open (or get) a backend.- * @param path backend path (without URI scheme)- * @param readOnly if true then the backend will never be synchronized to Solr,- * and updates will be silently ignored. Read-only backends can- * be safely closed and discarded after use.- * @return an instance of Solr backend.- * @throws IOException on Solr error when retrieving existing data- */- @Override- protected synchronized RrdBackend open(String path, boolean readOnly) throws IOException {- ensureOpen();- SolrRrdBackend backend = backends.computeIfAbsent(path, p -> new SolrRrdBackend(p, readOnly, this));- if (backend.isReadOnly()) {- if (readOnly) {- return backend;- } else {- // replace it with a writable one- backend = new SolrRrdBackend(path, readOnly, this);- backends.put(path, backend);- return backend;- }- } else {- if (readOnly) {- // return a throwaway unregistered read-only copy- return new SolrRrdBackend(backend);- } else {- return backend;- }- }- }-- SolrRrdBackend.SyncData getData(String path) throws IOException {- if (!persistent) {- return null;- }- try {- ModifiableSolrParams params = new ModifiableSolrParams();- params.add(CommonParams.Q, "{!term f=id}" + ID_PREFIX + ID_SEP + path);- params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);- QueryResponse rsp = solrClient.query(collection, params);- SolrDocumentList docs = rsp.getResults();- if (docs == null || docs.isEmpty()) {- return null;- }- if (docs.size() > 1) {- throw new SolrServerException("Expected at most 1 doc with id '" + path + "' but got " + docs);- }- SolrDocument doc = docs.get(0);- Object o = doc.getFieldValue(DATA_FIELD);- if (o == null) {- return null;- }- if (o instanceof byte[]) {- Object timeObj = doc.getFieldValue("timestamp_l");- Long time = timeObj instanceof Number ? ((Number)timeObj).longValue() : Long.parseLong(String.valueOf(timeObj));- return new SolrRrdBackend.SyncData((byte[])o, time);- } else {- throw new SolrServerException("Unexpected value of '" + DATA_FIELD + "' field: " + o.getClass().getName() + ": " + o);- }- } catch (SolrServerException e) {- throw new IOException(e);- }- }-- void unregisterBackend(String path) {- backends.remove(path);- }-- private static final class DbComparator implements Comparator<Pair<String, Long>> {- static final DbComparator INSTANCE = new DbComparator();-- @Override- public int compare(Pair<String, Long> o1, Pair<String, Long> o2) {- return o1.first().compareTo(o2.first());- }- }-- /**- * List all available databases created by this node name- * @param maxLength maximum number of results to return- * @return list of database names and their last update times, or empty- * @throws IOException on server errors- */- public List<Pair<String, Long>> list(int maxLength) throws IOException {- Map<String, Pair<String, Long>> byName = new HashMap<>();- if (persistent) {- try {- ModifiableSolrParams params = new ModifiableSolrParams();- params.add(CommonParams.Q, "*:*");- params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);- params.add(CommonParams.FL, "id,timestamp_l");- params.add(CommonParams.ROWS, String.valueOf(maxLength));- QueryResponse rsp = solrClient.query(collection, params);- SolrDocumentList docs = rsp.getResults();- if (docs != null) {- docs.forEach(d -> {- Object o = d.getFieldValue("timestamp_l");- if (o == null) {- return;- }- Long time = o instanceof Number ? ((Number)o).longValue() : Long.parseLong(String.valueOf(o));- Pair<String, Long> p = new Pair<>(((String)d.getFieldValue("id")).substring(idPrefixLength), time);- byName.put(p.first(), p);- });- }- } catch (SolrServerException e) {- log.warn("Error retrieving RRD list", e);- }- }- // add in-memory backends not yet stored, or replace with more recent versions- backends.forEach((name, db) -> {- long lastModifiedTime = db.getLastModifiedTime();- Pair<String, Long> stored = byName.get(name);- Pair<String, Long> inMemory = new Pair<>(name, lastModifiedTime);- if (stored != null) {- if (stored.second() < lastModifiedTime) {- byName.put(name, inMemory);- }- } else {- byName.put(name, inMemory);- }- });- ArrayList<Pair<String, Long>> list = new ArrayList<>(byName.values());- Collections.sort(list, DbComparator.INSTANCE);- return list;- }-- /**- * Remove all databases created by this node name.- * @throws IOException on server error- */- public void removeAll() throws IOException {- for (Iterator<SolrRrdBackend> it = backends.values().iterator(); it.hasNext(); ) {- SolrRrdBackend backend = it.next();- it.remove();- IOUtils.closeQuietly(backend);- }- if (!persistent) {- return;- }- // remove all Solr docs- try {- solrClient.deleteByQuery(collection,- "{!term f=" + CommonParams.TYPE + "}:" + DOC_TYPE, syncPeriod * 1000);- } catch (SolrServerException e) {- log.warn("Error deleting RRDs", e);- }- }-- /**- * Remove a database.- * @param path database path.- * @throws IOException on Solr exception- */- public void remove(String path) throws IOException {- SolrRrdBackend backend = backends.remove(path);- if (backend != null) {- IOUtils.closeQuietly(backend);- }- if (!persistent) {- return;- }- // remove Solr doc- try {- solrClient.deleteByQuery(collection, "{!term f=id}" + ID_PREFIX + ID_SEP + path);- } catch (SolrServerException | SolrException e) {- log.warn("Error deleting RRD for path {}", path, e);- }- }-- synchronized void maybeSyncBackends() {- if (closed) {- return;- }- if (!persistent) {- return;- }- if (Thread.interrupted()) {- return;- }- if (log.isDebugEnabled()) {- log.debug("-- maybe sync backends: {}", backends.keySet());- }- Map<String, SolrRrdBackend.SyncData> syncDatas = new HashMap<>();- backends.forEach((path, backend) -> {- SolrRrdBackend.SyncData syncData = backend.getSyncDataAndMarkClean();- if (syncData != null) {- syncDatas.put(backend.getPath(), syncData);- }- });- if (syncDatas.isEmpty()) {- return;- }- if (log.isDebugEnabled()) {- log.debug("-- syncing {}", syncDatas.keySet());- }- // write updates- try {- syncDatas.forEach((path, syncData) -> {- SolrInputDocument doc = new SolrInputDocument();- doc.setField("id", ID_PREFIX + ID_SEP + path);- doc.addField(CommonParams.TYPE, DOC_TYPE);- doc.addField(DATA_FIELD, syncData.data);- doc.setField("timestamp_l", syncData.timestamp);- try {- solrClient.add(collection, doc);- } catch (SolrServerException | IOException e) {- log.warn("Error updating RRD data for {}", path, e);- }- });- if (Thread.interrupted()) {- return;- }- try {- solrClient.commit(collection);- } catch (SolrServerException e) {- log.warn("Error committing RRD data updates", e);- }- } catch (IOException e) {- log.warn("Error sending RRD data updates", e);- }- }-- /**- * Check for existence of a backend.- * @param path backend path, without the URI scheme- * @return true when a backend exists. Note that a backend may exist only- * in memory if it was created recently within {@link #syncPeriod}.- * @throws IOException on Solr exception- */- @Override- public boolean exists(String path) throws IOException {- // check in-memory backends first- if (backends.containsKey(path)) {- return true;- }- if (!persistent) {- return false;- }- try {- ModifiableSolrParams params = new ModifiableSolrParams();- params.add(CommonParams.Q, "{!term f=id}" + ID_PREFIX + ID_SEP + path);- params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);- params.add(CommonParams.FL, "id");- QueryResponse rsp = solrClient.query(collection, params);- SolrDocumentList docs = rsp.getResults();- if (docs == null || docs.isEmpty()) {- return false;- }- if (docs.size() > 1) {- throw new SolrServerException("Expected at most 1 doc with id '" + path + "' but got " + docs);- }- return true;- } catch (SolrServerException e) {- throw new IOException(e);- }- }-- public boolean isPersistent() {- return persistent;- }-- public void setPersistent(boolean persistent) {- this.persistent = persistent;- }-- @Override- protected boolean shouldValidateHeader(String path) throws IOException {- return false;- }-- @Override- public String getName() {- return NAME;- }-- @Override- public boolean isClosed() {- return closed;- }-- @Override- public void close() {- if (closed) {- return;- }- if (log.isDebugEnabled()) {- log.debug("Closing {}", hashCode());- }- closed = true;- backends.values().forEach(IOUtils::closeQuietly);- backends.clear();- ExecutorUtil.shutdownNowAndAwaitTermination(syncService);- syncService = null;- }-}diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/package-info.java b/solr/core/src/java/org/apache/solr/metrics/rrd/package-info.javadeleted file mode 100644index 2fd018c8d1e..00000000000--- a/solr/core/src/java/org/apache/solr/metrics/rrd/package-info.java+++ /dev/null@@ -1,22 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--/**- * This package contains {@link org.rrd4j.core.RrdBackendFactory} implementation- * that supports storing metrics history in Solr.- */-package org.apache.solr.metrics.rrd;diff --git a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.javaindex 5d24a1dc862..c0c3f7bd117 100644--- a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java+++ b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java@@ -48,7 +48,6 @@ public interface PermissionNameProvider {SECURITY_EDIT_PERM("security-edit", null),SECURITY_READ_PERM("security-read", null),METRICS_READ_PERM("metrics-read", null),- METRICS_HISTORY_READ_PERM("metrics-history-read", null),FILESTORE_READ_PERM("filestore-read", null),FILESTORE_WRITE_PERM("filestore-write", null),PACKAGE_EDIT_PERM("package-edit", null),diff --git a/solr/core/src/test-files/solr/solr-solrreporter.xml b/solr/core/src/test-files/solr/solr-solrreporter.xmldeleted file mode 100644index 2a7416539d8..00000000000--- a/solr/core/src/test-files/solr/solr-solrreporter.xml+++ /dev/null@@ -1,69 +0,0 @@-<?xml version="1.0" encoding="UTF-8" ?>-<!--- Licensed to the Apache Software Foundation (ASF) under one or more- contributor license agreements. See the NOTICE file distributed with- this work for additional information regarding copyright ownership.- The ASF licenses this file to You under the Apache License, Version 2.0- (the "License"); you may not use this file except in compliance with- the License. You may obtain a copy of the License at-- http://www.apache.org/licenses/LICENSE-2.0-- Unless required by applicable law or agreed to in writing, software- distributed under the License is distributed on an "AS IS" BASIS,- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- See the License for the specific language governing permissions and- limitations under the License.--->--<solr>- <str name="allowPaths">${solr.allowPaths:}</str>-- <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">- <str name="urlScheme">${urlScheme:}</str>- <int name="socketTimeout">${socketTimeout:90000}</int>- <int name="connTimeout">${connTimeout:15000}</int>- </shardHandlerFactory>-- <solrcloud>- <str name="host">127.0.0.1</str>- <int name="hostPort">${hostPort:8983}</int>- <str name="hostContext">${hostContext:solr}</str>- <int name="zkClientTimeout">${solr.zkclienttimeout:30000}</int>- <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>- <int name="leaderVoteWait">${leaderVoteWait:10000}</int>- <int name="distribUpdateConnTimeout">${distribUpdateConnTimeout:45000}</int>- <int name="distribUpdateSoTimeout">${distribUpdateSoTimeout:340000}</int>- </solrcloud>-- <metrics>- <!-- disable default JMX reporter to avoid conflicts with multiple CoreContainers. -->- <reporter name="defaultJmx" class="org.apache.solr.metrics.reporters.SolrJmxReporter">- <bool name="enabled">false</bool>- </reporter>- <reporter name="test" group="shard" class="org.apache.solr.metrics.reporters.solr.SolrShardReporter">- <int name="period">5</int>- <str name="filter">UPDATE\./update.*requests</str>- <str name="filter">QUERY\./select.*requests</str>- </reporter>- <reporter name="test" group="cluster" class="org.apache.solr.metrics.reporters.solr.SolrClusterReporter">- <str name="handler">/admin/metrics/collector</str>- <int name="period">5</int>- <lst name="report">- <str name="group">cluster</str>- <str name="label">jvm</str>- <str name="registry">solr\.jvm</str>- <str name="filter">memory\.total\..*</str>- <str name="filter">memory\.heap\..*</str>- <str name="filter">os\.SystemLoadAverage</str>- <str name="filter">threads\.count</str>- </lst>- <lst name="report">- <str name="group">cluster</str>- <str name="label">leader.$1</str>- <str name="registry">solr\.collection\.(.*)\.leader</str>- <str name="filter">UPDATE\./update.*</str>- </lst>- </reporter>- </metrics>-</solr>diff --git a/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.javadeleted file mode 100644index 1a685d69994..00000000000--- a/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java+++ /dev/null@@ -1,190 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import java.io.ByteArrayInputStream;-import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.util.List;-import java.util.concurrent.TimeUnit;--import javax.imageio.ImageIO;--import org.apache.lucene.util.LuceneTestCase;-import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.SolrRequest;-import org.apache.solr.client.solrj.cloud.SolrCloudManager;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.request.GenericSolrRequest;-import org.apache.solr.common.params.CollectionAdminParams;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.params.SolrParams;-import org.apache.solr.common.util.Base64;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.common.util.SimpleOrderedMap;-import org.apache.solr.common.util.TimeSource;-import org.apache.solr.util.LogLevel;-import org.junit.AfterClass;-import org.junit.BeforeClass;-import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--/**- *- */-@LogLevel("org.apache.solr.handler.admin=DEBUG")-public class MetricsHistoryIntegrationTest extends SolrCloudTestCase {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- private static SolrCloudManager cloudManager;- private static SolrClient solrClient;- private static TimeSource timeSource;-- @BeforeClass- public static void setupCluster() throws Exception {- System.setProperty("metricsEnabled", "true");- configureCluster(1)- .addConfig("conf", configset("cloud-minimal"))- .configure();- cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();- solrClient = cluster.getSolrClient();- timeSource = cloudManager.getTimeSource();- // create .system- CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 1)- .process(solrClient);- CloudUtil.waitForState(cloudManager, CollectionAdminParams.SYSTEM_COLL,- 30, TimeUnit.SECONDS, CloudUtil.clusterShape(1, 1));- solrClient.query(CollectionAdminParams.SYSTEM_COLL, params(CommonParams.Q, "*:*"));- // sleep until next generation of kids grow up to allow the handler to collect some metrics- timeSource.sleep(100000);- }-- @AfterClass- public static void teardown() throws Exception {- solrClient = null;- cloudManager = null;- }-- @Test- public void testList() throws Exception {- NamedList<Object> rsp = solrClient.request(createHistoryRequest(params(CommonParams.ACTION, "list")));- assertNotNull(rsp);- // expected solr.jvm, solr.node and solr.collection..system- @SuppressWarnings({"unchecked"})- SimpleOrderedMap<Object> lst = (SimpleOrderedMap<Object>) rsp.get("metrics");- assertNotNull(lst);- assertEquals(lst.toString(), 3, lst.size());- assertNotNull(lst.toString(), lst.get("solr.jvm"));- assertNotNull(lst.toString(), lst.get("solr.node"));- assertNotNull(lst.toString(), lst.get("solr.collection..system"));- }-- @Test- @SuppressWarnings({"unchecked"})- public void testStatus() throws Exception {- NamedList<Object> rsp = solrClient.request(createHistoryRequest(- params(CommonParams.ACTION, "status", CommonParams.NAME, "solr.jvm")));- assertNotNull(rsp);- NamedList<Object> map = (NamedList<Object>)rsp.get("metrics");- assertEquals(map.toString(), 1, map.size());- map = (NamedList<Object>)map.get("solr.jvm");- assertNotNull(map);- NamedList<Object> status = (NamedList<Object>)map.get("status");- assertNotNull(status);- assertEquals(status.toString(), 7, status.size());- List<Object> lst = (List<Object>)status.get("datasources");- assertNotNull(lst);- assertEquals(lst.toString(), 3, lst.size());- lst = (List<Object>)status.get("archives");- assertNotNull(lst);- assertEquals(lst.toString(), 5, lst.size());- }-- @Test- @SuppressWarnings({"unchecked"})- public void testGet() throws Exception {- NamedList<Object> rsp = solrClient.request(createHistoryRequest(params(- CommonParams.ACTION, "get", CommonParams.NAME, "solr.jvm")));- assertNotNull(rsp);- // default format is LIST- NamedList<Object> data = (NamedList<Object>)rsp.findRecursive("metrics", "solr.jvm", "data");- assertNotNull(data);- data.forEach((k, v) -> {- NamedList<Object> entry = (NamedList<Object>)v;- List<Object> lst = entry.getAll("timestamps");- assertNotNull(lst);- assertTrue("timestamps", lst.size() > 0);- // 3 metrics, so the total size of values is 3 * the size of timestamps- entry = (NamedList<Object>)entry.get("values");- assertNotNull(entry);- assertEquals(lst.size() * 3, entry.size());- });-- // get STRING- rsp = solrClient.request(createHistoryRequest(params(- CommonParams.ACTION, "get", CommonParams.NAME, "solr.jvm", "format", "string")));- data = (NamedList<Object>)rsp.findRecursive("metrics", "solr.jvm", "data");- assertNotNull(data);- data.forEach((k, v) -> {- NamedList<Object> entry = (NamedList<Object>)v;- List<Object> lst = entry.getAll("timestamps");- assertNotNull(lst);- assertEquals("timestamps", 1, lst.size());- String timestampString = (String)lst.get(0);- String[] timestamps = timestampString.split(("\n"));- assertTrue(timestampString, timestamps.length > 1);- entry = (NamedList<Object>)entry.get("values");- assertNotNull(entry);- assertEquals(3, entry.size());- entry.forEach((vk, vv) -> {- String valString = (String)vv;- String[] values = valString.split("\n");- assertEquals(valString, timestamps.length, values.length);- });- });-- // get GRAPH- rsp = solrClient.request(createHistoryRequest(params(- CommonParams.ACTION, "get", CommonParams.NAME, "solr.jvm", "format", "graph")));- data = (NamedList<Object>)rsp.findRecursive("metrics", "solr.jvm", "data");- assertNotNull(data);- data.forEach((k, v) -> {- NamedList<Object> entry = (NamedList<Object>) v;- entry = (NamedList<Object>)entry.get("values");- assertNotNull(entry);- assertEquals(3, entry.size());- entry.forEach((vk, vv) -> {- String valString = (String)vv;- byte[] img = Base64.base64ToByteArray(valString);- try {- ImageIO.read(new ByteArrayInputStream(img));- } catch (IOException e) {- fail("Error reading image data: " + e.toString());- }- });- });- }-- @SuppressWarnings({"rawtypes"})- public static SolrRequest createHistoryRequest(SolrParams params) {- return new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/metrics/history", params);- }--}diff --git a/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryWithAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryWithAuthIntegrationTest.javadeleted file mode 100644index 6090361b9fd..00000000000--- a/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryWithAuthIntegrationTest.java+++ /dev/null@@ -1,93 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import java.util.List;--import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.cloud.SolrCloudManager;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.util.LogLevel;-import org.junit.AfterClass;-import org.junit.BeforeClass;-import org.junit.Test;--import static org.apache.solr.cloud.MetricsHistoryIntegrationTest.createHistoryRequest;--/**- * Tests that metrics history works even with Authentication enabled.- * We test that the scheduled calls to /admin/metrics use PKI auth and therefore succeeds- */-@LogLevel("org.apache.solr.handler.admin=DEBUG,org.apache.solr.security=DEBUG")-public class MetricsHistoryWithAuthIntegrationTest extends SolrCloudTestCase {-- private static SolrCloudManager cloudManager;- private static SolrClient solrClient;- private static final String SECURITY_JSON = "{\n" +- " 'authentication':{\n" +- " 'blockUnknown': false, \n" +- " 'class':'solr.BasicAuthPlugin',\n" +- " 'credentials':{'solr':'orwp2Ghgj39lmnrZOTm7Qtre1VqHFDfwAEzr0ApbN3Y= Ju5osoAqOX8iafhWpPP01E5P+sg8tK8tHON7rCYZRRw='}},\n" +- " 'authorization':{\n" +- " 'class':'solr.RuleBasedAuthorizationPlugin',\n" +- " 'user-role':{'solr':'admin'},\n" +- " 'permissions':[{'name':'metrics','collection': null,'path':'/admin/metrics','role':'admin'},\n" +- " {'name':'metrics','collection': null,'path':'/api/cluster/metrics','role':'admin'}]}}";- private static final CharSequence SOLR_XML_HISTORY_CONFIG =- "<history>\n" +- " <str name=\"collectPeriod\">2</str>\n" +- "</history>\n";-- @BeforeClass- public static void setupCluster() throws Exception {- String solrXml = MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML.replace("<metrics enabled=\"${metricsEnabled:false}\">\n",- "<metrics>\n" + SOLR_XML_HISTORY_CONFIG);- // Spin up a cluster with a protected /admin/metrics handler, and a 2 seconds metrics collectPeriod- configureCluster(1)- .addConfig("conf", configset("cloud-minimal"))- .withSecurityJson(SECURITY_JSON)- .withSolrXml(solrXml)- .configure();- cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();- solrClient = cluster.getSolrClient();- // sleep a little to allow the handler to collect some metrics- cloudManager.getTimeSource().sleep(3000);- }-- @AfterClass- public static void teardown() {- solrClient = null;- cloudManager = null;- }-- @SuppressWarnings("unchecked")- @Test- public void testValuesAreCollected() throws Exception {- NamedList<Object> rsp = solrClient.request(createHistoryRequest(params(- CommonParams.ACTION, "get", CommonParams.NAME, "solr.jvm")));- assertNotNull(rsp);- // default format is LIST- NamedList<Object> data = (NamedList<Object>)rsp.findRecursive("metrics", "solr.jvm", "data");- assertNotNull(data);-- // Has actual values. These will be 0.0 if metrics could not be collected- NamedList<Object> memEntry = (NamedList<Object>) ((NamedList<Object>) data.iterator().next().getValue()).get("values");- List<Double> heap = (List<Double>) memEntry.getAll("memory.heap.used").get(0);- assertTrue("Expected memory.heap.used > 0 in history", heap.get(240) > 0.01);- }-}\ No newline at end of filediff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.javadeleted file mode 100644index 835c8b190cc..00000000000--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java+++ /dev/null@@ -1,116 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.handler.admin;--import java.util.HashMap;-import java.util.List;-import java.util.Map;--import org.apache.lucene.util.LuceneTestCase;-import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.cloud.SolrCloudManager;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.cloud.CloudUtil;-import org.apache.solr.cloud.SolrCloudTestCase;-import org.apache.solr.common.params.CollectionAdminParams;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.util.Pair;-import org.apache.solr.common.util.TimeSource;-import org.apache.solr.core.SolrInfoBean;-import org.apache.solr.metrics.SolrMetricManager;-import org.apache.solr.metrics.SolrMetricsContext;-import org.apache.solr.util.LogLevel;-import org.junit.AfterClass;-import org.junit.BeforeClass;-import org.junit.Test;-import org.rrd4j.core.RrdDb;--/**- *- */-@LogLevel("org.apache.solr.cloud=DEBUG")-public class MetricsHistoryHandlerTest extends SolrCloudTestCase {-- private volatile static SolrCloudManager cloudManager;- private volatile static SolrMetricManager metricManager;- private volatile static TimeSource timeSource;- private volatile static SolrClient solrClient;- private volatile static int SPEED;-- private volatile static MetricsHistoryHandler handler;- private volatile static MetricsHandler metricsHandler;-- @BeforeClass- public static void beforeClass() throws Exception {- System.setProperty("metricsEnabled", "true");- Map<String, Object> args = new HashMap<>();- args.put(MetricsHistoryHandler.SYNC_PERIOD_PROP, 1);- args.put(MetricsHistoryHandler.COLLECT_PERIOD_PROP, 1);- configureCluster(1)- .addConfig("conf", configset("cloud-minimal"))- .configure();-- cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();- metricManager = cluster.getJettySolrRunner(0).getCoreContainer().getMetricManager();- solrClient = cluster.getSolrClient();- metricsHandler = new MetricsHandler(metricManager);- handler = new MetricsHistoryHandler(cluster.getJettySolrRunner(0).getNodeName(), metricsHandler, solrClient, cloudManager, args);- SolrMetricsContext solrMetricsContext = new SolrMetricsContext(metricManager, SolrInfoBean.Group.node.toString(), "");- handler.initializeMetrics(solrMetricsContext, CommonParams.METRICS_HISTORY_PATH);- SPEED = 1;- timeSource = cloudManager.getTimeSource();-- // create .system collection- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL,- "conf", 1, 1)- .setPerReplicaState(SolrCloudTestCase.USE_PER_REPLICA_STATE);- create.process(solrClient);- CloudUtil.waitForState(cloudManager, "failed to create " + CollectionAdminParams.SYSTEM_COLL,- CollectionAdminParams.SYSTEM_COLL, CloudUtil.clusterShape(1, 1));- }-- @AfterClass- public static void teardown() throws Exception {- if (handler != null) {- handler.close();- }- handler = null;- metricsHandler = null;- cloudManager = null;- metricManager = null;- solrClient = null;- }-- @Test- //Commented 14-Oct-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018- public void testBasic() throws Exception {- timeSource.sleep(15000);- List<Pair<String, Long>> list = handler.getFactory().list(100);- // solr.jvm, solr.node, solr.collection..system- assertEquals(list.toString(), 3, list.size());- for (Pair<String, Long> p : list) {- RrdDb db = new RrdDb(MetricsHistoryHandler.URI_PREFIX + p.first(), true, handler.getFactory());- int dsCount = db.getDsCount();- int arcCount = db.getArcCount();- assertTrue("dsCount should be > 0, was " + dsCount, dsCount > 0);- assertEquals("arcCount", 5, arcCount);- db.close();- }- }-}diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.javadeleted file mode 100644index c272439a7c1..00000000000--- a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java+++ /dev/null@@ -1,191 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.metrics.reporters.solr;--import java.nio.file.Paths;-import java.util.Map;--import org.apache.commons.io.IOUtils;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.cloud.SolrCloudTestCase;-import org.apache.solr.core.CoreContainer;-import org.apache.solr.core.SolrCore;-import org.apache.solr.metrics.AggregateMetric;-import org.apache.solr.metrics.SolrCoreContainerReporter;-import org.apache.solr.metrics.SolrCoreReporter;-import org.apache.solr.metrics.SolrMetricManager;-import org.apache.solr.metrics.SolrMetricReporter;-import org.apache.solr.metrics.reporters.SolrJmxReporter;-import org.junit.Before;-import org.junit.BeforeClass;-import org.junit.Test;--import com.codahale.metrics.Metric;--/**- *- */-public class SolrCloudReportersTest extends SolrCloudTestCase {- volatile int leaderRegistries;- volatile int clusterRegistries;- volatile int jmxReporter;---- @BeforeClass- public static void configureDummyCluster() throws Exception {- configureCluster(0).configure();- }-- @Before- public void closePreviousCluster() throws Exception {- shutdownCluster();- leaderRegistries = 0;- clusterRegistries = 0;- }-- @Test- // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018- public void testExplicitConfiguration() throws Exception {- String solrXml = IOUtils.toString(SolrCloudReportersTest.class.getResourceAsStream("/solr/solr-solrreporter.xml"), "UTF-8");- configureCluster(2)- .withSolrXml(solrXml).configure();- cluster.uploadConfigSet(Paths.get(TEST_PATH().toString(), "configsets", "minimal", "conf"), "test");-- CollectionAdminRequest.createCollection("test_collection", "test", 2, 2)- .process(cluster.getSolrClient());- cluster.waitForActiveCollection("test_collection", 2, 4);-- waitForState("Expected test_collection with 2 shards and 2 replicas", "test_collection", clusterShape(2, 4));-- // TODO this is no good- Thread.sleep(10000);-- cluster.getJettySolrRunners().forEach(jetty -> {- CoreContainer cc = jetty.getCoreContainer();- // verify registry names- for (String name : cc.getLoadedCoreNames()) {- SolrCore core = cc.getCore(name);- try {- String registryName = core.getCoreMetricManager().getRegistryName();- String leaderRegistryName = core.getCoreMetricManager().getLeaderRegistryName();- String coreName = core.getName();- String collectionName = core.getCoreDescriptor().getCollectionName();- String replicaName = coreName.substring(coreName.indexOf("_replica_") + 1);- String shardId = core.getCoreDescriptor().getCloudDescriptor().getShardId();-- assertEquals("solr.core." + collectionName + "." + shardId + "." + replicaName, registryName);- assertEquals("solr.collection." + collectionName + "." + shardId + ".leader", leaderRegistryName);-- } finally {- if (core != null) {- core.close();- }- }- }- SolrMetricManager metricManager = cc.getMetricManager();- Map<String, SolrMetricReporter> reporters = metricManager.getReporters("solr.cluster");- assertEquals(reporters.toString(), 1, reporters.size());- SolrMetricReporter reporter = reporters.get("test");- assertNotNull(reporter);- assertTrue(reporter.toString(), reporter instanceof SolrClusterReporter);- assertEquals(5, reporter.getPeriod());- assertTrue(reporter.toString(), reporter instanceof SolrCoreContainerReporter);- SolrCoreContainerReporter solrCoreContainerReporter = (SolrCoreContainerReporter)reporter;- assertNotNull(solrCoreContainerReporter.getCoreContainer());- for (String registryName : metricManager.registryNames(".*\\.shard[0-9]\\.replica.*")) {- reporters = metricManager.getReporters(registryName);- jmxReporter = 0;- reporters.forEach((k, v) -> {- if (v instanceof SolrJmxReporter) {- jmxReporter++;- }- });- assertEquals(reporters.toString(), 1 + jmxReporter, reporters.size());- reporter = null;- for (String name : reporters.keySet()) {- if (name.startsWith("test")) {- reporter = reporters.get(name);- }- }- assertNotNull(reporter);- assertTrue(reporter.toString(), reporter instanceof SolrShardReporter);- assertEquals(5, reporter.getPeriod());- assertTrue(reporter.toString(), reporter instanceof SolrCoreReporter);- SolrCoreReporter solrCoreReporter = (SolrCoreReporter)reporter;- assertNotNull(solrCoreReporter.getCore());- }- for (String registryName : metricManager.registryNames(".*\\.leader")) {- leaderRegistries++;- reporters = metricManager.getReporters(registryName);- // no reporters registered for leader registry- assertEquals(reporters.toString(), 0, reporters.size());- // verify specific metrics- Map<String, Metric> metrics = metricManager.registry(registryName).getMetrics();- String key = "QUERY./select.requests";- assertTrue(key, metrics.containsKey(key));- assertTrue(key, metrics.get(key) instanceof AggregateMetric);- key = "UPDATE./update.requests";- assertTrue(key, metrics.containsKey(key));- assertTrue(key, metrics.get(key) instanceof AggregateMetric);- }- if (metricManager.registryNames().contains("solr.cluster")) {- clusterRegistries++;- Map<String,Metric> metrics = metricManager.registry("solr.cluster").getMetrics();- String key = "jvm.memory.heap.init";- assertTrue(key, metrics.containsKey(key));- assertTrue(key, metrics.get(key) instanceof AggregateMetric);- key = "leader.test_collection.shard1.UPDATE./update.requests.max";- assertTrue(key, metrics.containsKey(key));- assertTrue(key, metrics.get(key) instanceof AggregateMetric);- }- });-- assertEquals("leaderRegistries", 2, leaderRegistries);- assertEquals("clusterRegistries", 1, clusterRegistries);- }-- @Test- // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018- public void testDefaultPlugins() throws Exception {- String solrXml = IOUtils.toString(SolrCloudReportersTest.class.getResourceAsStream("/solr/solr.xml"), "UTF-8");- configureCluster(2)- .withSolrXml(solrXml).configure();- cluster.uploadConfigSet(Paths.get(TEST_PATH().toString(), "configsets", "minimal", "conf"), "test");-- CollectionAdminRequest.createCollection("test_collection", "test", 2, 2)- .process(cluster.getSolrClient());- cluster.waitForActiveCollection("test_collection", 2, 4);- waitForState("Expected test_collection with 2 shards and 2 replicas", "test_collection", clusterShape(2, 4));- cluster.getJettySolrRunners().forEach(jetty -> {- CoreContainer cc = jetty.getCoreContainer();- SolrMetricManager metricManager = cc.getMetricManager();- Map<String, SolrMetricReporter> reporters = metricManager.getReporters("solr.cluster");- assertEquals(reporters.toString(), 0, reporters.size());- for (String registryName : metricManager.registryNames(".*\\.shard[0-9]\\.replica.*")) {- reporters = metricManager.getReporters(registryName);- jmxReporter = 0;- reporters.forEach((k, v) -> {- if (v instanceof SolrJmxReporter) {- jmxReporter++;- }- });- assertEquals(reporters.toString(), 0 + jmxReporter, reporters.size());- }- });- }-}diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.javadeleted file mode 100644index 3d8669acf0e..00000000000--- a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.java+++ /dev/null@@ -1,130 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.metrics.reporters.solr;--import java.lang.invoke.MethodHandles;-import java.util.Map;--import com.codahale.metrics.Metric;-import org.apache.solr.client.solrj.embedded.JettySolrRunner;-import org.apache.solr.cloud.AbstractFullDistribZkTestBase;-import org.apache.solr.cloud.CloudDescriptor;-import org.apache.solr.common.cloud.ClusterState;-import org.apache.solr.common.cloud.DocCollection;-import org.apache.solr.common.cloud.Slice;-import org.apache.solr.common.util.Utils;-import org.apache.solr.core.CoreContainer;-import org.apache.solr.core.CoreDescriptor;-import org.apache.solr.metrics.AggregateMetric;-import org.apache.solr.metrics.SolrCoreMetricManager;-import org.apache.solr.metrics.SolrMetricManager;-import org.junit.AfterClass;-import org.junit.BeforeClass;-import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--/**- *- */-public class SolrShardReporterTest extends AbstractFullDistribZkTestBase {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- public SolrShardReporterTest() {- schemaString = "schema15.xml"; // we need a string id- }-- @BeforeClass- public static void shardReporterBeforeClass() {- System.setProperty("solr.allowPaths", "*");- }-- @AfterClass- public static void shardReporterAfterClass() {- System.clearProperty("solr.allowPaths");- }-- @Override- public String getSolrXml() {- return "solr-solrreporter.xml";- }-- @Test- public void test() throws Exception {- waitForRecoveriesToFinish("control_collection",- jettys.get(0).getCoreContainer().getZkController().getZkStateReader(), false);- waitForRecoveriesToFinish("collection1",- jettys.get(0).getCoreContainer().getZkController().getZkStateReader(), false);- printLayout();- // wait for at least two reports- Thread.sleep(10000);- ClusterState state = jettys.get(0).getCoreContainer().getZkController().getClusterState();- for (JettySolrRunner jetty : jettys) {- CoreContainer cc = jetty.getCoreContainer();- SolrMetricManager metricManager = cc.getMetricManager();- for (final String coreName : cc.getLoadedCoreNames()) {- CoreDescriptor cd = cc.getCoreDescriptor(coreName);- if (cd.getCloudDescriptor() == null) { // not a cloud collection- continue;- }- CloudDescriptor cloudDesc = cd.getCloudDescriptor();- DocCollection docCollection = state.getCollection(cloudDesc.getCollectionName());- String replicaName = Utils.parseMetricsReplicaName(cloudDesc.getCollectionName(), coreName);- if (replicaName == null) {- replicaName = cloudDesc.getCoreNodeName();- }- String registryName = SolrCoreMetricManager.createRegistryName(true,- cloudDesc.getCollectionName(), cloudDesc.getShardId(), replicaName, null);- String leaderRegistryName = SolrCoreMetricManager.createLeaderRegistryName(true,- cloudDesc.getCollectionName(), cloudDesc.getShardId());- boolean leader = cloudDesc.isLeader();- Slice slice = docCollection.getSlice(cloudDesc.getShardId());- int numReplicas = slice.getReplicas().size();- if (leader) {- assertTrue(metricManager.registryNames() + " doesn't contain " + leaderRegistryName,- metricManager.registryNames().contains(leaderRegistryName));- Map<String, Metric> metrics = metricManager.registry(leaderRegistryName).getMetrics();- metrics.forEach((k, v) -> {- assertTrue("Unexpected type of " + k + ": " + v.getClass().getName() + ", " + v,- v instanceof AggregateMetric);- AggregateMetric am = (AggregateMetric)v;- if (!k.startsWith("REPLICATION.peerSync")) {- assertEquals(coreName + "::" + registryName + "::" + k + ": " + am.toString(), numReplicas, am.size());- }- });- } else {- assertFalse(metricManager.registryNames() + " contains " + leaderRegistryName +- " but it's not a leader!",- metricManager.registryNames().contains(leaderRegistryName));- Map<String, Metric> metrics = metricManager.registry(leaderRegistryName).getMetrics();- metrics.forEach((k, v) -> {- assertTrue("Unexpected type of " + k + ": " + v.getClass().getName() + ", " + v,- v instanceof AggregateMetric);- AggregateMetric am = (AggregateMetric)v;- if (!k.startsWith("REPLICATION.peerSync")) {- assertEquals(coreName + "::" + registryName + "::" + k + ": " + am.toString(), 1, am.size());- }- });- }- assertTrue(metricManager.registryNames() + " doesn't contain " + registryName,- metricManager.registryNames().contains(registryName));- }- }- SolrMetricManager metricManager = controlJetty.getCoreContainer().getMetricManager();- assertTrue(metricManager.registryNames().contains("solr.cluster"));- }-}diff --git a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.javadeleted file mode 100644index 2c7ea9d7242..00000000000--- a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java+++ /dev/null@@ -1,256 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.metrics.rrd;--import java.util.LinkedHashMap;-import java.util.List;-import java.util.Map;-import java.util.concurrent.TimeUnit;--import org.apache.solr.SolrTestCaseJ4;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.params.CollectionAdminParams;-import org.apache.solr.common.util.Pair;-import org.apache.solr.common.util.TimeSource;-import org.apache.solr.common.util.Utils;-import org.apache.solr.util.LogLevel;-import org.apache.solr.util.MockSearchableSolrClient;-import org.apache.solr.util.TimeOut;-import org.junit.After;-import org.junit.Before;-import org.junit.Test;-import org.rrd4j.ConsolFun;-import org.rrd4j.DsType;-import org.rrd4j.core.FetchData;-import org.rrd4j.core.FetchRequest;-import org.rrd4j.core.RrdDb;-import org.rrd4j.core.RrdDef;-import org.rrd4j.core.Sample;--/**- *- */-@LogLevel("org.apache.solr.metrics.rrd=DEBUG")-public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {-- private SolrRrdBackendFactory factory;- private MockSearchableSolrClient solrClient;- private TimeSource timeSource;-- @Before- public void setup() {- solrClient = new MockSearchableSolrClient();- if (random().nextBoolean()) {- timeSource = TimeSource.NANO_TIME;- } else {- timeSource = TimeSource.get("simTime:50");- }- factory = new SolrRrdBackendFactory(solrClient, CollectionAdminParams.SYSTEM_COLL, 1, timeSource);- }-- @After- public void teardown() throws Exception {- if (factory != null) {- factory.close();- }- }-- private RrdDef createDef(long startTime) {- RrdDef def = new RrdDef("solr:foo", 60);- def.setStartTime(startTime);- def.addDatasource("one", DsType.COUNTER, 120, Double.NaN, Double.NaN);- def.addDatasource("two", DsType.GAUGE, 120, Double.NaN, Double.NaN);- def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 120); // 2 hours- def.addArchive(ConsolFun.AVERAGE, 0.5, 10, 288); // 48 hours- def.addArchive(ConsolFun.AVERAGE, 0.5, 60, 336); // 2 weeks- def.addArchive(ConsolFun.AVERAGE, 0.5, 240, 180); // 2 months- return def;- }-- @Test- //commented 9-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018- // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018- public void testBasic() throws Exception {- long startTime = 1000000000;- RrdDb db = new RrdDb(createDef(startTime), factory);- long lastNumUpdates = solrClient.getNumUpdates();- List<Pair<String, Long>> list = factory.list(100);- assertEquals(list.toString(), 1, list.size());- assertEquals(list.toString(), "foo", list.get(0).first());-- timeSource.sleep(4000);- lastNumUpdates = waitForUpdates(lastNumUpdates);-- // wait until updates stop coming - the first update could have been partial- lastNumUpdates = waitForUpdatesToStop(lastNumUpdates);-- // there should be one sync data- assertEquals(solrClient.docs.toString(), 1, solrClient.docs.size());- String id = SolrRrdBackendFactory.ID_PREFIX + SolrRrdBackendFactory.ID_SEP + "foo";- SolrInputDocument doc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);- long timestamp = (Long)doc.getFieldValue("timestamp_l");-- timeSource.sleep(4000);-- SolrInputDocument newDoc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);- assertEquals(newDoc.toString(), newDoc, doc);- // make sure the update doesn't race with the sampling boundaries- long lastTime = startTime + 30;- // update the db- Sample s = db.createSample();- for (int i = 0; i < 100; i++) {- s.setTime(lastTime);- s.setValue("one", 1000 + i * 60);- s.setValue("two", 100);- s.update();- lastTime = lastTime + 60;- }- timeSource.sleep(3000);- lastNumUpdates = waitForUpdates(lastNumUpdates);-- newDoc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);- assertFalse(newDoc.toString(), newDoc.equals(doc));- long newTimestamp = (Long)newDoc.getFieldValue("timestamp_l");- assertNotSame(newTimestamp, timestamp);- // don't race with the sampling boundary- FetchRequest fr = db.createFetchRequest(ConsolFun.AVERAGE, startTime + 20, lastTime - 20, 60);- FetchData fd = fr.fetchData();- int rowCount = fd.getRowCount();- double[] one = fd.getValues("one");- double[] two = fd.getValues("two");- String dump = dumpData(db, fd);- assertEquals("one: " + dump, 101, one.length);- assertEquals(dump, Double.NaN, one[0], 0.00001);- assertEquals(dump, Double.NaN, one[100], 0.00001);- for (int i = 1; i < 100; i++) {- assertEquals(dump + "\npos=" + i, 1.0, one[i], 0.00001);- }- assertEquals("two: " + dump, Double.NaN, two[100], 0.00001);- for (int i = 0; i < 100; i++) {- assertEquals(dump + "\ntwo pos=" + i, 100.0, two[i], 0.00001);- }- db.close();-- // should still be listed- list = factory.list(100);- assertEquals(list.toString(), 1, list.size());- assertEquals(list.toString(), "foo", list.get(0).first());-- lastNumUpdates = solrClient.getNumUpdates();-- // re-open read-write- db = new RrdDb("solr:foo", factory);- s = db.createSample();- s.setTime(lastTime);- s.setValue("one", 7000);- s.setValue("two", 100);- s.update();- timeSource.sleep(3000);- lastNumUpdates = waitForUpdates(lastNumUpdates);-- // should update- timestamp = newTimestamp;- doc = newDoc;- newDoc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);- assertFalse(newDoc.toString(), newDoc.equals(doc));- newTimestamp = (Long)newDoc.getFieldValue("timestamp_l");- assertNotSame(newTimestamp, timestamp);- fr = db.createFetchRequest(ConsolFun.AVERAGE, startTime + 20, lastTime + 20, 60);- fd = fr.fetchData();- dump = dumpData(db, fd);- rowCount = fd.getRowCount();- one = fd.getValues("one");- assertEquals("one: " + dump, 102, one.length);- assertEquals(dump, Double.NaN, one[0], 0.01);- assertEquals(dump, Double.NaN, one[101], 0.01);- for (int i = 1; i < 101; i++) {- assertEquals(dump, 1.0, one[i], 0.01);- }- two = fd.getValues("two");- assertEquals("two: " + dump, Double.NaN, two[101], 0.001);- for (int i = 1; i < 101; i++) {- assertEquals(dump, 100.0, two[i], 0.001);- }-- db.close();-- // open a read-only version of the db- RrdDb readOnly = new RrdDb("solr:foo", true, factory);- s = readOnly.createSample();- s.setTime(lastTime + 120);- s.setValue("one", 10000001);- s.setValue("two", 100);- s.update();- // these updates should not be synced- timeSource.sleep(3000);- doc = newDoc;- timestamp = newTimestamp;- newDoc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);- assertTrue(newDoc.toString(), newDoc.equals(doc));- newTimestamp = (Long)newDoc.getFieldValue("timestamp_l");- assertEquals(newTimestamp, timestamp);- readOnly.close();- }-- private String dumpData(RrdDb db, FetchData fd) throws Exception {- Map<String, Object> map = new LinkedHashMap<>();- map.put("dbLastUpdateTime", db.getLastUpdateTime());- map.put("firstTimestamp", fd.getFirstTimestamp());- map.put("lastTimestamp", fd.getLastTimestamp());- map.put("timestamps", fd.getTimestamps());- map.put("data", fd.dump());- return Utils.toJSONString(map);- }-- private long waitForUpdates(long lastNumUpdates) throws Exception {- TimeOut timeOut = new TimeOut(30, TimeUnit.SECONDS, timeSource);- while (!timeOut.hasTimedOut()) {- timeOut.sleep(1000);- if (solrClient.getNumUpdates() > lastNumUpdates) {- return solrClient.getNumUpdates();- }- }- if (solrClient.getNumUpdates() > lastNumUpdates) {- return solrClient.getNumUpdates();- }- throw new Exception("time out waiting for updates");- }--- private long waitForUpdatesToStop(long lastNumUpdates) throws Exception {- TimeOut timeOut = new TimeOut(30, TimeUnit.SECONDS, timeSource);- int stopped = 0;- while (!timeOut.hasTimedOut()) {- timeOut.sleep(1000);- if (solrClient.getNumUpdates() > lastNumUpdates) {- stopped = 0;- lastNumUpdates = solrClient.getNumUpdates();- continue;- } else {- stopped++;- if (stopped > 2) {- return lastNumUpdates;- }- }- timeOut.sleep(1000);- }- throw new Exception("time out waiting for updates");- }---}diff --git a/solr/licenses/rrd4j-3.5.jar.sha1 b/solr/licenses/rrd4j-3.5.jar.sha1deleted file mode 100644index 8277f7ee829..00000000000--- a/solr/licenses/rrd4j-3.5.jar.sha1+++ /dev/null@@ -1 +0,0 @@-540c946b471dc915b0beb7c07069e3946665ef5ddiff --git a/solr/licenses/rrd4j-LICENSE-ASL.txt b/solr/licenses/rrd4j-LICENSE-ASL.txtdeleted file mode 100644index 7a4a3ea2424..00000000000--- a/solr/licenses/rrd4j-LICENSE-ASL.txt+++ /dev/null@@ -1,202 +0,0 @@-- Apache License- Version 2.0, January 2004- http://www.apache.org/licenses/-- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION-- 1. Definitions.-- "License" shall mean the terms and conditions for use, reproduction,- and distribution as defined by Sections 1 through 9 of this document.-- "Licensor" shall mean the copyright owner or entity authorized by- the copyright owner that is granting the License.-- "Legal Entity" shall mean the union of the acting entity and all- other entities that control, are controlled by, or are under common- control with that entity. For the purposes of this definition,- "control" means (i) the power, direct or indirect, to cause the- direction or management of such entity, whether by contract or- otherwise, or (ii) ownership of fifty percent (50%) or more of the- outstanding shares, or (iii) beneficial ownership of such entity.-- "You" (or "Your") shall mean an individual or Legal Entity- exercising permissions granted by this License.-- "Source" form shall mean the preferred form for making modifications,- including but not limited to software source code, documentation- source, and configuration files.-- "Object" form shall mean any form resulting from mechanical- transformation or translation of a Source form, including but- not limited to compiled object code, generated documentation,- and conversions to other media types.-- "Work" shall mean the work of authorship, whether in Source or- Object form, made available under the License, as indicated by a- copyright notice that is included in or attached to the work- (an example is provided in the Appendix below).-- "Derivative Works" shall mean any work, whether in Source or Object- form, that is based on (or derived from) the Work and for which the- editorial revisions, annotations, elaborations, or other modifications- represent, as a whole, an original work of authorship. For the purposes- of this License, Derivative Works shall not include works that remain- separable from, or merely link (or bind by name) to the interfaces of,- the Work and Derivative Works thereof.-- "Contribution" shall mean any work of authorship, including- the original version of the Work and any modifications or additions- to that Work or Derivative Works thereof, that is intentionally- submitted to Licensor for inclusion in the Work by the copyright owner- or by an individual or Legal Entity authorized to submit on behalf of- the copyright owner. For the purposes of this definition, "submitted"- means any form of electronic, verbal, or written communication sent- to the Licensor or its representatives, including but not limited to- communication on electronic mailing lists, source code control systems,- and issue tracking systems that are managed by, or on behalf of, the- Licensor for the purpose of discussing and improving the Work, but- excluding communication that is conspicuously marked or otherwise- designated in writing by the copyright owner as "Not a Contribution."-- "Contributor" shall mean Licensor and any individual or Legal Entity- on behalf of whom a Contribution has been received by Licensor and- subsequently incorporated within the Work.-- 2. Grant of Copyright License. Subject to the terms and conditions of- this License, each Contributor hereby grants to You a perpetual,- worldwide, non-exclusive, no-charge, royalty-free, irrevocable- copyright license to reproduce, prepare Derivative Works of,- publicly display, publicly perform, sublicense, and distribute the- Work and such Derivative Works in Source or Object form.-- 3. Grant of Patent License. Subject to the terms and conditions of- this License, each Contributor hereby grants to You a perpetual,- worldwide, non-exclusive, no-charge, royalty-free, irrevocable- (except as stated in this section) patent license to make, have made,- use, offer to sell, sell, import, and otherwise transfer the Work,- where such license applies only to those patent claims licensable- by such Contributor that are necessarily infringed by their- Contribution(s) alone or by combination of their Contribution(s)- with the Work to which such Contribution(s) was submitted. If You- institute patent litigation against any entity (including a- cross-claim or counterclaim in a lawsuit) alleging that the Work- or a Contribution incorporated within the Work constitutes direct- or contributory patent infringement, then any patent licenses- granted to You under this License for that Work shall terminate- as of the date such litigation is filed.-- 4. Redistribution. You may reproduce and distribute copies of the- Work or Derivative Works thereof in any medium, with or without- modifications, and in Source or Object form, provided that You- meet the following conditions:-- (a) You must give any other recipients of the Work or- Derivative Works a copy of this License; and-- (b) You must cause any modified files to carry prominent notices- stating that You changed the files; and-- (c) You must retain, in the Source form of any Derivative Works- that You distribute, all copyright, patent, trademark, and- attribution notices from the Source form of the Work,- excluding those notices that do not pertain to any part of- the Derivative Works; and-- (d) If the Work includes a "NOTICE" text file as part of its- distribution, then any Derivative Works that You distribute must- include a readable copy of the attribution notices contained- within such NOTICE file, excluding those notices that do not- pertain to any part of the Derivative Works, in at least one- of the following places: within a NOTICE text file distributed- as part of the Derivative Works; within the Source form or- documentation, if provided along with the Derivative Works; or,- within a display generated by the Derivative Works, if and- wherever such third-party notices normally appear. The contents- of the NOTICE file are for informational purposes only and- do not modify the License. You may add Your own attribution- notices within Derivative Works that You distribute, alongside- or as an addendum to the NOTICE text from the Work, provided- that such additional attribution notices cannot be construed- as modifying the License.-- You may add Your own copyright statement to Your modifications and- may provide additional or different license terms and conditions- for use, reproduction, or distribution of Your modifications, or- for any such Derivative Works as a whole, provided Your use,- reproduction, and distribution of the Work otherwise complies with- the conditions stated in this License.-- 5. Submission of Contributions. Unless You explicitly state otherwise,- any Contribution intentionally submitted for inclusion in the Work- by You to the Licensor shall be under the terms and conditions of- this License, without any additional terms or conditions.- Notwithstanding the above, nothing herein shall supersede or modify- the terms of any separate license agreement you may have executed- with Licensor regarding such Contributions.-- 6. Trademarks. This License does not grant permission to use the trade- names, trademarks, service marks, or product names of the Licensor,- except as required for reasonable and customary use in describing the- origin of the Work and reproducing the content of the NOTICE file.-- 7. Disclaimer of Warranty. Unless required by applicable law or- agreed to in writing, Licensor provides the Work (and each- Contributor provides its Contributions) on an "AS IS" BASIS,- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or- implied, including, without limitation, any warranties or conditions- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A- PARTICULAR PURPOSE. You are solely responsible for determining the- appropriateness of using or redistributing the Work and assume any- risks associated with Your exercise of permissions under this License.-- 8. Limitation of Liability. In no event and under no legal theory,- whether in tort (including negligence), contract, or otherwise,- unless required by applicable law (such as deliberate and grossly- negligent acts) or agreed to in writing, shall any Contributor be- liable to You for damages, including any direct, indirect, special,- incidental, or consequential damages of any character arising as a- result of this License or out of the use or inability to use the- Work (including but not limited to damages for loss of goodwill,- work stoppage, computer failure or malfunction, or any and all- other commercial damages or losses), even if such Contributor- has been advised of the possibility of such damages.-- 9. Accepting Warranty or Additional Liability. While redistributing- the Work or Derivative Works thereof, You may choose to offer,- and charge a fee for, acceptance of support, warranty, indemnity,- or other liability obligations and/or rights consistent with this- License. However, in accepting such obligations, You may act only- on Your own behalf and on Your sole responsibility, not on behalf- of any other Contributor, and only if You agree to indemnify,- defend, and hold each Contributor harmless for any liability- incurred by, or claims asserted against, such Contributor by reason- of your accepting any such warranty or additional liability.-- END OF TERMS AND CONDITIONS-- APPENDIX: How to apply the Apache License to your work.-- To apply the Apache License to your work, attach the following- boilerplate notice, with the fields enclosed by brackets "[]"- replaced with your own identifying information. (Don't include- the brackets!) The text should be enclosed in the appropriate- comment syntax for the file format. We also recommend that a- file or class name and description of purpose be included on the- same "printed page" as the copyright notice for easier- identification within third-party archives.-- Copyright [yyyy] [name of copyright owner]-- Licensed under the Apache License, Version 2.0 (the "License");- you may not use this file except in compliance with the License.- You may obtain a copy of the License at-- http://www.apache.org/licenses/LICENSE-2.0-- Unless required by applicable law or agreed to in writing, software- distributed under the License is distributed on an "AS IS" BASIS,- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- See the License for the specific language governing permissions and- limitations under the License.\ No newline at end of filediff --git a/solr/licenses/rrd4j-NOTICE.txt b/solr/licenses/rrd4j-NOTICE.txtdeleted file mode 100644index 841dd309998..00000000000--- a/solr/licenses/rrd4j-NOTICE.txt+++ /dev/null@@ -1,2 +0,0 @@-GitHub: https://github.com/rrd4j/rrd4j/-Maven-generated site: http://rrd4j.org/site/diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-8.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-8.adocindex 33dcccf2ff4..a1b1aac89cc 100644--- a/solr/solr-ref-guide/src/major-changes-in-solr-8.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-8.adoc@@ -485,7 +485,7 @@ Existing users of this feature should not have to change anything. However, they** If a reporter configures the `group="shard"` attribute then please also configure the `class="org.apache.solr.metrics.reporters.solr.SolrShardReporter"` attribute.** If a reporter configures the `group="cluster"` attribute then please also configure the `class="org.apache.solr.metrics.reporters.solr.SolrClusterReporter"` attribute.+-See the section <<metrics-reporting.adoc#shard-and-cluster-reporters,Shard and Cluster Reporters>> for more information.+See the section in Metrics Reporting: Shard and Cluster Reporters for more information.*Streaming Expressions*diff --git a/solr/solr-ref-guide/src/metrics-history.adoc b/solr/solr-ref-guide/src/metrics-history.adocdeleted file mode 100644index 37138fcf48d..00000000000--- a/solr/solr-ref-guide/src/metrics-history.adoc+++ /dev/null@@ -1,488 +0,0 @@-= Metrics History-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Solr collects long-term history of certain key metrics both in SolrCloud and in standalone mode.--This information can be used for very simple monitoring and troubleshooting, but also some-SolrCloud components can use this data for making informed decisions based on-long-term trends of selected metrics.--[IMPORTANT]-====-The `.system` collection must exist if metrics history should be persisted. If this collection-is absent then metrics history will still be collected and kept in memory but it will be lost-on node restart.-====--== Design--Before discussing how to configure metrics storage, a bit of explanation about how it works may be helpful.--=== Round-Robin Databases--The metrics history data is maintained as multi-resolution time series, with a fixed total number of data points-per metric history (a fixed size window). Multi-resolution refers to the fact that data from the most detailed-time series is periodically resampled to create coarser-grained time series, which in turn-are periodically resampled again to build even coarser-grained series.--In the default configuration selected metrics are sampled every 60 seconds, and the following-time series are built:--* 240 samples, every 60 sec (4 hours)-* 288 samples, every 600 sec (48 hours)-* 336 samples, every 1h (2 weeks)-* 180 samples, every 4h (2 months)-* 365 samples, every 1 day (1 year)--This means that the total number of samples in all data series is constant, and consequently-the size of this data structure is also constant (because the size of the moving window is fixed, and-older samples are replaced by newer ones). This arrangement is referred to as a-round-robin database, and Solr uses implementation of this concept provided by the https://github.com/rrd4j/rrd4j[RRD4j] library.--=== Storage-Databases created with RRD4j are compact - for the time series specified above the total size-of data is around 11kB for each of the primary time series, including its resampled data. Each database may contain-several primary time series ("datasources" in RRD4j parlance) and their re-sampled versions (called-"archives").--This data is updated in memory and then periodically stored in the `.system`-collection in the form of Solr documents with a binary `data_bin` field, each document-containing data of one full database. This method of storage is much more compact and generates less-update operations than storing each data point in a separate Solr document. The Metrics History API allows retrieving-detailed data from each database, including retrieval of all individual datapoints.--Databases are identified primarily by their corresponding metric registry name, so for databases that-keep track of aggregated metrics this will be e.g., `solr.jvm`, `solr.node`, `solr.collection.gettingstarted`.-For databases with non-aggregated metrics the name consists of the registry name, optionally with a node name-to identify databases with the same name coming from different nodes. For example, per-node databases are-named like this: `solr.jvm.localhost:8983_solr`, `solr.node.localhost:7574_solr`, but per-replica names are-already unique across the cluster so they are named like this: `solr.core.gettingstarted.shard1.replica_n1`.--=== Collected Metrics-Currently the following selected metrics are tracked:--* Non-aggregated `solr.core` and aggregated `solr.collection` metrics:-** `QUERY./select.requests`-** `UPDATE./update.requests`-** `INDEX.sizeInBytes`-** `numShards` (aggregated, active shards)-** `numReplicas` (aggregated, active replicas)--* `solr.node` metrics:-** `CONTAINER.fs.coreRoot.usableSpace`-** `numNodes` (aggregated, number of live nodes)--* `solr.jvm` metrics:-** `memory.heap.used`-** `os.processCpuLoad`-** `os.systemLoadAverage`--Separate databases are created for each of these groups, and each database keeps data for-all metrics listed in that group.--NOTE: Currently this list is not configurable. Also, if you change this list in the code then-all existing databases must be first removed from the `.system` collection because RRD4j doesn't allow-adding new datasources once the database is created.--=== SolrRrdBackendFactory-This component is responsible for managing in-memory databases and periodically saving them-to the `.system` collection. If the `.system` collection is not available the updates to the-databases will be kept in memory, until the time when `.system` collection becomes available.--If the `.system` collection is permanently unavailable then data will not be saved and it will-be lost when the Solr node is shut down.--=== MetricsHistoryHandler-This component provides a REST API for accessing the metrics history. It is also responsible for-collecting and periodically updating the in-memory databases.--This handler also performs aggregation of metrics on per-collection level, and on a cluster level.-By default only these aggregated metrics are tracked - historic data from each node and each replica-in each collection is not collected separately. Aggregated databases are managed on the Overseer leader-node but they are still accessible from other nodes even if they are not persisted - the handler redirects-the call from originating node to the current Overseer leader.--The handler assumes that a simple aggregation (sum of partial metric values from each resource) is-sufficient. This happens to make sense for the default built-in sets of metrics. Future extensions will-provide other aggregation strategies (such as, average, max, min, etc.).--This handler is automatically disabled when metrics collection is disabled using the-`<metrics enabled="false">` element in `solr.xml`.--== Metrics History Configuration-There are two ways to configure this subsystem:--* `/clusterprops.json` - this is the primary mechanism. It uses the cluster properties JSON-file in ZooKeeper. Configuration is stored in the `/metrics/history` element in a JSON map.--* `solr.xml` - this is the secondary mechanism, which is not recommended but provided for consistency-with the existing metrics configuration section in this file. Configuration is stored in the-`/solr/metrics/history` element of this file.--Currently the following configuration options are supported:--`enable`:: boolean, default is `true`.-If this is `false` then metrics history is not collected.-When this is `true` then metrics are periodically collected, aggregated and saved.--`enableReplicas`:: boolean, default is `false`. When this is `true` non-aggregated history will be-collected for each replica in each collection. When this is `false` then only aggregated history-is collected for each collection.--`enableNodes`:: boolean, default is `false`. When this is `true` then non-aggregated history will be-collected separately for each node (for node and JVM metrics), with database names consisting of-base registry name with appended node name, e.g., `solr.jvm.localhost:8983_solr`. When this is `false`-then only aggregated history will be collected in a single `solr.jvm` and `solr.node` cluster-wide-databases.--`collectPeriod`:: integer, in seconds, default is `60`. Metrics values will be collected and respective-databases updated every `collectPeriod` seconds.-+-[IMPORTANT]-====-Value of `collectPeriod` must be at least 1, and if it's changed then all previously existing databases-with their historic data must be manually removed (new databases will be created automatically).-====--`syncPeriod`:: integer, in seconds, default is `60`. Data from modified databases will be saved to Solr-every `syncPeriod` seconds. When accessing the databases via REST API in `index` mode the visibility of-most recent data depends on this period, because requests accessing the data from other nodes see only-the version of the data that is stored in the `.system` collection.--=== Example Configuration-Example `/clusterprops.json` file with metrics history configuration that turns on the collection of-per-node metrics history for node and JVM metrics. Typically this file will also contain other-properties unrelated to Metrics History API.--[source,json]------{- "metrics" : {- "history" : {- "enable" : true,- "enableNodes" : true,- "syncPeriod" : 300- }- }-}-------== Metrics History API-Main entry point for accessing metrics history is `/admin/metrics/history` (or `/api/cluster/metrics/history`-for v2 API).--The following sections describe actions available in this API. All calls have at least one-required parameter `action`.--All responses contain a section named `state`, which reports the current internal state of the API:--`enableReplicas`:: boolean, corresponds to the `enableReplicas` configuration setting.--`enableNodes`:: boolean, corresponds to the `enableNodes` configuration setting.--`mode`:: one of the following values:--* `inactive` - when metrics collection is disabled (but access to existing metrics history is still available).-* `memory` - when metrics history is kept only in memory because `.system` collection doesn't exist. In this mode-clients can access metrics history available on the node that received the request and on the Overseer leader.-* `index` - when metrics history is periodically stored in the `.system` collection. Data available in memory on-the node that accepted the request is retrieved from memory, any other data is retrieved from the-`.system` collection (so it's at least `syncPeriod` old).--Also, the response header section (`responseHeader`) contains `zkConnected` boolean property that indicates-whether the current node is a part of SolrCloud cluster.--=== List Databases-The query parameter `action=list` produces a list of available databases. It supports the following parameters:--`rows`:: optional integer, default is `500`. Maximum number of results to return.--Example:-In this SolrCloud example the API is in `memory` mode, and the request was made to a node that is-not Overseer leader. The API transparently forwarded the request to Overseer leader.--[source,bash]------curl http://localhost:7574/solr/admin/metrics/history?action=list&rows=10------[source,json]------{- "responseHeader": {- "zkConnected": true,- "status": 0,- "QTime": 9- },- "metrics": {- "solr.collection..system": {- "lastModified": 1528360138,- "node": "127.0.0.1:8983_solr"- },- "solr.collection.gettingstarted": {- "lastModified": 1528360138,- "node": "127.0.0.1:8983_solr"- },- "solr.jvm": {- "lastModified": 1528360138,- "node": "127.0.0.1:8983_solr"- },- "solr.node": {- "lastModified": 1528360138,- "node": "127.0.0.1:8983_solr"- }- },- "state": {- "enableReplicas": false,- "enableNodes": false,- "mode": "memory"- }-}-------Note the presence of the `node` element in each section, which shows where the information is coming-from - when API is in `memory` mode this indicates which results are local and which ones are retrieved-from the Overseer leader node. When the API is in `index` mode this element always shows the node name that-received the request (because the data is retrieved from the `.system` collection anyway).--Each section also contains a `lastModified` element, which contains the last modification time when the-database was update. All timestamps returned from this API correspond to Unix epoch time in seconds.--=== Database Status-The query parameter `action=status` provides detailed status of the selected database.--The following parameters are supported:--`name`:: string, required: database name.--Example:-[source,bash]------curl http://localhost:7574/solr/admin/metrics/history?action=status&name=solr.collection.gettingstarted------[source,json]------{- "responseHeader": {- "zkConnected": true,- "status": 0,- "QTime": 46- },- "metrics": {- "solr.collection.gettingstarted": {- "status": {- "lastModified": 1528318361,- "step": 60,- "datasourceCount": 5,- "archiveCount": 5,- "datasourceNames": [- "numShards",- "numReplicas",- "QUERY./select.requests",- "UPDATE./update.requests",- "INDEX.sizeInBytes"- ],- "datasources": [- {- "datasource": "DS:numShards:GAUGE:120:U:U",- "lastValue": 2- },- {- "datasource": "DS:numReplicas:GAUGE:120:U:U",- "lastValue": 4- },- "..."- ],- "archives": [- {- "archive": "RRA:AVERAGE:0.5:1:240",- "steps": 1,- "consolFun": "AVERAGE",- "xff": 0.5,- "startTime": 1528303980,- "endTime": 1528318320,- "rows": 240- },- {- "archive": "RRA:AVERAGE:0.5:10:288",- "steps": 10,- "consolFun": "AVERAGE",- "xff": 0.5,- "startTime": 1528146000,- "endTime": 1528318200,- "rows": 288- },- "..."- ]- },- "node": "127.0.0.1:7574_solr"- }- },- "state": {- "enableReplicas": false,- "enableNodes": false,- "mode": "index"- }-}-------=== Get Database Data-The query parameter `action=get` retrieves all data collected in the specified database.--The following parameters are supported:--`name`:: string, required: database name.-`format`:: string, optional, default is `list`. Format of the data. Currently the-following formats are supported:--* `list` - each datapoint is returned as separate JSON element. For efficiency, for each-datasource in a database for each time series the timestamps are provided separately from-values (because points from all datasources in a given time series share the same timestamps).-* `string` - all datapoint values and timestamps are returned as strings, with values separated by new line character.-* `graph` - data is returned as PNG images, Base64-encoded, containing graphs of each time series values over time.--In each case the response is structured in a similar way: archive identifiers are keys in a JSON map,-all data is placed in a `data` element, with timestamps / datapoints / graphs as values in lists or maps.--==== Examples-This is the output using the default `list` format:-[source,bash]------curl http://localhost:8983/solr/admin/metrics/history?action=get&name=solr.collection.gettingstarted------[source,json]------{- "responseHeader": {- "zkConnected": true,- "status": 0,- "QTime": 4- },- "metrics": {- "solr.collection.gettingstarted": {- "data": {- "RRA:AVERAGE:0.5:1:240": {- "timestamps": [- 1528304160,- 1528304220,- "..."- ],- "values": {- "numShards": [- "NaN",- 2.0,- "..."- ],- "numReplicas": [- "NaN",- 4.0,- "..."- ],- }- },- "RRA:AVERAGE:0.5:10:288": {- "timestamps": [- 1528145400,- 1528146000,- ],- "lastModified": 1528318606,- "node": "127.0.0.1:8983_solr"- }- },- "state": {- "enableReplicas": false,- "enableNodes": false,- "mode": "index"- }}}}-------This is the output when using the `string` format:-[source,bash]------curl http://localhost:8983/solr/admin/metrics/history?action=get&name=solr.collection.gettingstarted&format=string------[source,json]------{- "responseHeader": {- "zkConnected": true,- "status": 0,- "QTime": 2- },- "metrics": {- "solr.collection.gettingstarted": {- "data": {- "RRA:AVERAGE:0.5:1:240": {- "timestamps": "1527254820\n1527254880\n1527254940\n...",- "values": {- "numShards": "NaN\n2.0\n2.0\n2.0\n2.0\n2.0\n2.0\n...",- "numReplicas": "NaN\n4.0\n4.0\n4.0\n4.0\n4.0\n4.0\n...",- "QUERY./select.requests": "NaN\n123\n456\n789\n...",- "..."- }- },- "RRA:AVERAGE:0.5:10:288": {- "..."- }}}}}-------This is the output when using the `graph` format:-[source,bash]------curl http://localhost:8983/solr/admin/metrics/history?action=get&name=solr.collection.gettingstarted&format=graph------[source,json]------{- "responseHeader": {- "zkConnected": true,- "status": 0,- "QTime": 2- },- "metrics": {- "solr.collection.gettingstarted": {- "data": {- "RRA:AVERAGE:0.5:1:240": {- "values": {- "numShards": "iVBORw0KGgoAAAANSUhEUgAAAkQAAA...",- "numReplicas": "iVBORw0KGgoAAAANSUhEUgAAAkQA...",- "QUERY./select.requests": "iVBORw0KGgoAAAANS...",- "..."- }- },- "RRA:AVERAGE:0.5:10:288": {- "values": {- "numShards": "iVBORw0KGgoAAAANSUhEUgAAAkQAAA...",- "..."- }- }}}}}-------.Example 60 sec resolution history graph for `QUERY./select.requests` metric-image::images/metrics-history/query-graph-60s.png[image]--.Example 10 min resolution history graph for `QUERY./select.requests` metric-image::images/metrics-history/query-graph-10min.png[image]--.Example 60 sec resolution history graph for `UPDATE./update.requests` metric-image::images/metrics-history/update-graph-60s.png[image]--.Example 60 sec resolution history graph for `memory.heap.used` metric-image::images/metrics-history/memHeap-60s.png[image]--.Example 60 sec resolution history graph for `os.systemLoadAverage` metric-image::images/metrics-history/loadAvg-60s.png[image]diff --git a/solr/solr-ref-guide/src/metrics-reporting.adoc b/solr/solr-ref-guide/src/metrics-reporting.adocindex 73c244322bd..6fd1fe4c194 100644--- a/solr/solr-ref-guide/src/metrics-reporting.adoc+++ b/solr/solr-ref-guide/src/metrics-reporting.adoc@@ -89,8 +89,6 @@ This registry is returned at `solr.jetty` and includes the following information* connection and request timers,* meters for responses by HTTP class (1xx, 2xx, etc.)-In the future, metrics will be added for shard leaders and cluster nodes, including aggregations from per-core metrics.-== Metrics ConfigurationThe metrics available in your system can be customized by modifying the `<metrics>` element in `solr.xml`.@@ -103,13 +101,12 @@ for example `<metrics enabled="true">`.The default value of this attribute is `true`, meaning that metrics are being collected, processed andreported by Solr according to the configured metric reporters. They are also available from the-metrics and metrics history APIs.+metrics APIs.-The `false` value of this attribute (`<metrics enabled="false">`) turns off metrics collection, processing,-and the collection of metrics history. Internally, all metrics suppliers are replaced by singleton no-op+The `false` value of this attribute (`<metrics enabled="false">`) turns off metrics collection and processing.+Internally, all metrics suppliers are replaced by singleton no-opimplementations, which effectively removes nearly all overheads related to metrics collection.-All reporter configurations are skipped, and the metrics-and metrics history APIs stop reporting any metrics and only return an `<error>`+All reporter configurations are skipped, and the metrics APIs stop reporting any metrics and only return an `<error>`element in their responses.=== The <metrics> <hiddenSysProps> Element@@ -427,169 +424,6 @@ example.solr.jetty.org.eclipse.jetty.server.handler.DefaultHandler.2xx-responsesexample.solr.jetty.org.eclipse.jetty.server.handler.DefaultHandler.2xx-responses.mean_rate 0.5698031798408144 1482932097-----=== Ganglia Reporter--The http://ganglia.info[Ganglia] reporter uses the `org.apache.solr.metrics.reporters.SolrGangliaReporter` class.--It take the following arguments, in addition to the common arguments <<Reporter Arguments,above>>.--`host`::-The host name where Ganglia server is running (required).--`port`::-The port number for the server.--`multicast`::-When `true` use multicast UDP communication, otherwise use UDP unicast. Default is `false`.--=== Shard and Cluster Reporters-These two reporters can be used for aggregation of metrics reported from replicas to shard leader (the "shard" reporter),-and from any local registry to the Overseer node.--Metric reports from these reporters are periodically sent as batches of regular SolrInputDocuments,-so they can be processed by any Solr handler. By default they are sent to `/admin/metrics/collector` handler-(an instance of `MetricsCollectorHandler`) on a target node, which aggregates these reports and keeps them in-additional local metric registries so that they can be accessed using `/admin/metrics` handler,-and re-reported elsewhere as necessary.--In case of shard reporter the target node is the shard leader, in case of cluster reporter the-target node is the Overseer leader.--==== Shard Reporter-This reporter uses predefined `shard` group, and the implementing class must be (a subclass of)-`solr.SolrShardReporter`. It publishes selected metrics from replicas to the node where shard leader is-located. Reports use a target registry name that is the replica's registry name with a `.leader` suffix, e.g., for a-SolrCore name `collection1_shard1_replica_n3` the target registry name is-`solr.core.collection1.shard1.replica_n3.leader`.--The following configuration properties are supported:--`handler`::-The handler path where reports are sent. Default is `/admin/metrics/collector`.--`period`::-How often reports are sent, in seconds. Default is `60`. Setting this to `0` disables the reporter.--`filter`::-An optional regular expression(s) matching selected metrics to be reported.-+-The following filter expressions are used by default:-+-[source,text]------TLOG.*-CORE\.fs.*-REPLICATION.*-INDEX\.flush.*-INDEX\.merge\.major.*-UPDATE\./update/.*requests-QUERY\./select.*requests-------Example configuration:-[source,xml]------ <reporter name="test" group="shard" class="solr.SolrShardReporter">- <int name="period">11</int>- <str name="filter">UPDATE\./update/.*requests</str>- <str name="filter">QUERY\./select.*requests</str>- </reporter>-------==== Cluster Reporter-This reporter uses predefined `cluster` group and the implementing class must be (a subclass of)-`solr.SolrClusterReporter`. It publishes selected metrics from any local registry to the Overseer leader node.--The following configuration properties are supported:--`handler`::-The handler path where reports are sent. Default is `/admin/metrics/collector`.--`period`::-How often reports are sent, in seconds. Default is `60`. Setting this to `0` disables the reporter.--`report`::-report configuration(s), see below.--Each report configuration consists of the following properties:--`registry`::-A regular expression pattern matching local source registries (see `SolrMetricManager.registryNames(String...)`), may contain regex capture groups (required).--`group`::-The target registry name where metrics will be grouped. This can be a regular expression pattern that contains back-references to capture groups collected by registry pattern (required).--`label`::-An optional prefix to prepend to metric names, may contain back-references to capture groups collected by registry pattern.--`filter`::-An optional regular expression(s) matching selected metrics to be reported.--The following report specifications are used by default (their result is a single additional metric registry in Overseer, called-`solr.cluster`):--[source,xml]------ <lst name="report">- <str name="group">cluster</str>- <str name="registry">solr\.jetty</str>- <str name="label">jetty</str>- </lst>- <lst name="report">- <str name="group">cluster</str>- <str name="registry">solr\.node</str>- <str name="label">node</str>- <str name="filter">CONTAINER\.cores\..*</str>- <str name="filter">CONTAINER\.fs\..*</str>- </lst>- <lst name="report">- <str name="group">cluster</str>- <str name="label">jvm</str>- <str name="registry">solr\.jvm</str>- <str name="filter">memory\.total\..*</str>- <str name="filter">memory\.heap\..*</str>- <str name="filter">os\.SystemLoadAverage</str>- <str name="filter">os\.FreePhysicalMemorySize</str>- <str name="filter">os\.FreeSwapSpaceSize</str>- <str name="filter">os\.OpenFileDescriptorCount</str>- <str name="filter">threads\.count</str>- </lst>- <lst name="report">- <str name="group">cluster</str>- <str name="registry">solr\.core\.(.*)\.leader</str>- <str name="label">leader.$1</str>- <str name="filter">QUERY\./select/.*</str>- <str name="filter">UPDATE\./update/.*</str>- <str name="filter">INDEX\..*</str>- <str name="filter">TLOG\..*</str>- </lst>--------Example configuration:-[source,xml]------ <reporter name="test" group="cluster" class="solr.SolrClusterReporter">- <str name="handler">/admin/metrics/collector</str>- <int name="period">11</int>- <lst name="report">- <str name="group">aggregated_jvms</str>- <str name="label">jvm</str>- <str name="registry">solr\.jvm</str>- <str name="filter">memory\.total\..*</str>- <str name="filter">memory\.heap\..*</str>- <str name="filter">os\.SystemLoadAverage</str>- <str name="filter">threads\.count</str>- </lst>- <lst name="report">- <str name="group">aggregated_shard_leaders</str>- <str name="registry">solr\.collection\.(.*)\.leader</str>- <str name="label">leader.$1</str>- <str name="filter">UPDATE\./update/.*</str>- </lst>- </reporter>------== Core Level MetricsThese metrics are available only on a per-core basis. Metrics can be aggregated across cores using Shard and Cluster reporters.diff --git a/solr/solr-ref-guide/src/monitoring-solr.adoc b/solr/solr-ref-guide/src/monitoring-solr.adocindex bee3cab1b87..957abf4a154 100644--- a/solr/solr-ref-guide/src/monitoring-solr.adoc+++ b/solr/solr-ref-guide/src/monitoring-solr.adoc@@ -1,5 +1,5 @@= Monitoring Solr-:page-children: metrics-reporting, metrics-history, mbean-request-handler, configuring-logging, using-jmx-with-solr, monitoring-solr-with-prometheus-and-grafana, performance-statistics-reference, solr-tracing+:page-children: metrics-reporting, mbean-request-handler, configuring-logging, using-jmx-with-solr, monitoring-solr-with-prometheus-and-grafana, performance-statistics-reference, solr-tracing// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -24,8 +24,6 @@ Common administrative tasks include:<<metrics-reporting.adoc#,Metrics Reporting>>: Details of Solr's metrics registries and Metrics API.-<<metrics-history.adoc#,Metrics History>>: Metrics history collection, configuration and API.-<<mbean-request-handler.adoc#,MBean Request Handler>>: How to use Solr's MBeans for programmatic access to the system plugins and stats.<<configuring-logging.adoc#,Configuring Logging>>: Describes how to configure logging for Solr.diff --git a/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc b/solr/solr-ref-guide/src/rule-based-authorization-plugin.adocindex bc2cb282238..3c82cfc79ed 100644--- a/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc+++ b/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc@@ -213,7 +213,6 @@ The predefined permission names (and their effects) are:* *config-edit*: this permission is allowed to edit a collection's configuration using the <<config-api.adoc#,Config API>>, the <<request-parameters-api.adoc#,Request Parameters API>>, and other APIs which modify `configoverlay.json`. Note that this allows configuration edit permissions for _all_ collections. If edit permissions should only be applied to specific collections, a custom permission would need to be created.* *config-read*: this permission is allowed to read a collection's configuration using the <<config-api.adoc#,Config API>>, the <<request-parameters-api.adoc#,Request Parameters API>>, and other APIs which modify `configoverlay.json`. Note that this allows configuration read permissions for _all_ collections. If read permissions should only be applied to specific collections, a custom permission would need to be created.* *metrics-read*: this permission allows access to Solr's <<metrics-reporting.adoc#metrics-api,Metrics API>>-* *metrics-history-read*: this permission allows access to Solr's <<metrics-history.adoc#metrics-history-api,Metrics History API>>, which provides long-term history for a select set of key Solr metrics.* *core-admin-edit*: Core admin commands that can mutate the system state.* *core-admin-read*: Read operations on the core admin API* *collection-admin-edit*: this permission is allowed to edit a collection's configuration using the <<collections-api.adoc#,Collections API>>. Note that this allows configuration edit permissions for _all_ collections. If edit permissions should only be applied to specific collections, a custom permission would need to be created. Specifically, the following actions of the Collections API would be allowed:diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.javaindex 41d1d6d8de2..2136c58ec74 100644--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java@@ -200,7 +200,6 @@ public interface CommonParams {String ZK_STATUS_PATH = "/admin/zookeeper/status";String SYSTEM_INFO_PATH = "/admin/info/system";String METRICS_PATH = "/admin/metrics";- String METRICS_HISTORY_PATH = "/admin/metrics/history";String STATUS = "status";@@ -217,8 +216,7 @@ public interface CommonParams {ZK_STATUS_PATH,AUTHC_PATH,AUTHZ_PATH,- METRICS_PATH,- METRICS_HISTORY_PATH);+ METRICS_PATH);String APISPEC_LOCATION = "apispec/";String INTROSPECT = "/_introspect";diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.javaindex 8acb63e6e0d..12e55002c43 100644--- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java+++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java@@ -103,8 +103,6 @@ public abstract class CoreAdminParamspublic static final String DELETE_INSTANCE_DIR = "deleteInstanceDir";- public static final String DELETE_METRICS_HISTORY = "deleteMetricsHistory";-public static final String LOAD_ON_STARTUP = "loadOnStartup";public static final String TRANSIENT = "transient";diff --git a/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/TestHarness.javaindex 2f18f26245d..b4c4a31d5cc 100644--- a/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java+++ b/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java@@ -22,7 +22,6 @@ import java.io.StringWriter;import java.nio.charset.StandardCharsets;import java.nio.file.Path;import java.nio.file.Paths;-import java.util.Collections;import java.util.HashMap;import java.util.List;import java.util.Map;@@ -207,9 +206,6 @@ public class TestHarness extends BaseTestHarness {PluginInfo defaultPlugin = new PluginInfo("reporter", attributes);MetricsConfig metricsConfig = new MetricsConfig.MetricsConfigBuilder().setMetricReporterPlugins(new PluginInfo[]{defaultPlugin})- .setHistoryHandler(- Boolean.getBoolean("metricsHistory")- ? null : new PluginInfo("typeUnused", Collections.singletonMap("enable", "false"))).build();return new NodeConfig.NodeConfigBuilder("testNode", solrHome)diff --git a/versions.lock b/versions.lockindex b0919f8c49c..8e591959652 100644--- a/versions.lock+++ b/versions.lock@@ -220,7 +220,6 @@ org.jdom:jdom2:2.0.6 (1 constraints: 0a05fb35)org.locationtech.spatial4j:spatial4j:0.7 (1 constraints: 58105398)org.ow2.asm:asm:7.2 (2 constraints: 4e19d5db)org.ow2.asm:asm-commons:7.2 (1 constraints: 6b0f7267)-org.rrd4j:rrd4j:3.5 (1 constraints: ac04252c)org.slf4j:jcl-over-slf4j:1.7.24 (1 constraints: 4005473b)org.slf4j:slf4j-api:1.7.24 (14 constraints: c8af2e3b)org.tallison:jmatio:1.5 (1 constraints: aa041f2c)diff --git a/versions.props b/versions.propsindex 199f6b2aac1..bb2e9b439ee 100644--- a/versions.props+++ b/versions.props@@ -116,7 +116,6 @@ org.locationtech.spatial4j:*=0.7org.mockito:mockito-core=3.8.0org.objenesis:objenesis=2.6org.ow2.asm:*=7.2-org.rrd4j:rrd4j=3.5org.slf4j:*=1.7.24org.tallison:jmatio=1.5org.threeten:threetenbp=1.3.3