Apache Solr open-source search software
SOLR-14915: Prometheus-exporter should not depend on Solr-core (#1972)
David Smiley021de9f45f56
diff --git a/gradle/solr/packaging.gradle b/gradle/solr/packaging.gradleindex 3b1ea92d7ed..39b4bc68ebc 100644--- a/gradle/solr/packaging.gradle+++ b/gradle/solr/packaging.gradle@@ -76,7 +76,8 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr:contribreturn true}}- return externalLibs - configurations.solrPlatformLibs+ // libExt has logging libs, which we don't want. Lets users decide what they want.+ return externalLibs - configurations.solrPlatformLibs - project(':solr:server').configurations.getByName('libExt')}, {exclude "lucene-*"into "lib"diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex f1eb0c07c70..a379b1e5d11 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -42,8 +42,6 @@ Improvements* SOLR-14880: Support coreRootDirectory setting when create new cores from command line, in standalone mode (Alexandre Rafalovitch)-* SOLR-14972: Change default port of prometheus exporter to 8989 because it clashed with default embedded zookeeper port (janhoy)-* SOLR-14926, SOLR-14926, SOLR-13506: Modernize and clean up search results clustering contrib. This issue upgradesthe clustering contrib to the new Carrot2 4.x line, dropping several CVE-prone dependencies along the way.The parameters and configuration of the contrib extensions have changed. The documentation in Solr ref guidediff --git a/solr/contrib/prometheus-exporter/CHANGES.md b/solr/contrib/prometheus-exporter/CHANGES.mdnew file mode 100644index 00000000000..42deac6baec--- /dev/null+++ b/solr/contrib/prometheus-exporter/CHANGES.md@@ -0,0 +1,20 @@+This file lists release notes for this module.+Prior to version 9, changes were in Solr's CHANGES.txt++9.0.0+======================++Improvements+----------------------+* SOLR-14972: Change default port of prometheus exporter to 8989+ because it clashed with default embedded zookeeper port (janhoy)++Other Changes+----------------------+* SOLR-14915: Reduced dependencies from Solr server down to just SolrJ. Don't add WEB-INF/lib.+ * Can run via gradle, "gradlew run"+ * Has own log4j2.xml now+ * Was missing some dependencies in lib/; now has all except SolrJ & logging.+ (David Smiley, Houston Putman)++* SOLR-14957: Add Prometheus Exporter to docker PATH. Fix classpath issues. (Houston Putman)diff --git a/solr/contrib/prometheus-exporter/bin/solr-exporter b/solr/contrib/prometheus-exporter/bin/solr-exporterindex b601a785117..e53c46149db 100755--- a/solr/contrib/prometheus-exporter/bin/solr-exporter+++ b/solr/contrib/prometheus-exporter/bin/solr-exporter@@ -82,10 +82,6 @@ for JAR in $(find "$BASEDIR"/../../dist/solrj-lib -name '*.jar')doCLASSPATH="$CLASSPATH":"$JAR"done-for JAR in $(find "$BASEDIR"/../../dist -name 'solr-core-*.jar')-do- CLASSPATH="$CLASSPATH":"$JAR"-donefor JAR in $(find "$BASEDIR"/../../dist -name 'solr-solrj-*.jar')doCLASSPATH="$CLASSPATH":"$JAR"@@ -94,14 +90,6 @@ for JAR in $(find "$BASEDIR"/../../dist -name 'solr-prometheus-exporter-*.jar')doCLASSPATH="$CLASSPATH":"$JAR"done-for JAR in $(find "$BASEDIR"/lucene-libs -name '*.jar')-do- CLASSPATH="$CLASSPATH":"$JAR"-done-for JAR in $(find "$BASEDIR"/../../server/solr-webapp/webapp/WEB-INF/lib -name '*.jar')-do- CLASSPATH="$CLASSPATH":"$JAR"-donefor JAR in $(find "$BASEDIR"/../../server/lib/ext -name '*.jar')doCLASSPATH="$CLASSPATH":"$JAR"@@ -123,8 +111,6 @@ elseGC_TUNE="$GC_TUNE"fi-EXTRA_JVM_ARGUMENTS="-Dlog4j.configurationFile=file:"$BASEDIR"/../../server/resources/log4j2-console.xml"-# For Cygwin, switch paths to Windows format before running javaif $cygwin; then[ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --windows "$CLASSPATH"`@@ -163,7 +149,6 @@ exec "$JAVACMD" \$JAVA_MEM_OPTS \$GC_TUNE \$JAVA_OPTS \- $EXTRA_JVM_ARGUMENTS \$ZK_CREDS_AND_ACLS \-classpath "$CLASSPATH" \-Dapp.name="solr-exporter" \diff --git a/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd b/solr/contrib/prometheus-exporter/bin/solr-exporter.cmdindex efa92a1608e..113306a84ee 100644--- a/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd+++ b/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd@@ -71,8 +71,7 @@ if "%JAVACMD%"=="" set JAVACMD=javaif "%REPO%"=="" set REPO=%BASEDIR%\lib-set CLASSPATH=%REPO%\*;%BASEDIR%\conf;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\*;%BASEDIR%\lucene-libs\*;%BASEDIR%\..\..\server\solr-webapp\webapp\WEB-INF\lib\*;%BASEDIR%\..\..\server\lib\ext\*-set EXTRA_JVM_ARGUMENTS=-Dlog4j.configurationFile=file:///%BASEDIR%\..\..\server\resources\log4j2-console.xml+set CLASSPATH=%REPO%\*;%BASEDIR%\conf;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\*;%BASEDIR%\..\..\server\lib\ext\*@REM Convert Environment Variables to Command Line Optionsset EXPORTER_ARGS=@@ -88,7 +87,7 @@ goto endInit@REM Reaching here means variables are defined and arguments have been captured:endInit-%JAVACMD% %JAVA_MEM% %GC_TUNE% %JAVA_OPTS% %EXTRA_JVM_ARGUMENTS% %ZK_CREDS_AND_ACLS% -classpath "%CLASSPATH_PREFIX%;%CLASSPATH%" -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" org.apache.solr.prometheus.exporter.SolrExporter %EXPORTER_ARGS% %CMD_LINE_ARGS%+%JAVACMD% %JAVA_MEM% %GC_TUNE% %JAVA_OPTS% %ZK_CREDS_AND_ACLS% -classpath "%CLASSPATH_PREFIX%;%CLASSPATH%" -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" org.apache.solr.prometheus.exporter.SolrExporter %EXPORTER_ARGS% %CMD_LINE_ARGS%if ERRORLEVEL 1 goto errorgoto enddiff --git a/solr/contrib/prometheus-exporter/build.gradle b/solr/contrib/prometheus-exporter/build.gradleindex 13a9748bc44..ca4f625d073 100644--- a/solr/contrib/prometheus-exporter/build.gradle+++ b/solr/contrib/prometheus-exporter/build.gradle@@ -15,14 +15,13 @@* limitations under the License.*/-apply plugin: 'java-library'+// this is actually more of an 'application' but we don't want all of what Gradle addsdescription = 'Prometheus exporter for exposing metrics from Solr using Metrics API and Search API'dependencies {- implementation project(':solr:core')- implementation project(':lucene:analysis:common')+ implementation project(':solr:solrj')implementation ('io.prometheus:simpleclient')implementation ('io.prometheus:simpleclient_common')@@ -31,17 +30,57 @@ dependencies {exclude group: "org.jruby.joni", module: "joni"})implementation ('net.sourceforge.argparse4j:argparse4j')+ implementation ('com.github.ben-manes.caffeine:caffeine', {+ exclude group: "org.checkerframework", module: "checker-qual"+ exclude group: "com.google.errorprone", module: "error_prone_annotations"+ })- testImplementation ('org.apache.httpcomponents:httpcore')- testImplementation ('org.eclipse.jetty:jetty-servlet')+ runtimeOnly 'org.apache.logging.log4j:log4j-api'+ runtimeOnly 'org.apache.logging.log4j:log4j-core'+ runtimeOnly 'org.apache.logging.log4j:log4j-slf4j-impl'+ runtimeOnly 'com.lmax:disruptor'testImplementation project(':solr:test-framework')}-// Add two folders to default packaging.+ext {+ mainClass = 'org.apache.solr.prometheus.exporter.SolrExporter'+}++task run(type: JavaExec) {+ group = 'application'+ description = 'Run the main class with JavaExecTask'+ main = project.ext.mainClass+ classpath = sourceSets.main.runtimeClasspath+ systemProperties = ["log4j.configurationFile":"file:conf/log4j2.xml"]+}++jar {+ manifest {+ attributes('Main-Class': project.ext.mainClass)+ }+}+assemblePackaging {+ // Add two folders to default packaging.from(projectDir, {include "bin/**"include "conf/**"})+ // Add all libs except those provided by SolrJ & Logging+ from ({+ def externalLibs = configurations.runtimeLibs.copyRecursive { dep ->+ if (dep instanceof org.gradle.api.artifacts.ProjectDependency) {+ return !dep.dependencyProject.path.startsWith(":solr")+ } else {+ return true+ }+ }+ return externalLibs - project(':solr:server').configurations.getByName('libExt')+ }, {+ into "lib"+ })+++ into deps}\ No newline at end of filediff --git a/solr/contrib/prometheus-exporter/conf/log4j2.xml b/solr/contrib/prometheus-exporter/conf/log4j2.xmlnew file mode 100644index 00000000000..47f533f1054--- /dev/null+++ b/solr/contrib/prometheus-exporter/conf/log4j2.xml@@ -0,0 +1,40 @@+<?xml version="1.0" encoding="UTF-8"?>+<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+ -->++<!-- Use this file for logging exlusively to the console, useful for+ some development tasks. Should not be used for production -->+<!-- Default production configuration is asnychronous logging -->+<Configuration>+ <Appenders>+ <Console name="STDERR" target="SYSTEM_ERR">+ <PatternLayout>+ <Pattern>+ %maxLen{%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %c; %m%notEmpty{ =>%ex{short}}}{10240}%n+ </Pattern>+ </PatternLayout>+ </Console>+ </Appenders>+ <Loggers>+ <!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->+ <AsyncRoot level="INFO">+ <AppenderRef ref="STDERR"/>+ </AsyncRoot>+ </Loggers>+</Configuration>++diff --git a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsConfiguration.java b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsConfiguration.javaindex f976f3e564d..a4e630dbcbf 100644--- a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsConfiguration.java+++ b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsConfiguration.java@@ -17,14 +17,27 @@package org.apache.solr.prometheus.exporter;+import javax.xml.parsers.DocumentBuilderFactory;+import javax.xml.xpath.XPath;+import javax.xml.xpath.XPathConstants;+import javax.xml.xpath.XPathFactory;+import java.io.File;+import java.io.InputStream;+import java.lang.invoke.MethodHandles;+import java.nio.file.Files;+import java.nio.file.Path;import java.util.Collections;import java.util.List;import net.thisptr.jackson.jq.exception.JsonQueryException;-import org.apache.solr.core.XmlConfigFile;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;+import org.w3c.dom.Document;import org.w3c.dom.Node;+import org.w3c.dom.NodeList;public class MetricsConfiguration {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());private final PrometheusExporterSettings settings;@@ -66,13 +79,36 @@ public class MetricsConfiguration {return searchConfiguration;}- public static MetricsConfiguration from(XmlConfigFile config) throws Exception {- Node settings = config.getNode("/config/settings", false);+ public static MetricsConfiguration from(String resource) throws Exception {+ // See solr-core XmlConfigFile+ final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();+ try {+ dbf.setXIncludeAware(true);+ dbf.setNamespaceAware(true);+ } catch (UnsupportedOperationException e) {+ log.warn("{} XML parser doesn't support XInclude option", resource);+ }++ Document document;+ Path path = Path.of(resource);+ if (Files.exists(path)) {+ document = dbf.newDocumentBuilder().parse(path.toUri().toASCIIString());+ } else {+ try (InputStream configInputStream = MethodHandles.lookup().lookupClass().getClassLoader().getResourceAsStream(resource.replace(File.separatorChar, '/'))) {+ document = dbf.newDocumentBuilder().parse(configInputStream);+ }+ }++ return from(document);+ }- Node pingConfig = config.getNode("/config/rules/ping", false);- Node metricsConfig = config.getNode("/config/rules/metrics", false);- Node collectionsConfig = config.getNode("/config/rules/collections", false);- Node searchConfiguration = config.getNode("/config/rules/search", false);+ public static MetricsConfiguration from(Document config) throws Exception {+ Node settings = getNode(config, "/config/settings");++ Node pingConfig = getNode(config, "/config/rules/ping");+ Node metricsConfig = getNode(config, "/config/rules/metrics");+ Node collectionsConfig = getNode(config, "/config/rules/collections");+ Node searchConfiguration = getNode(config, "/config/rules/search");return new MetricsConfiguration(settings == null ? PrometheusExporterSettings.builder().build() : PrometheusExporterSettings.from(settings),@@ -83,6 +119,28 @@ public class MetricsConfiguration {);}+ static final XPathFactory xpathFactory = XPathFactory.newInstance();++ private static Node getNode(Document doc, String path) {+ // Copied from solr-core XmlConfigFile.getNode with simplifications+ XPath xpath = xpathFactory.newXPath();+ String xstr = path; //normalize(path);++ try {+ NodeList nodes = (NodeList) xpath.evaluate(xstr, doc,+ XPathConstants.NODESET);+ if (nodes == null || 0 == nodes.getLength()) {+ return null;+ }+ if (1 < nodes.getLength()) {+ throw new RuntimeException("more than one value");+ }+ return nodes.item(0);+ } catch (Exception e) {+ throw new RuntimeException("Error in xpath:" + xstr, e);+ }+ }+private static List<MetricsQuery> toMetricQueries(Node node) throws JsonQueryException {if (node == null) {return Collections.emptyList();diff --git a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.javaindex d3da263fb77..b4b9f565201 100644--- a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java+++ b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java@@ -19,8 +19,6 @@ package org.apache.solr.prometheus.exporter;import java.io.IOException;import java.lang.invoke.MethodHandles;import java.net.InetSocketAddress;-import java.nio.file.Path;-import java.nio.file.Paths;import java.util.Locale;import java.util.concurrent.ExecutorService;@@ -32,8 +30,6 @@ import net.sourceforge.argparse4j.inf.ArgumentParserException;import net.sourceforge.argparse4j.inf.Namespace;import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.common.util.IOUtils;-import org.apache.solr.core.SolrResourceLoader;-import org.apache.solr.core.XmlConfigFile;import org.apache.solr.prometheus.collector.MetricsCollectorFactory;import org.apache.solr.prometheus.collector.SchedulerMetricsCollector;import org.apache.solr.prometheus.scraper.SolrCloudScraper;@@ -202,7 +198,7 @@ public class SolrExporter {res.getInt(ARG_NUM_THREADS_DEST),res.getInt(ARG_SCRAPE_INTERVAL_DEST),scrapeConfiguration,- loadMetricsConfiguration(Paths.get(res.getString(ARG_CONFIG_DEST))));+ loadMetricsConfiguration(res.getString(ARG_CONFIG_DEST)));log.info("Starting Solr Prometheus Exporting");solrExporter.start();@@ -214,12 +210,11 @@ public class SolrExporter {}}- private static MetricsConfiguration loadMetricsConfiguration(Path configPath) {- try (SolrResourceLoader loader = new SolrResourceLoader(configPath.getParent())) {- XmlConfigFile config = new XmlConfigFile(loader, configPath.getFileName().toString(), null, null);- return MetricsConfiguration.from(config);+ private static MetricsConfiguration loadMetricsConfiguration(String configPath) {+ try {+ return MetricsConfiguration.from(configPath);} catch (Exception e) {- log.error("Could not load scrape configuration from {}", configPath.toAbsolutePath());+ log.error("Could not load scrape configuration from {}", configPath);throw new RuntimeException(e);}}diff --git a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.javaindex e4b98e75811..5efedc36af7 100644--- a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java+++ b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java@@ -20,13 +20,12 @@ import java.io.IOException;import java.util.List;import java.util.Map;import java.util.Set;-import java.util.concurrent.ExecutionException;import java.util.concurrent.ExecutorService;import java.util.function.Function;import java.util.stream.Collectors;-import com.google.common.cache.Cache;-import com.google.common.cache.CacheBuilder;+import com.github.benmanes.caffeine.cache.Cache;+import com.github.benmanes.caffeine.cache.Caffeine;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.impl.HttpSolrClient;import org.apache.solr.common.cloud.DocCollection;@@ -42,7 +41,7 @@ public class SolrCloudScraper extends SolrScraper {private final CloudSolrClient solrClient;private final SolrClientFactory solrClientFactory;- private Cache<String, HttpSolrClient> hostClientCache = CacheBuilder.newBuilder().build();+ private Cache<String, HttpSolrClient> hostClientCache = Caffeine.newBuilder().build();public SolrCloudScraper(CloudSolrClient solrClient, ExecutorService executor, SolrClientFactory solrClientFactory) {super(executor);@@ -83,15 +82,8 @@ public class SolrCloudScraper extends SolrScraper {private Map<String, HttpSolrClient> createHttpSolrClients() throws IOException {return getBaseUrls().stream()- .map(url -> {- try {- return hostClientCache.get(url, () -> solrClientFactory.createStandaloneSolrClient(url));- } catch (ExecutionException e) {- throw new RuntimeException(e);- }- })+ .map(url -> hostClientCache.get(url, solrClientFactory::createStandaloneSolrClient)).collect(Collectors.toMap(HttpSolrClient::getBaseURL, Function.identity()));-}@Overridediff --git a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.javaindex 5dcfb715335..6870cb2aacc 100644--- a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java+++ b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java@@ -19,28 +19,19 @@ package org.apache.solr.prometheus.utils;import java.io.File;import java.io.IOException;-import java.nio.file.Path;-import java.nio.file.Paths;import java.util.Objects;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;-import org.apache.solr.core.SolrResourceLoader;-import org.apache.solr.core.XmlConfigFile;import org.apache.solr.prometheus.PrometheusExporterTestBase;import org.apache.solr.prometheus.exporter.MetricsConfiguration;public class Helpers {- public static MetricsConfiguration loadConfiguration(String path) throws Exception {- Path configPath = Paths.get(path);-- try (SolrResourceLoader loader = new SolrResourceLoader(configPath.getParent())) {- XmlConfigFile config = new XmlConfigFile(loader, configPath.getFileName().toString());- return MetricsConfiguration.from(config);- }+ public static MetricsConfiguration loadConfiguration(String pathRsrc) throws Exception {+ return MetricsConfiguration.from(SolrTestCaseJ4.getFile(pathRsrc).getPath());}public static void indexAllDocs(SolrClient client) throws IOException, SolrServerException {