Apache Solr open-source search software
SOLR-17806: Migrate suggest component and cross-dc metrics to OTEL (#3668)
Matthew Biscochobed6205d9b4a
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.javaindex ea2b9a08e8f..2d31429eab4 100644--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java@@ -21,7 +21,7 @@ import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT;import static org.apache.solr.handler.admin.MetricsHandler.PROMETHEUS_METRICS_WT;import static org.apache.solr.metrics.SolrCoreMetricManager.COLLECTION_ATTR;import static org.apache.solr.metrics.SolrCoreMetricManager.CORE_ATTR;-import static org.apache.solr.metrics.SolrCoreMetricManager.REPLICA_ATTR;+import static org.apache.solr.metrics.SolrCoreMetricManager.REPLICA_TYPE_ATTR;import static org.apache.solr.metrics.SolrCoreMetricManager.SHARD_ATTR;import com.github.benmanes.caffeine.cache.Cache;@@ -572,8 +572,8 @@ public class SolrCore implements SolrInfoBean, Closeable {.put(CORE_ATTR, getName()).put(SHARD_ATTR, coreDescriptor.getCloudDescriptor().getShardId()).put(- REPLICA_ATTR,- Utils.parseMetricsReplicaName(coreDescriptor.getCollectionName(), getName()))+ REPLICA_TYPE_ATTR,+ coreDescriptor.getCloudDescriptor().getReplicaType().toString()).build(): Attributes.builder().put(CORE_ATTR, getName()).build();}diff --git a/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.javaindex f32934451ac..2702f79e58e 100644--- a/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java+++ b/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java@@ -43,13 +43,14 @@ import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.ShardParams;import org.apache.solr.common.params.SolrParams;+import org.apache.solr.common.util.IOUtils;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrEventListener;-import org.apache.solr.metrics.MetricsMap;import org.apache.solr.metrics.SolrMetricProducer;import org.apache.solr.metrics.SolrMetricsContext;+import org.apache.solr.metrics.otel.OtelUnit;import org.apache.solr.search.QueryLimits;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.spelling.suggest.SolrSuggester;@@ -95,6 +96,8 @@ public class SuggestComponent extends SearchComponent*/protected Map<String, SolrSuggester> suggesters = new ConcurrentHashMap<>();+ private AutoCloseable toClose;+/** Container for various labels used in the responses generated by this component */private static class SuggesterResultLabels {static final String SUGGEST = "suggest";@@ -382,24 +385,20 @@ public class SuggestComponent extends SearchComponentreturn "Suggester component";}- // TODO SOLR-17458: Migrate to OTEL@Overridepublic void initializeMetrics(SolrMetricsContext parentContext, Attributes attributes, String scope) {super.initializeMetrics(parentContext, attributes, scope);-- this.solrMetricsContext.gauge(- () -> ramBytesUsed(), true, "totalSizeInBytes", getCategory().toString());- MetricsMap suggestersMap =- new MetricsMap(- map -> {- for (Map.Entry<String, SolrSuggester> entry : suggesters.entrySet()) {- SolrSuggester suggester = entry.getValue();- map.putNoEx(entry.getKey(), suggester.toString());- }- });- this.solrMetricsContext.gauge(- suggestersMap, true, "suggesters", getCategory().toString(), scope);+ var suggesterAttributes =+ attributes.toBuilder().put(CATEGORY_ATTR, getCategory().toString()).build();+ this.toClose =+ this.solrMetricsContext.observableLongGauge(+ "solr_core_suggester_total_size",+ "Total memory size in bytes of all suggester",+ (observableLongMeasurement) -> {+ observableLongMeasurement.record(ramBytesUsed(), suggesterAttributes);+ },+ OtelUnit.BYTES);}@Override@@ -520,6 +519,12 @@ public class SuggestComponent extends SearchComponentreturn result;}+ @Override+ public void close() throws IOException {+ IOUtils.closeQuietly(toClose);+ super.close();+ }+/** Listener to build or reload the maintained {@link SolrSuggester} by this component */private static class SuggesterListener implements SolrEventListener {private final SolrCore core;diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.javaindex aa64dcbacb1..99dec259ff6 100644--- a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java+++ b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java@@ -41,7 +41,8 @@ public class SolrCoreMetricManager implements Closeable {public static final AttributeKey<String> COLLECTION_ATTR = AttributeKey.stringKey("collection");public static final AttributeKey<String> CORE_ATTR = AttributeKey.stringKey("core");public static final AttributeKey<String> SHARD_ATTR = AttributeKey.stringKey("shard");- public static final AttributeKey<String> REPLICA_ATTR = AttributeKey.stringKey("replica");+ public static final AttributeKey<String> REPLICA_TYPE_ATTR =+ AttributeKey.stringKey("replica_type");private final SolrCore core;private SolrMetricsContext solrMetricsContext;diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.javaindex 51dd2c83930..14e44b4fcc0 100644--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java@@ -16,10 +16,6 @@*/package org.apache.solr.search;-import static org.apache.solr.metrics.SolrCoreMetricManager.COLLECTION_ATTR;-import static org.apache.solr.metrics.SolrCoreMetricManager.CORE_ATTR;-import static org.apache.solr.metrics.SolrCoreMetricManager.REPLICA_ATTR;-import static org.apache.solr.metrics.SolrCoreMetricManager.SHARD_ATTR;import static org.apache.solr.search.CpuAllowedLimit.TIMING_CONTEXT;import com.codahale.metrics.Gauge;@@ -102,7 +98,6 @@ import org.apache.solr.common.util.ExecutorUtil.MDCAwareThreadPoolExecutor;import org.apache.solr.common.util.IOUtils;import org.apache.solr.common.util.ObjectReleaseTracker;import org.apache.solr.common.util.SolrNamedThreadFactory;-import org.apache.solr.common.util.Utils;import org.apache.solr.core.DirectoryFactory;import org.apache.solr.core.DirectoryFactory.DirContext;import org.apache.solr.core.NodeConfig;@@ -621,23 +616,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrIcore.getCoreAttributes().toBuilder().put(NAME_ATTR, cache.name()).build(),"solr_searcher_cache");}- Attributes baseAttributes;- if (core.getCoreContainer().isZooKeeperAware()) {- baseAttributes =- Attributes.builder()- .put(COLLECTION_ATTR, core.getCoreDescriptor().getCollectionName())- .put(CORE_ATTR, core.getCoreDescriptor().getName())- .put(SHARD_ATTR, core.getCoreDescriptor().getCloudDescriptor().getShardId())- .put(- REPLICA_ATTR,- Utils.parseMetricsReplicaName(- core.getCoreDescriptor().getCollectionName(), core.getName()))- .build();- } else {- baseAttributes =- Attributes.builder().put(CORE_ATTR, core.getCoreDescriptor().getName()).build();- }- initializeMetrics(solrMetricsContext, baseAttributes, STATISTICS_KEY);+ initializeMetrics(solrMetricsContext, core.getCoreAttributes(), STATISTICS_KEY);registerTime = new Date();}diff --git a/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.javaindex 607715f9aa4..a42e4bfb7e0 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java@@ -245,9 +245,12 @@ public abstract class TestBaseStatsCacheCloud extends SolrCloudTestCase {* "solr_searcher_stats_cache{...type="lookups",...}" -> "lookups"*/private String extractTypeAttribute(String line) {- int typeStart = line.indexOf("type=\"");- int typeEnd = line.indexOf('"', typeStart += 6);- return line.substring(typeStart, typeEnd);+ java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\\btype=\"([^\"]+)\"");+ java.util.regex.Matcher matcher = pattern.matcher(line);+ if (matcher.find()) {+ return matcher.group(1);+ }+ throw new IllegalArgumentException("No type attribute found in line: " + line);}/**diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.javaindex 003745fe383..47c8d50791f 100644--- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java+++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java@@ -87,7 +87,7 @@ public class TestPrometheusResponseWriterCloud extends SolrCloudTestCase {&& line.contains("handler=\"/select\"")&& line.contains("collection=\"collection1\"")&& line.contains("core=\"collection1_shard1_replica_n1\"")- && line.contains("replica=\"replica_n1\"")+ && line.contains("replica_type=\"NRT\"")&& line.contains("shard=\"shard1\"")));}}diff --git a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessor.java b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessor.javaindex 3b391f83b83..346bf19d81b 100644--- a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessor.java+++ b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessor.java@@ -143,7 +143,7 @@ public class MirroringUpdateProcessor extends UpdateRequestProcessor {estimatedDocSizeInBytes,maxMirroringDocSizeBytes);}- producerMetrics.getDocumentSize().update(estimatedDocSizeInBytes);+ producerMetrics.getDocumentSize().record(estimatedDocSizeInBytes);final boolean tooLargeForKafka = estimatedDocSizeInBytes > maxMirroringDocSizeBytes;if (tooLargeForKafka && !indexUnmirrorableDocs) {throw new SolrException(diff --git a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/update/processor/ProducerMetrics.java b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/update/processor/ProducerMetrics.javaindex 1efdb7ded95..65fd30668a8 100644--- a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/update/processor/ProducerMetrics.java+++ b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/update/processor/ProducerMetrics.java@@ -16,52 +16,73 @@*/package org.apache.solr.crossdc.update.processor;-import com.codahale.metrics.Counter;-import com.codahale.metrics.Histogram;+import static org.apache.solr.metrics.SolrMetricProducer.TYPE_ATTR;+import org.apache.solr.core.SolrCore;import org.apache.solr.metrics.SolrMetricsContext;+import org.apache.solr.metrics.otel.OtelUnit;+import org.apache.solr.metrics.otel.instruments.AttributedLongCounter;+import org.apache.solr.metrics.otel.instruments.AttributedLongHistogram;/** Metrics presented for each SolrCore using `crossdc.producer.` path. */public class ProducerMetrics {- private final Counter local;- private final Counter localError;- private final Counter submitted;- private final Counter submitError;- private final Histogram documentSize;- private final Counter documentTooLarge;+ private final AttributedLongCounter local;+ private final AttributedLongCounter localError;+ private final AttributedLongCounter submitted;+ private final AttributedLongCounter submitError;+ private final AttributedLongHistogram documentSize;+ private final AttributedLongCounter documentTooLarge;public ProducerMetrics(SolrMetricsContext solrMetricsContext, SolrCore solrCore) {+ var attributes = solrCore.getCoreAttributes();++ var localProcessed =+ solrMetricsContext.longCounter(+ "solr_core_crossdc_producer_local_processed",+ "The number of local documents processed (success or error)");+ var localSubmitted =+ solrMetricsContext.longCounter(+ "solr_core_crossdc_producer_submitted",+ "The number of documents submitted to the Kafka topic (success or error)");+ var histogramDocSizes =+ solrMetricsContext.longHistogram(+ "solr_core_crossdc_producer_document_size",+ "Histogram of the processed document sizes processed",+ OtelUnit.BYTES);+ var tooLargeErrors =+ solrMetricsContext.longCounter(+ "solr_core_crossdc_producer_doc_too_large_errors",+ "The number of documents that were too large to send to the Kafka topic");+this.local =- solrMetricsContext.counter(String.valueOf(solrCore), "local", "crossdc", "producer");+ new AttributedLongCounter(+ localProcessed, attributes.toBuilder().put(TYPE_ATTR, "success").build());this.localError =- solrMetricsContext.counter(- String.valueOf(solrCore), "local", "crossdc", "producer", "errors");+ new AttributedLongCounter(+ localProcessed, attributes.toBuilder().put(TYPE_ATTR, "error").build());this.submitted =- solrMetricsContext.counter(String.valueOf(solrCore), "submitted", "crossdc", "producer");+ new AttributedLongCounter(+ localSubmitted, attributes.toBuilder().put(TYPE_ATTR, "success").build());this.submitError =- solrMetricsContext.counter(- String.valueOf(solrCore), "submit", "crossdc", "producer", "errors");- this.documentSize =- solrMetricsContext.histogram(- String.valueOf(solrCore), "documentSize", "crossdc", "producer");- this.documentTooLarge =- solrMetricsContext.counter(- String.valueOf(solrCore), "documentTooLarge", "crossdc", "producer", "errors");+ new AttributedLongCounter(+ localSubmitted, attributes.toBuilder().put(TYPE_ATTR, "error").build());+ this.documentSize = new AttributedLongHistogram(histogramDocSizes, attributes);+ this.documentTooLarge = new AttributedLongCounter(tooLargeErrors, attributes);}/** Counter representing the number of local documents processed successfully. */- public Counter getLocal() {+ public AttributedLongCounter getLocal() {return this.local;}/** Counter representing the number of local documents processed with error. */- public Counter getLocalError() {+ public AttributedLongCounter getLocalError() {return this.localError;}/** Counter representing the number of documents submitted to the Kafka topic. */- public Counter getSubmitted() {+ public AttributedLongCounter getSubmitted() {return this.submitted;}@@ -69,19 +90,19 @@ public class ProducerMetrics {* Counter representing the number of documents that were not submitted to the Kafka topic because* of exception during execution.*/- public Counter getSubmitError() {+ public AttributedLongCounter getSubmitError() {return this.submitError;}/** Histogram of the processed document size. */- public Histogram getDocumentSize() {+ public AttributedLongHistogram getDocumentSize() {return this.documentSize;}/*** Counter representing the number of documents that were too large to send to the Kafka topic.*/- public Counter getDocumentTooLarge() {+ public AttributedLongCounter getDocumentTooLarge() {return this.documentTooLarge;}}diff --git a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessorTest.java b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessorTest.javaindex a24f8728717..1c30f956dbb 100644--- a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessorTest.java+++ b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessorTest.java@@ -23,8 +23,7 @@ import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;-import com.codahale.metrics.Counter;-import com.codahale.metrics.Histogram;+import io.opentelemetry.api.common.Attributes;import java.io.IOException;import java.util.Map;import org.apache.solr.SolrTestCaseJ4;@@ -48,6 +47,8 @@ import org.apache.solr.core.CoreDescriptor;import org.apache.solr.core.SolrCore;import org.apache.solr.crossdc.common.CrossDcConf;import org.apache.solr.metrics.SolrMetricsContext;+import org.apache.solr.metrics.otel.instruments.AttributedLongCounter;+import org.apache.solr.metrics.otel.instruments.AttributedLongHistogram;import org.apache.solr.request.SolrQueryRequestBase;import org.apache.solr.schema.IndexSchema;import org.apache.solr.update.AddUpdateCommand;@@ -113,39 +114,42 @@ public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {commitUpdateCommand.openSearcher = true;commitUpdateCommand.waitSearcher = true;+ core = mock(SolrCore.class);+ when(core.getCoreAttributes()).thenReturn(Attributes.empty());+producerMetrics =spy(- new ProducerMetrics(mock(SolrMetricsContext.class), mock(SolrCore.class)) {- private final Counter counterMock = mock(Counter.class);+ new ProducerMetrics(mock(SolrMetricsContext.class), core) {+ private final AttributedLongCounter counterMock = mock(AttributedLongCounter.class);@Override- public Counter getLocal() {+ public AttributedLongCounter getLocal() {return counterMock;}@Override- public Counter getLocalError() {+ public AttributedLongCounter getLocalError() {return counterMock;}@Override- public Counter getSubmitted() {+ public AttributedLongCounter getSubmitted() {return counterMock;}@Override- public Counter getDocumentTooLarge() {+ public AttributedLongCounter getDocumentTooLarge() {return counterMock;}@Override- public Counter getSubmitError() {+ public AttributedLongCounter getSubmitError() {return counterMock;}@Override- public Histogram getDocumentSize() {- return mock(Histogram.class);+ public AttributedLongHistogram getDocumentSize() {+ return mock(AttributedLongHistogram.class);}});@@ -168,8 +172,6 @@ public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {return requestMock;}};-- core = mock(SolrCore.class);CoreDescriptor coreDesc = mock(CoreDescriptor.class);cloudDesc = mock(CloudDescriptor.class);when(cloudDesc.getShardId()).thenReturn("shard1");diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientRetryTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientRetryTest.javaindex 7b5a9b8a0f5..07d829b2718 100644--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientRetryTest.java+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientRetryTest.java@@ -49,7 +49,7 @@ public class CloudHttp2SolrClientRetryTest extends SolrCloudTestCase {public void testRetry() throws Exception {String collectionName = "testRetry";String prometheusMetric =- "solr_core_requests_total{category=\"UPDATE\",collection=\"testRetry\",core=\"testRetry_shard1_replica_n1\",handler=\"/update\",otel_scope_name=\"org.apache.solr\",replica=\"replica_n1\",shard=\"shard1\"}";+ "solr_core_requests_total{category=\"UPDATE\",collection=\"testRetry\",core=\"testRetry_shard1_replica_n1\",handler=\"/update\",otel_scope_name=\"org.apache.solr\",replica_type=\"NRT\",shard=\"shard1\"}";try (CloudSolrClient solrClient =new CloudHttp2SolrClient.Builder(Collections.singletonList(cluster.getZkServer().getZkAddress()), Optional.empty())diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientRetryTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientRetryTest.javaindex 1bf8c9135cf..19c182485ab 100644--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientRetryTest.java+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientRetryTest.java@@ -49,7 +49,7 @@ public class CloudSolrClientRetryTest extends SolrCloudTestCase {CloudSolrClient solrClient = cluster.getSolrClient();CollectionAdminRequest.createCollection(collectionName, 1, 1).process(solrClient);String prometheusMetric =- "solr_core_requests_total{category=\"UPDATE\",collection=\"testRetry\",core=\"testRetry_shard1_replica_n1\",handler=\"/update\",otel_scope_name=\"org.apache.solr\",replica=\"replica_n1\",shard=\"shard1\"}";+ "solr_core_requests_total{category=\"UPDATE\",collection=\"testRetry\",core=\"testRetry_shard1_replica_n1\",handler=\"/update\",otel_scope_name=\"org.apache.solr\",replica_type=\"NRT\",shard=\"shard1\"}";solrClient.add(collectionName, new SolrInputDocument("id", "1"));assertEquals(1.0, getPrometheusMetricValue(solrClient, prometheusMetric), 0.0);diff --git a/solr/test-framework/src/java/org/apache/solr/util/SolrMetricTestUtils.java b/solr/test-framework/src/java/org/apache/solr/util/SolrMetricTestUtils.javaindex 1fb3b0082cf..6bd14012567 100644--- a/solr/test-framework/src/java/org/apache/solr/util/SolrMetricTestUtils.java+++ b/solr/test-framework/src/java/org/apache/solr/util/SolrMetricTestUtils.java@@ -33,7 +33,6 @@ import java.util.List;import java.util.Map;import java.util.Random;import org.apache.lucene.tests.util.TestUtil;-import org.apache.solr.common.util.Utils;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrInfoBean;@@ -193,9 +192,8 @@ public final class SolrMetricTestUtils {.label("shard", core.getCoreDescriptor().getCloudDescriptor().getShardId()).label("core", core.getName()).label(- "replica",- Utils.parseMetricsReplicaName(- core.getCoreDescriptor().getCollectionName(), core.getName()))+ "replica_type",+ core.getCoreDescriptor().getCloudDescriptor().getReplicaType().toString()).label("otel_scope_name", "org.apache.solr");}