Apache Solr open-source search software
SOLR-17865: Support aggregateNodeLevelMetricsEnabled (#3734)
Matthew Biscocho315bf57ee493
diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.javaindex f726842d773..f2407a4273e 100644--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java@@ -21,8 +21,6 @@ import static org.apache.solr.response.SolrQueryResponse.haveCompleteResults;import io.opentelemetry.api.common.AttributeKey;import io.opentelemetry.api.common.Attributes;-import io.opentelemetry.api.metrics.LongCounter;-import io.opentelemetry.api.metrics.LongHistogram;import java.io.IOException;import java.lang.invoke.MethodHandles;import java.util.Collection;@@ -41,11 +39,10 @@ import org.apache.solr.core.PluginBag;import org.apache.solr.core.PluginInfo;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrInfoBean;-import org.apache.solr.metrics.SolrDelegateRegistryMetricsContext;import org.apache.solr.metrics.SolrMetricManager;-import org.apache.solr.metrics.SolrMetricProducer;import org.apache.solr.metrics.SolrMetricsContext;import org.apache.solr.metrics.otel.OtelUnit;+import org.apache.solr.metrics.otel.instruments.AttributedInstrumentFactory;import org.apache.solr.metrics.otel.instruments.AttributedLongCounter;import org.apache.solr.metrics.otel.instruments.AttributedLongTimer;import org.apache.solr.request.SolrQueryRequest;@@ -72,6 +69,7 @@ public abstract class RequestHandlerBasePermissionNameProvider {public static final String REQUEST_CPU_TIMER_CONTEXT = "publishCpuTime";+ public static final AttributeKey<String> SOURCE_ATTR = AttributeKey.stringKey("source");protected NamedList<?> initArgs = null;protected SolrParams defaults;protected SolrParams appends;@@ -167,21 +165,13 @@ public abstract class RequestHandlerBase@Overridepublic void initializeMetrics(SolrMetricsContext parentContext, Attributes attributes, String scope) {- if (aggregateNodeLevelMetricsEnabled) {- this.solrMetricsContext =- new SolrDelegateRegistryMetricsContext(- parentContext.getMetricManager(),- parentContext.getRegistryName(),- SolrMetricProducer.getUniqueMetricTag(this, parentContext.getTag()),- SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));- } else {- this.solrMetricsContext = parentContext.getChildContext(this);- }+ this.solrMetricsContext = parentContext.getChildContext(this);metrics =new HandlerMetrics(solrMetricsContext,- attributes.toBuilder().put(CATEGORY_ATTR, getCategory().toString()).build());+ attributes.toBuilder().put(CATEGORY_ATTR, getCategory().toString()).build(),+ aggregateNodeLevelMetricsEnabled);}/** Metrics for this handler. */@@ -189,7 +179,8 @@ public abstract class RequestHandlerBasepublic static final HandlerMetrics NO_OP =new HandlerMetrics(new SolrMetricsContext(new SolrMetricManager(null), "NO_OP", "NO_OP"),- Attributes.empty());+ Attributes.empty(),+ false);public AttributedLongCounter requests;public AttributedLongCounter numServerErrors;@@ -197,53 +188,41 @@ public abstract class RequestHandlerBasepublic AttributedLongCounter numTimeouts;public AttributedLongTimer requestTimes;- public HandlerMetrics(SolrMetricsContext solrMetricsContext, Attributes attributes) {- LongCounter requestMetric;- LongCounter errorRequestMetric;- LongCounter timeoutRequestMetric;- LongHistogram requestTimeMetric;-- if (solrMetricsContext.getRegistryName().equals("solr.node")) {- requestMetric =- solrMetricsContext.longCounter("solr_node_requests", "Http Solr node requests");- errorRequestMetric =- solrMetricsContext.longCounter(- "solr_node_requests_errors", "HTTP Solr node request errors");- timeoutRequestMetric =- solrMetricsContext.longCounter(- "solr_node_requests_timeout", "HTTP Solr node request timeouts");- requestTimeMetric =- solrMetricsContext.longHistogram(- "solr_node_requests_times", "HTTP Solr node request times", OtelUnit.MILLISECONDS);- } else {- requestMetric =- solrMetricsContext.longCounter("solr_core_requests", "HTTP Solr core requests");- errorRequestMetric =- solrMetricsContext.longCounter(- "solr_core_requests_errors", "HTTP Solr core request errors");- timeoutRequestMetric =- solrMetricsContext.longCounter(- "solr_core_requests_timeout", "HTTP Solr core request timeouts");- requestTimeMetric =- solrMetricsContext.longHistogram(- "solr_core_requests_times", "HTTP Solr core request times", OtelUnit.MILLISECONDS);- }+ public HandlerMetrics(+ SolrMetricsContext solrMetricsContext,+ Attributes coreAttributes,+ boolean aggregateNodeLevelMetricsEnabled) {- requests = new AttributedLongCounter(requestMetric, attributes);+ AttributedInstrumentFactory factory =+ new AttributedInstrumentFactory(+ solrMetricsContext, coreAttributes, aggregateNodeLevelMetricsEnabled);++ requests =+ factory.attributedLongCounter(+ "solr_core_requests", "HTTP Solr requests", Attributes.empty());numServerErrors =- new AttributedLongCounter(- errorRequestMetric,- attributes.toBuilder().put(AttributeKey.stringKey("source"), "server").build());+ factory.attributedLongCounter(+ "solr_core_requests_errors",+ "HTTP Solr request errors",+ Attributes.of(SOURCE_ATTR, "server"));numClientErrors =- new AttributedLongCounter(- errorRequestMetric,- attributes.toBuilder().put(AttributeKey.stringKey("source"), "client").build());-- numTimeouts = new AttributedLongCounter(timeoutRequestMetric, attributes);-- requestTimes = new AttributedLongTimer(requestTimeMetric, attributes);+ factory.attributedLongCounter(+ "solr_core_requests_errors",+ "HTTP Solr request errors",+ Attributes.of(SOURCE_ATTR, "client"));++ numTimeouts =+ factory.attributedLongCounter(+ "solr_core_requests_timeout", "HTTP Solr request timeouts", Attributes.empty());++ requestTimes =+ factory.attributedLongTimer(+ "solr_core_requests_times",+ "HTTP Solr request times",+ OtelUnit.MILLISECONDS,+ Attributes.empty());}}diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.javaindex b42e62d187b..cc7ad157651 100644--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java@@ -98,8 +98,6 @@ public class SearchHandler extends RequestHandlerBasestatic final String INIT_FIRST_COMPONENTS = "first-components";static final String INIT_LAST_COMPONENTS = "last-components";- protected static final String SHARD_HANDLER_SUFFIX = "[shard]";-private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());/**@@ -171,7 +169,8 @@ public class SearchHandler extends RequestHandlerBase.putAll(attributes).put(CATEGORY_ATTR, getCategory().toString()).put(INTERNAL_ATTR, true)- .build());+ .build(),+ false);}@Overridediff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.javaindex 43adf8ad087..e2f28293e4a 100644--- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java@@ -29,7 +29,7 @@ public class AttributedDoubleCounter {this.attributes = attributes;}- public void inc() {+ public final void inc() {add(1.0);}diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.javaindex 23c0d3b9a53..ca2808a596f 100644--- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java@@ -29,11 +29,11 @@ public class AttributedDoubleUpDownCounter {this.attributes = attributes;}- public void inc() {+ public final void inc() {add(1.0);}- public void dec() {+ public final void dec() {add(-1.0);}diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.javanew file mode 100644index 00000000000..a443eab92b0--- /dev/null+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java@@ -0,0 +1,160 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.metrics.otel.instruments;++import static org.apache.solr.handler.component.SearchHandler.INTERNAL_ATTR;+import static org.apache.solr.metrics.SolrCoreMetricManager.COLLECTION_ATTR;+import static org.apache.solr.metrics.SolrCoreMetricManager.CORE_ATTR;+import static org.apache.solr.metrics.SolrCoreMetricManager.REPLICA_TYPE_ATTR;+import static org.apache.solr.metrics.SolrCoreMetricManager.SHARD_ATTR;++import io.opentelemetry.api.common.AttributeKey;+import io.opentelemetry.api.common.Attributes;+import io.opentelemetry.api.metrics.LongCounter;+import io.opentelemetry.api.metrics.LongHistogram;+import io.opentelemetry.api.metrics.LongUpDownCounter;+import java.util.Set;+import org.apache.solr.core.SolrInfoBean;+import org.apache.solr.metrics.SolrMetricManager;+import org.apache.solr.metrics.SolrMetricsContext;+import org.apache.solr.metrics.otel.OtelUnit;++/**+ * Factory for creating metrics instruments that can write to either single or dual registries (core+ * and node).+ */+public class AttributedInstrumentFactory {++ private static final Set<AttributeKey<?>> FILTER_ATTRS_SET =+ Set.of(COLLECTION_ATTR, CORE_ATTR, SHARD_ATTR, REPLICA_TYPE_ATTR, INTERNAL_ATTR);+ private final SolrMetricsContext primaryMetricsContext;+ private final Attributes primaryAttributes;+ private final boolean aggregateToNodeRegistry;+ private final boolean primaryIsNodeRegistry;+ private SolrMetricsContext nodeMetricsContext = null;+ private Attributes nodeAttributes = null;++ public AttributedInstrumentFactory(+ SolrMetricsContext primaryMetricsContext,+ Attributes primaryAttributes,+ boolean aggregateToNodeRegistry) {+ this.primaryMetricsContext = primaryMetricsContext;+ this.primaryAttributes = primaryAttributes;+ this.aggregateToNodeRegistry = aggregateToNodeRegistry;++ // Primary could be a node+ this.primaryIsNodeRegistry =+ primaryMetricsContext+ .getRegistryName()+ .equals(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));++ // Only create node registry if we want dual registry mode AND primary is not already a node+ // registry+ if (aggregateToNodeRegistry && !primaryIsNodeRegistry) {+ this.nodeMetricsContext =+ new SolrMetricsContext(+ primaryMetricsContext.getMetricManager(),+ SolrMetricManager.getRegistryName(SolrInfoBean.Group.node),+ null);+ this.nodeAttributes = createNodeAttributes(primaryAttributes);+ }+ }++ public AttributedLongCounter attributedLongCounter(+ String metricName, String description, Attributes additionalAttributes) {+ Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes, additionalAttributes);++ if (aggregateToNodeRegistry && nodeMetricsContext != null) {+ Attributes finalNodeAttrs = appendAttributes(nodeAttributes, additionalAttributes);++ LongCounter primaryCounter = primaryMetricsContext.longCounter(metricName, description);+ LongCounter nodeCounter =+ nodeMetricsContext.longCounter(toNodeMetricName(metricName), description);+ return new DualRegistryAttributedLongCounter(+ primaryCounter, finalPrimaryAttrs, nodeCounter, finalNodeAttrs);+ } else {+ String finalMetricName = primaryIsNodeRegistry ? toNodeMetricName(metricName) : metricName;+ LongCounter counter = primaryMetricsContext.longCounter(finalMetricName, description);+ return new AttributedLongCounter(counter, finalPrimaryAttrs);+ }+ }++ public AttributedLongUpDownCounter attributedLongUpDownCounter(+ String metricName, String description, Attributes additionalAttributes) {+ Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes, additionalAttributes);++ if (aggregateToNodeRegistry && nodeMetricsContext != null) {+ Attributes finalNodeAttrs = appendAttributes(nodeAttributes, additionalAttributes);++ LongUpDownCounter primaryCounter =+ primaryMetricsContext.longUpDownCounter(metricName, description);+ LongUpDownCounter nodeCounter =+ nodeMetricsContext.longUpDownCounter(toNodeMetricName(metricName), description);+ return new DualRegistryAttributedLongUpDownCounter(+ primaryCounter, finalPrimaryAttrs, nodeCounter, finalNodeAttrs);+ } else {+ String finalMetricName = primaryIsNodeRegistry ? toNodeMetricName(metricName) : metricName;+ LongUpDownCounter counter =+ primaryMetricsContext.longUpDownCounter(finalMetricName, description);+ return new AttributedLongUpDownCounter(counter, finalPrimaryAttrs);+ }+ }++ public AttributedLongTimer attributedLongTimer(+ String metricName, String description, OtelUnit unit, Attributes additionalAttributes) {+ Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes, additionalAttributes);++ if (aggregateToNodeRegistry && nodeMetricsContext != null) {+ Attributes finalNodeAttrs = appendAttributes(nodeAttributes, additionalAttributes);+ LongHistogram primaryHistogram =+ primaryMetricsContext.longHistogram(metricName, description, unit);+ LongHistogram nodeHistogram =+ nodeMetricsContext.longHistogram(toNodeMetricName(metricName), description, unit);+ return new DualRegistryAttributedLongTimer(+ primaryHistogram, finalPrimaryAttrs, nodeHistogram, finalNodeAttrs);+ } else {+ String finalMetricName = primaryIsNodeRegistry ? toNodeMetricName(metricName) : metricName;+ LongHistogram histogram =+ primaryMetricsContext.longHistogram(finalMetricName, description, unit);+ return new AttributedLongTimer(histogram, finalPrimaryAttrs);+ }+ }++ /** Replace core metric name prefix to node prefix */+ private String toNodeMetricName(String coreMetricName) {+ return coreMetricName.replace("solr_core", "solr_node");+ }++ /** Filter out core attributes and keep all others for node-level metrics */+ @SuppressWarnings("unchecked")+ private Attributes createNodeAttributes(Attributes coreAttributes) {+ var builder = Attributes.builder();++ coreAttributes.forEach(+ (key, value) -> {+ if (!FILTER_ATTRS_SET.contains(key)) {+ builder.put((AttributeKey<Object>) key, value);+ }+ });++ return builder.build();+ }++ private Attributes appendAttributes(Attributes base, Attributes additional) {+ return base.toBuilder().putAll(additional).build();+ }+}diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.javaindex f33b4784c53..e2a477eac35 100644--- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java@@ -29,7 +29,7 @@ public class AttributedLongCounter {this.attributes = attributes;}- public void inc() {+ public final void inc() {add(1L);}diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.javaindex 91487c9e677..e3ae3e98edd 100644--- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java@@ -29,11 +29,11 @@ public class AttributedLongUpDownCounter {this.attributes = attributes;}- public void inc() {+ public final void inc() {add(1L);}- public void dec() {+ public final void dec() {add(-1L);}diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.javanew file mode 100644index 00000000000..15fc27b85c5--- /dev/null+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java@@ -0,0 +1,45 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.metrics.otel.instruments;++import io.opentelemetry.api.common.Attributes;+import io.opentelemetry.api.metrics.LongCounter;++/**+ * An AttributedLongCounter that writes to both core and node registries with corresponding+ * attributes.+ */+public class DualRegistryAttributedLongCounter extends AttributedLongCounter {++ private final AttributedLongCounter nodeCounter;++ public DualRegistryAttributedLongCounter(+ LongCounter coreCounter,+ Attributes coreAttributes,+ LongCounter nodeCounter,+ Attributes nodeAttributes) {+ super(coreCounter, coreAttributes);+ assert coreCounter != nodeCounter;+ this.nodeCounter = new AttributedLongCounter(nodeCounter, nodeAttributes);+ }++ @Override+ public void add(Long value) {+ super.add(value);+ nodeCounter.add(value);+ }+}diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.javanew file mode 100644index 00000000000..b42d869892c--- /dev/null+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java@@ -0,0 +1,45 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.metrics.otel.instruments;++import io.opentelemetry.api.common.Attributes;+import io.opentelemetry.api.metrics.LongHistogram;++/**+ * An AttributedLongTimer that records to both core and node registries with corresponding+ * attributes.+ */+public class DualRegistryAttributedLongTimer extends AttributedLongTimer {++ private final AttributedLongTimer nodeTimer;++ public DualRegistryAttributedLongTimer(+ LongHistogram coreHistogram,+ Attributes coreAttributes,+ LongHistogram nodeHistogram,+ Attributes nodeAttributes) {+ super(coreHistogram, coreAttributes);+ assert coreHistogram != nodeHistogram;+ this.nodeTimer = new AttributedLongTimer(nodeHistogram, nodeAttributes);+ }++ @Override+ public void record(Long value) {+ super.record(value);+ nodeTimer.record(value);+ }+}diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.javanew file mode 100644index 00000000000..6cf505fb54e--- /dev/null+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java@@ -0,0 +1,45 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.metrics.otel.instruments;++import io.opentelemetry.api.common.Attributes;+import io.opentelemetry.api.metrics.LongUpDownCounter;++/**+ * An AttributedLongUpDownCounter that writes to both core and node registries with corresponding+ * attributes.+ */+public class DualRegistryAttributedLongUpDownCounter extends AttributedLongUpDownCounter {++ private final AttributedLongUpDownCounter nodeUpDownCounter;++ public DualRegistryAttributedLongUpDownCounter(+ LongUpDownCounter coreUpDownCounter,+ Attributes coreAttributes,+ LongUpDownCounter nodeUpDownCounter,+ Attributes nodeAttributes) {+ super(coreUpDownCounter, coreAttributes);+ assert coreUpDownCounter != nodeUpDownCounter;+ this.nodeUpDownCounter = new AttributedLongUpDownCounter(nodeUpDownCounter, nodeAttributes);+ }++ @Override+ public void add(Long value) {+ super.add(value);+ nodeUpDownCounter.add(value);+ }+}diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.javaindex 0a027282c46..dc802afc2b7 100644--- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java+++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java@@ -53,11 +53,9 @@ import org.apache.solr.common.util.IOUtils;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.SolrConfig.UpdateHandlerInfo;import org.apache.solr.core.SolrCore;-import org.apache.solr.core.SolrInfoBean;-import org.apache.solr.metrics.SolrDelegateRegistryMetricsContext;-import org.apache.solr.metrics.SolrMetricManager;import org.apache.solr.metrics.SolrMetricProducer;import org.apache.solr.metrics.SolrMetricsContext;+import org.apache.solr.metrics.otel.instruments.AttributedInstrumentFactory;import org.apache.solr.metrics.otel.instruments.AttributedLongCounter;import org.apache.solr.metrics.otel.instruments.AttributedLongUpDownCounter;import org.apache.solr.request.LocalSolrQueryRequest;@@ -227,84 +225,121 @@ public class DirectUpdateHandler2 extends UpdateHandler@Overridepublic void initializeMetrics(SolrMetricsContext parentContext, Attributes attributes, String scope) {- if (core.getSolrConfig().getUpdateHandlerInfo().aggregateNodeLevelMetricsEnabled) {- this.solrMetricsContext =- new SolrDelegateRegistryMetricsContext(- parentContext.getMetricManager(),- parentContext.getRegistryName(),- SolrMetricProducer.getUniqueMetricTag(this, parentContext.getTag()),- SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));- } else {- this.solrMetricsContext = parentContext.getChildContext(this);- }- final List<AutoCloseable> observables = new ArrayList<>();+ this.solrMetricsContext = parentContext.getChildContext(this);var baseAttributes =attributes.toBuilder().put(AttributeKey.stringKey("category"), getCategory().toString()).build();- var baseCommandsMetric =- solrMetricsContext.longUpDownCounter(- "solr_core_update_cumulative_ops",- "Cumulative number of update commands processed. Cumulative can decrease from rollback command");+ boolean aggregateNodeLevelMetricsEnabled =+ core.getSolrConfig().getUpdateHandlerInfo().aggregateNodeLevelMetricsEnabled;++ createMetrics(baseAttributes, aggregateNodeLevelMetricsEnabled);+ }++ private void createMetrics(Attributes baseAttributes, boolean aggregateToNodeRegistry) {+ final List<AutoCloseable> observables = new ArrayList<>();++ AttributedInstrumentFactory factory =+ new AttributedInstrumentFactory(+ solrMetricsContext, baseAttributes, aggregateToNodeRegistry);addCommandsCumulative =- new AttributedLongUpDownCounter(- baseCommandsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build());+ factory.attributedLongUpDownCounter(+ "solr_core_update_cumulative_ops",+ "Cumulative number of update commands processed. Cumulative can decrease from rollback command",+ Attributes.of(OPERATION_ATTR, "adds"));deleteByIdCommandsCumulative =- new AttributedLongUpDownCounter(- baseCommandsMetric,- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_id").build());+ factory.attributedLongUpDownCounter(+ "solr_core_update_cumulative_ops",+ "Cumulative number of update commands processed. Cumulative can decrease from rollback command",+ Attributes.of(OPERATION_ATTR, "deletes_by_id"));deleteByQueryCommandsCumulative =- new AttributedLongUpDownCounter(- baseCommandsMetric,- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_query").build());-- var baseCommitOpsMetric =- solrMetricsContext.longCounter(- "solr_core_update_commit_ops", "Total number of commit operations");+ factory.attributedLongUpDownCounter(+ "solr_core_update_cumulative_ops",+ "Cumulative number of update commands processed. Cumulative can decrease from rollback command",+ Attributes.of(OPERATION_ATTR, "deletes_by_query"));commitCommands =- new AttributedLongCounter(- baseCommitOpsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "commits").build());+ factory.attributedLongCounter(+ "solr_core_update_commit_ops",+ "Total number of commit operations",+ Attributes.of(OPERATION_ATTR, "commits"));optimizeCommands =- new AttributedLongCounter(- baseCommitOpsMetric,- baseAttributes.toBuilder().put(OPERATION_ATTR, "optimize").build());+ factory.attributedLongCounter(+ "solr_core_update_commit_ops",+ "Total number of commit operations",+ Attributes.of(OPERATION_ATTR, "optimize"));mergeIndexesCommands =- new AttributedLongCounter(- baseCommitOpsMetric,- baseAttributes.toBuilder().put(OPERATION_ATTR, "merge_indexes").build());+ factory.attributedLongCounter(+ "solr_core_update_commit_ops",+ "Total number of commit operations",+ Attributes.of(OPERATION_ATTR, "merge_indexes"));expungeDeleteCommands =- new AttributedLongCounter(- baseCommitOpsMetric,- baseAttributes.toBuilder().put(OPERATION_ATTR, "expunge_deletes").build());-- var baseMaintenanceMetric =- solrMetricsContext.longCounter(- "solr_core_update_maintenance_ops", "Total number of maintenance operations");+ factory.attributedLongCounter(+ "solr_core_update_commit_ops",+ "Total number of commit operations",+ Attributes.of(OPERATION_ATTR, "expunge_deletes"));rollbackCommands =- new AttributedLongCounter(- baseMaintenanceMetric,- baseAttributes.toBuilder().put(OPERATION_ATTR, "rollback").build());+ factory.attributedLongCounter(+ "solr_core_update_maintenance_ops",+ "Total number of maintenance operations",+ Attributes.of(OPERATION_ATTR, "rollback"));splitCommands =- new AttributedLongCounter(- baseMaintenanceMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "split").build());-- var baseErrorsMetric =- solrMetricsContext.longCounter("solr_core_update_errors", "Total number of update errors");+ factory.attributedLongCounter(+ "solr_core_update_maintenance_ops",+ "Total number of maintenance operations",+ Attributes.of(OPERATION_ATTR, "split"));numErrorsCumulative =- new AttributedLongCounter(baseErrorsMetric, baseAttributes.toBuilder().build());+ factory.attributedLongCounter(+ "solr_core_update_errors", "Total number of update errors", Attributes.empty());++ submittedAdds =+ factory.attributedLongCounter(+ "solr_core_update_submitted_ops",+ "Total number of submitted update operations",+ Attributes.of(OPERATION_ATTR, "adds"));++ submittedDeleteById =+ factory.attributedLongCounter(+ "solr_core_update_submitted_ops",+ "Total number of submitted update operations",+ Attributes.of(OPERATION_ATTR, "deletes_by_id"));+ submittedDeleteByQuery =+ factory.attributedLongCounter(+ "solr_core_update_submitted_ops",+ "Total number of submitted update operations",+ Attributes.of(OPERATION_ATTR, "deletes_by_query"));++ committedAdds =+ factory.attributedLongCounter(+ "solr_core_update_committed_ops",+ "Total number of committed update operations",+ Attributes.of(OPERATION_ATTR, "adds"));++ committedDeleteById =+ factory.attributedLongCounter(+ "solr_core_update_committed_ops",+ "Total number of committed update operations",+ Attributes.of(OPERATION_ATTR, "deletes_by_id"));++ committedDeleteByQuery =+ factory.attributedLongCounter(+ "solr_core_update_committed_ops",+ "Total number of committed update operations",+ Attributes.of(OPERATION_ATTR, "deletes_by_query"));++ // Create observable metrics only for core registryobservables.add(solrMetricsContext.observableLongCounter("solr_core_update_auto_commits",@@ -361,38 +396,6 @@ public class DirectUpdateHandler2 extends UpdateHandler}));this.toClose = Collections.unmodifiableList(observables);-- var baseSubmittedOpsMetric =- solrMetricsContext.longCounter(- "solr_core_update_submitted_ops", "Total number of submitted update operations");-- var baseCommittedOpsMetric =- solrMetricsContext.longCounter(- "solr_core_update_committed_ops", "Total number of committed update operations");-- submittedAdds =- new AttributedLongCounter(- baseSubmittedOpsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build());- submittedDeleteById =- new AttributedLongCounter(- baseSubmittedOpsMetric,- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_id").build());- submittedDeleteByQuery =- new AttributedLongCounter(- baseSubmittedOpsMetric,- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_query").build());-- committedAdds =- new AttributedLongCounter(- baseCommittedOpsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build());- committedDeleteById =- new AttributedLongCounter(- baseCommittedOpsMetric,- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_id").build());- committedDeleteByQuery =- new AttributedLongCounter(- baseCommittedOpsMetric,- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_query").build());}private void deleteAll() throws IOException {diff --git a/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xmlindex f23456d060a..2b1586ae253 100644--- a/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml+++ b/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml@@ -29,7 +29,7 @@<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>- <updateHandler class="solr.DirectUpdateHandler2">+ <updateHandler class="solr.DirectUpdateHandler2" aggregateNodeLevelMetricsEnabled="true"><commitWithin><softCommit>${solr.commitwithin.softcommit:true}</softCommit></commitWithin>@@ -52,4 +52,4 @@<indexConfig><mergeScheduler class="${solr.mscheduler:org.apache.lucene.index.ConcurrentMergeScheduler}"/></indexConfig>-</config>\ No newline at end of file+</config>diff --git a/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java b/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.javaindex 74d3d6b2510..2320b526b08 100644--- a/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java+++ b/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java@@ -17,6 +17,7 @@package org.apache.solr.handler;+import static org.apache.solr.metrics.SolrMetricProducer.HANDLER_ATTR;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.argThat;import static org.mockito.ArgumentMatchers.eq;@@ -196,6 +197,6 @@ public class RequestHandlerBaseTest extends SolrTestCaseJ4 {when(metricsContext.longHistogram(any(), any())).thenReturn(mockLongHistogram);return new RequestHandlerBase.HandlerMetrics(- metricsContext, Attributes.of(AttributeKey.stringKey("/handler"), "/someBaseMetricPath"));+ metricsContext, Attributes.of(HANDLER_ATTR, "/someBaseMetricPath"), false);}}diff --git a/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java b/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.javaindex e9618297760..e338939b30c 100644--- a/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java+++ b/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java@@ -16,19 +16,17 @@*/package org.apache.solr.handler;+import io.prometheus.metrics.model.snapshots.CounterSnapshot;+import io.prometheus.metrics.model.snapshots.Labels;import java.io.IOException;-import java.util.HashSet;-import java.util.Map;-import java.util.Set;import org.apache.solr.client.solrj.SolrQuery;-import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.request.GenericSolrRequest;import org.apache.solr.cloud.SolrCloudTestCase;import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.SolrCore;+import org.apache.solr.util.SolrMetricTestUtils;import org.junit.AfterClass;import org.junit.Before;import org.junit.BeforeClass;@@ -54,10 +52,7 @@ public class RequestHandlerMetricsTest extends SolrCloudTestCase {System.clearProperty("metricsEnabled");}- // TODO: Migrate aggregateNodeLevelMetricsEnabled for OTEL metrics@Test- @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17865")- @SuppressWarnings({"unchecked"})public void testAggregateNodeLevelMetrics() throws SolrServerException, IOException {String collection1 = "testRequestHandlerMetrics1";String collection2 = "testRequestHandlerMetrics2";@@ -82,97 +77,83 @@ public class RequestHandlerMetricsTest extends SolrCloudTestCase {cloudClient.query(collection1, solrQuery);cloudClient.query(collection2, solrQuery);- NamedList<Object> response =- cloudClient.request(- new GenericSolrRequest(- SolrRequest.METHOD.GET, "/admin/metrics", SolrRequest.SolrRequestType.ADMIN));-- NamedList<Object> metrics = (NamedList<Object>) response.get("metrics");-- final double[] minQueryTime = {Double.MAX_VALUE};- final double[] maxQueryTime = {-1.0};- final double[] minUpdateTime = {Double.MAX_VALUE};- final double[] maxUpdateTime = {-1.0};- Set<NamedList<Object>> coreMetrics = new HashSet<>();- metrics.forEach(- (key, coreMetric) -> {- if (key.startsWith("solr.core.testRequestHandlerMetrics")) {- coreMetrics.add((NamedList<Object>) coreMetric);- }- });- assertEquals(2, coreMetrics.size());- coreMetrics.forEach(- metric -> {- assertEquals(- 1L,- ((Map<String, Number>) metric.get("QUERY./select.requestTimes"))- .get("count")- .longValue());- minQueryTime[0] =- Math.min(- minQueryTime[0],- ((Map<String, Number>) metric.get("QUERY./select.requestTimes"))- .get("min_ms")- .doubleValue());- maxQueryTime[0] =- Math.max(- maxQueryTime[0],- ((Map<String, Number>) metric.get("QUERY./select.requestTimes"))- .get("max_ms")- .doubleValue());- assertEquals(- 1L,- ((Map<String, Number>) metric.get("UPDATE./update.requestTimes"))- .get("count")- .longValue());- minUpdateTime[0] =- Math.min(- minUpdateTime[0],- ((Map<String, Number>) metric.get("UPDATE./update.requestTimes"))- .get("min_ms")- .doubleValue());- maxUpdateTime[0] =- Math.max(- maxUpdateTime[0],- ((Map<String, Number>) metric.get("UPDATE./update.requestTimes"))- .get("max_ms")- .doubleValue());- });-- NamedList<Object> nodeMetrics = (NamedList<Object>) metrics.get("solr.node");- assertEquals(- 2L,- ((Map<String, Number>) nodeMetrics.get("QUERY./select.requestTimes"))- .get("count")- .longValue());- assertEquals(- minQueryTime[0],- ((Map<String, Number>) nodeMetrics.get("QUERY./select.requestTimes"))- .get("min_ms")- .doubleValue(),- 0.0);- assertEquals(- maxQueryTime[0],- ((Map<String, Number>) nodeMetrics.get("QUERY./select.requestTimes"))- .get("max_ms")- .doubleValue(),- 0.0);- assertEquals(- 2L,- ((Map<String, Number>) nodeMetrics.get("UPDATE./update.requestTimes"))- .get("count")- .longValue());- assertEquals(- minUpdateTime[0],- ((Map<String, Number>) nodeMetrics.get("UPDATE./update.requestTimes"))- .get("min_ms")- .doubleValue(),- 0.0);- assertEquals(- maxUpdateTime[0],- ((Map<String, Number>) nodeMetrics.get("UPDATE./update.requestTimes"))- .get("max_ms")- .doubleValue(),- 0.0);+ var coreContainer = cluster.getJettySolrRunners().get(0).getCoreContainer();++ try (SolrCore core1 = coreContainer.getCore(coreContainer.getAllCoreNames().get(0));+ SolrCore core2 = coreContainer.getCore(coreContainer.getAllCoreNames().get(1))) {++ CounterSnapshot.CounterDataPointSnapshot actualCore1Selects =+ SolrMetricTestUtils.newCloudSelectRequestsDatapoint(core1);+ CounterSnapshot.CounterDataPointSnapshot actualCore1Updates =+ SolrMetricTestUtils.newCloudUpdateRequestsDatapoint(core1);+ CounterSnapshot.CounterDataPointSnapshot actualCore2Selects =+ SolrMetricTestUtils.newCloudSelectRequestsDatapoint(core2);+ CounterSnapshot.CounterDataPointSnapshot actualCore2Updates =+ SolrMetricTestUtils.newCloudUpdateRequestsDatapoint(core2);+ CounterSnapshot.CounterDataPointSnapshot actualCore1SubmittedOps =+ SolrMetricTestUtils.getCounterDatapoint(+ core1,+ "solr_core_update_submitted_ops",+ SolrMetricTestUtils.newCloudLabelsBuilder(core1)+ .label("category", "UPDATE")+ .label("ops", "adds")+ .build());+ CounterSnapshot.CounterDataPointSnapshot actualCore2SubmittedOps =+ SolrMetricTestUtils.getCounterDatapoint(+ core2,+ "solr_core_update_submitted_ops",+ SolrMetricTestUtils.newCloudLabelsBuilder(core2)+ .label("category", "UPDATE")+ .label("ops", "adds")+ .build());++ assertEquals(1.0, actualCore1Selects.getValue(), 0.0);+ assertEquals(1.0, actualCore1Updates.getValue(), 0.0);+ assertEquals(1.0, actualCore2Updates.getValue(), 0.0);+ assertEquals(1.0, actualCore2Selects.getValue(), 0.0);+ assertEquals(1.0, actualCore1SubmittedOps.getValue(), 0.0);+ assertEquals(1.0, actualCore2SubmittedOps.getValue(), 0.0);++ // Get node metrics and the select/update requests should be the sum of both cores requests+ var nodeReader = SolrMetricTestUtils.getPrometheusMetricReader(coreContainer, "solr.node");++ CounterSnapshot.CounterDataPointSnapshot nodeSelectRequests =+ (CounterSnapshot.CounterDataPointSnapshot)+ SolrMetricTestUtils.getDataPointSnapshot(+ nodeReader,+ "solr_node_requests",+ Labels.builder()+ .label("category", "QUERY")+ .label("handler", "/select")+ .label("otel_scope_name", "org.apache.solr")+ .build());+ CounterSnapshot.CounterDataPointSnapshot nodeUpdateRequests =+ (CounterSnapshot.CounterDataPointSnapshot)+ SolrMetricTestUtils.getDataPointSnapshot(+ nodeReader,+ "solr_node_requests",+ Labels.builder()+ .label("category", "UPDATE")+ .label("handler", "/update")+ .label("otel_scope_name", "org.apache.solr")+ .build());+ CounterSnapshot.CounterDataPointSnapshot nodeSubmittedOps =+ (CounterSnapshot.CounterDataPointSnapshot)+ SolrMetricTestUtils.getDataPointSnapshot(+ nodeReader,+ "solr_node_update_submitted_ops",+ Labels.builder()+ .label("category", "UPDATE")+ .label("ops", "adds")+ .label("otel_scope_name", "org.apache.solr")+ .build());++ assertNotNull("Node select requests should be recorded", nodeSelectRequests);+ assertNotNull("Node update requests should be recorded", nodeUpdateRequests);+ assertNotNull("Node submitted update operations should be recorded", nodeSubmittedOps);+ assertEquals(2.0, nodeSelectRequests.getValue(), 0.0);+ assertEquals(2.0, nodeUpdateRequests.getValue(), 0.0);+ assertEquals(2.0, nodeSubmittedOps.getValue(), 0.0);+ }}}