Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-17865: Support aggregateNodeLevelMetricsEnabled (#3734)

Matthew Biscocho315bf57ee493
diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
index f726842d773..f2407a4273e 100644
--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
@@ -21,8 +21,6 @@ import static org.apache.solr.response.SolrQueryResponse.haveCompleteResults;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.common.Attributes;
-import io.opentelemetry.api.metrics.LongCounter;
-import io.opentelemetry.api.metrics.LongHistogram;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collection;
@@ -41,11 +39,10 @@ import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
-import org.apache.solr.metrics.SolrDelegateRegistryMetricsContext;
import org.apache.solr.metrics.SolrMetricManager;
-import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.metrics.otel.OtelUnit;
+import org.apache.solr.metrics.otel.instruments.AttributedInstrumentFactory;
import org.apache.solr.metrics.otel.instruments.AttributedLongCounter;
import org.apache.solr.metrics.otel.instruments.AttributedLongTimer;
import org.apache.solr.request.SolrQueryRequest;
@@ -72,6 +69,7 @@ public abstract class RequestHandlerBase
PermissionNameProvider {
public static final String REQUEST_CPU_TIMER_CONTEXT = "publishCpuTime";
+ public static final AttributeKey<String> SOURCE_ATTR = AttributeKey.stringKey("source");
protected NamedList<?> initArgs = null;
protected SolrParams defaults;
protected SolrParams appends;
@@ -167,21 +165,13 @@ public abstract class RequestHandlerBase
@Override
public void initializeMetrics(
SolrMetricsContext parentContext, Attributes attributes, String scope) {
- if (aggregateNodeLevelMetricsEnabled) {
- this.solrMetricsContext =
- new SolrDelegateRegistryMetricsContext(
- parentContext.getMetricManager(),
- parentContext.getRegistryName(),
- SolrMetricProducer.getUniqueMetricTag(this, parentContext.getTag()),
- SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));
- } else {
- this.solrMetricsContext = parentContext.getChildContext(this);
- }
+ this.solrMetricsContext = parentContext.getChildContext(this);
metrics =
new HandlerMetrics(
solrMetricsContext,
- attributes.toBuilder().put(CATEGORY_ATTR, getCategory().toString()).build());
+ attributes.toBuilder().put(CATEGORY_ATTR, getCategory().toString()).build(),
+ aggregateNodeLevelMetricsEnabled);
}
/** Metrics for this handler. */
@@ -189,7 +179,8 @@ public abstract class RequestHandlerBase
public static final HandlerMetrics NO_OP =
new HandlerMetrics(
new SolrMetricsContext(new SolrMetricManager(null), "NO_OP", "NO_OP"),
- Attributes.empty());
+ Attributes.empty(),
+ false);
public AttributedLongCounter requests;
public AttributedLongCounter numServerErrors;
@@ -197,53 +188,41 @@ public abstract class RequestHandlerBase
public AttributedLongCounter numTimeouts;
public AttributedLongTimer requestTimes;
- public HandlerMetrics(SolrMetricsContext solrMetricsContext, Attributes attributes) {
- LongCounter requestMetric;
- LongCounter errorRequestMetric;
- LongCounter timeoutRequestMetric;
- LongHistogram requestTimeMetric;
-
- if (solrMetricsContext.getRegistryName().equals("solr.node")) {
- requestMetric =
- solrMetricsContext.longCounter("solr_node_requests", "Http Solr node requests");
- errorRequestMetric =
- solrMetricsContext.longCounter(
- "solr_node_requests_errors", "HTTP Solr node request errors");
- timeoutRequestMetric =
- solrMetricsContext.longCounter(
- "solr_node_requests_timeout", "HTTP Solr node request timeouts");
- requestTimeMetric =
- solrMetricsContext.longHistogram(
- "solr_node_requests_times", "HTTP Solr node request times", OtelUnit.MILLISECONDS);
- } else {
- requestMetric =
- solrMetricsContext.longCounter("solr_core_requests", "HTTP Solr core requests");
- errorRequestMetric =
- solrMetricsContext.longCounter(
- "solr_core_requests_errors", "HTTP Solr core request errors");
- timeoutRequestMetric =
- solrMetricsContext.longCounter(
- "solr_core_requests_timeout", "HTTP Solr core request timeouts");
- requestTimeMetric =
- solrMetricsContext.longHistogram(
- "solr_core_requests_times", "HTTP Solr core request times", OtelUnit.MILLISECONDS);
- }
+ public HandlerMetrics(
+ SolrMetricsContext solrMetricsContext,
+ Attributes coreAttributes,
+ boolean aggregateNodeLevelMetricsEnabled) {
- requests = new AttributedLongCounter(requestMetric, attributes);
+ AttributedInstrumentFactory factory =
+ new AttributedInstrumentFactory(
+ solrMetricsContext, coreAttributes, aggregateNodeLevelMetricsEnabled);
+
+ requests =
+ factory.attributedLongCounter(
+ "solr_core_requests", "HTTP Solr requests", Attributes.empty());
numServerErrors =
- new AttributedLongCounter(
- errorRequestMetric,
- attributes.toBuilder().put(AttributeKey.stringKey("source"), "server").build());
+ factory.attributedLongCounter(
+ "solr_core_requests_errors",
+ "HTTP Solr request errors",
+ Attributes.of(SOURCE_ATTR, "server"));
numClientErrors =
- new AttributedLongCounter(
- errorRequestMetric,
- attributes.toBuilder().put(AttributeKey.stringKey("source"), "client").build());
-
- numTimeouts = new AttributedLongCounter(timeoutRequestMetric, attributes);
-
- requestTimes = new AttributedLongTimer(requestTimeMetric, attributes);
+ factory.attributedLongCounter(
+ "solr_core_requests_errors",
+ "HTTP Solr request errors",
+ Attributes.of(SOURCE_ATTR, "client"));
+
+ numTimeouts =
+ factory.attributedLongCounter(
+ "solr_core_requests_timeout", "HTTP Solr request timeouts", Attributes.empty());
+
+ requestTimes =
+ factory.attributedLongTimer(
+ "solr_core_requests_times",
+ "HTTP Solr request times",
+ OtelUnit.MILLISECONDS,
+ Attributes.empty());
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index b42e62d187b..cc7ad157651 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -98,8 +98,6 @@ public class SearchHandler extends RequestHandlerBase
static final String INIT_FIRST_COMPONENTS = "first-components";
static final String INIT_LAST_COMPONENTS = "last-components";
- protected static final String SHARD_HANDLER_SUFFIX = "[shard]";
-
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/**
@@ -171,7 +169,8 @@ public class SearchHandler extends RequestHandlerBase
.putAll(attributes)
.put(CATEGORY_ATTR, getCategory().toString())
.put(INTERNAL_ATTR, true)
- .build());
+ .build(),
+ false);
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java
index 43adf8ad087..e2f28293e4a 100644
--- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java
@@ -29,7 +29,7 @@ public class AttributedDoubleCounter {
this.attributes = attributes;
}
- public void inc() {
+ public final void inc() {
add(1.0);
}
diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java
index 23c0d3b9a53..ca2808a596f 100644
--- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java
@@ -29,11 +29,11 @@ public class AttributedDoubleUpDownCounter {
this.attributes = attributes;
}
- public void inc() {
+ public final void inc() {
add(1.0);
}
- public void dec() {
+ public final void dec() {
add(-1.0);
}
diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java
new file mode 100644
index 00000000000..a443eab92b0
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.otel.instruments;
+
+import static org.apache.solr.handler.component.SearchHandler.INTERNAL_ATTR;
+import static org.apache.solr.metrics.SolrCoreMetricManager.COLLECTION_ATTR;
+import static org.apache.solr.metrics.SolrCoreMetricManager.CORE_ATTR;
+import static org.apache.solr.metrics.SolrCoreMetricManager.REPLICA_TYPE_ATTR;
+import static org.apache.solr.metrics.SolrCoreMetricManager.SHARD_ATTR;
+
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.LongCounter;
+import io.opentelemetry.api.metrics.LongHistogram;
+import io.opentelemetry.api.metrics.LongUpDownCounter;
+import java.util.Set;
+import org.apache.solr.core.SolrInfoBean;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.metrics.SolrMetricsContext;
+import org.apache.solr.metrics.otel.OtelUnit;
+
+/**
+ * Factory for creating metrics instruments that can write to either single or dual registries (core
+ * and node).
+ */
+public class AttributedInstrumentFactory {
+
+ private static final Set<AttributeKey<?>> FILTER_ATTRS_SET =
+ Set.of(COLLECTION_ATTR, CORE_ATTR, SHARD_ATTR, REPLICA_TYPE_ATTR, INTERNAL_ATTR);
+ private final SolrMetricsContext primaryMetricsContext;
+ private final Attributes primaryAttributes;
+ private final boolean aggregateToNodeRegistry;
+ private final boolean primaryIsNodeRegistry;
+ private SolrMetricsContext nodeMetricsContext = null;
+ private Attributes nodeAttributes = null;
+
+ public AttributedInstrumentFactory(
+ SolrMetricsContext primaryMetricsContext,
+ Attributes primaryAttributes,
+ boolean aggregateToNodeRegistry) {
+ this.primaryMetricsContext = primaryMetricsContext;
+ this.primaryAttributes = primaryAttributes;
+ this.aggregateToNodeRegistry = aggregateToNodeRegistry;
+
+ // Primary could be a node
+ this.primaryIsNodeRegistry =
+ primaryMetricsContext
+ .getRegistryName()
+ .equals(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));
+
+ // Only create node registry if we want dual registry mode AND primary is not already a node
+ // registry
+ if (aggregateToNodeRegistry && !primaryIsNodeRegistry) {
+ this.nodeMetricsContext =
+ new SolrMetricsContext(
+ primaryMetricsContext.getMetricManager(),
+ SolrMetricManager.getRegistryName(SolrInfoBean.Group.node),
+ null);
+ this.nodeAttributes = createNodeAttributes(primaryAttributes);
+ }
+ }
+
+ public AttributedLongCounter attributedLongCounter(
+ String metricName, String description, Attributes additionalAttributes) {
+ Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes, additionalAttributes);
+
+ if (aggregateToNodeRegistry && nodeMetricsContext != null) {
+ Attributes finalNodeAttrs = appendAttributes(nodeAttributes, additionalAttributes);
+
+ LongCounter primaryCounter = primaryMetricsContext.longCounter(metricName, description);
+ LongCounter nodeCounter =
+ nodeMetricsContext.longCounter(toNodeMetricName(metricName), description);
+ return new DualRegistryAttributedLongCounter(
+ primaryCounter, finalPrimaryAttrs, nodeCounter, finalNodeAttrs);
+ } else {
+ String finalMetricName = primaryIsNodeRegistry ? toNodeMetricName(metricName) : metricName;
+ LongCounter counter = primaryMetricsContext.longCounter(finalMetricName, description);
+ return new AttributedLongCounter(counter, finalPrimaryAttrs);
+ }
+ }
+
+ public AttributedLongUpDownCounter attributedLongUpDownCounter(
+ String metricName, String description, Attributes additionalAttributes) {
+ Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes, additionalAttributes);
+
+ if (aggregateToNodeRegistry && nodeMetricsContext != null) {
+ Attributes finalNodeAttrs = appendAttributes(nodeAttributes, additionalAttributes);
+
+ LongUpDownCounter primaryCounter =
+ primaryMetricsContext.longUpDownCounter(metricName, description);
+ LongUpDownCounter nodeCounter =
+ nodeMetricsContext.longUpDownCounter(toNodeMetricName(metricName), description);
+ return new DualRegistryAttributedLongUpDownCounter(
+ primaryCounter, finalPrimaryAttrs, nodeCounter, finalNodeAttrs);
+ } else {
+ String finalMetricName = primaryIsNodeRegistry ? toNodeMetricName(metricName) : metricName;
+ LongUpDownCounter counter =
+ primaryMetricsContext.longUpDownCounter(finalMetricName, description);
+ return new AttributedLongUpDownCounter(counter, finalPrimaryAttrs);
+ }
+ }
+
+ public AttributedLongTimer attributedLongTimer(
+ String metricName, String description, OtelUnit unit, Attributes additionalAttributes) {
+ Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes, additionalAttributes);
+
+ if (aggregateToNodeRegistry && nodeMetricsContext != null) {
+ Attributes finalNodeAttrs = appendAttributes(nodeAttributes, additionalAttributes);
+ LongHistogram primaryHistogram =
+ primaryMetricsContext.longHistogram(metricName, description, unit);
+ LongHistogram nodeHistogram =
+ nodeMetricsContext.longHistogram(toNodeMetricName(metricName), description, unit);
+ return new DualRegistryAttributedLongTimer(
+ primaryHistogram, finalPrimaryAttrs, nodeHistogram, finalNodeAttrs);
+ } else {
+ String finalMetricName = primaryIsNodeRegistry ? toNodeMetricName(metricName) : metricName;
+ LongHistogram histogram =
+ primaryMetricsContext.longHistogram(finalMetricName, description, unit);
+ return new AttributedLongTimer(histogram, finalPrimaryAttrs);
+ }
+ }
+
+ /** Replace core metric name prefix to node prefix */
+ private String toNodeMetricName(String coreMetricName) {
+ return coreMetricName.replace("solr_core", "solr_node");
+ }
+
+ /** Filter out core attributes and keep all others for node-level metrics */
+ @SuppressWarnings("unchecked")
+ private Attributes createNodeAttributes(Attributes coreAttributes) {
+ var builder = Attributes.builder();
+
+ coreAttributes.forEach(
+ (key, value) -> {
+ if (!FILTER_ATTRS_SET.contains(key)) {
+ builder.put((AttributeKey<Object>) key, value);
+ }
+ });
+
+ return builder.build();
+ }
+
+ private Attributes appendAttributes(Attributes base, Attributes additional) {
+ return base.toBuilder().putAll(additional).build();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
index f33b4784c53..e2a477eac35 100644
--- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
@@ -29,7 +29,7 @@ public class AttributedLongCounter {
this.attributes = attributes;
}
- public void inc() {
+ public final void inc() {
add(1L);
}
diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java
index 91487c9e677..e3ae3e98edd 100644
--- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java
@@ -29,11 +29,11 @@ public class AttributedLongUpDownCounter {
this.attributes = attributes;
}
- public void inc() {
+ public final void inc() {
add(1L);
}
- public void dec() {
+ public final void dec() {
add(-1L);
}
diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java
new file mode 100644
index 00000000000..15fc27b85c5
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.otel.instruments;
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.LongCounter;
+
+/**
+ * An AttributedLongCounter that writes to both core and node registries with corresponding
+ * attributes.
+ */
+public class DualRegistryAttributedLongCounter extends AttributedLongCounter {
+
+ private final AttributedLongCounter nodeCounter;
+
+ public DualRegistryAttributedLongCounter(
+ LongCounter coreCounter,
+ Attributes coreAttributes,
+ LongCounter nodeCounter,
+ Attributes nodeAttributes) {
+ super(coreCounter, coreAttributes);
+ assert coreCounter != nodeCounter;
+ this.nodeCounter = new AttributedLongCounter(nodeCounter, nodeAttributes);
+ }
+
+ @Override
+ public void add(Long value) {
+ super.add(value);
+ nodeCounter.add(value);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java
new file mode 100644
index 00000000000..b42d869892c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.otel.instruments;
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.LongHistogram;
+
+/**
+ * An AttributedLongTimer that records to both core and node registries with corresponding
+ * attributes.
+ */
+public class DualRegistryAttributedLongTimer extends AttributedLongTimer {
+
+ private final AttributedLongTimer nodeTimer;
+
+ public DualRegistryAttributedLongTimer(
+ LongHistogram coreHistogram,
+ Attributes coreAttributes,
+ LongHistogram nodeHistogram,
+ Attributes nodeAttributes) {
+ super(coreHistogram, coreAttributes);
+ assert coreHistogram != nodeHistogram;
+ this.nodeTimer = new AttributedLongTimer(nodeHistogram, nodeAttributes);
+ }
+
+ @Override
+ public void record(Long value) {
+ super.record(value);
+ nodeTimer.record(value);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java
new file mode 100644
index 00000000000..6cf505fb54e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.otel.instruments;
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.LongUpDownCounter;
+
+/**
+ * An AttributedLongUpDownCounter that writes to both core and node registries with corresponding
+ * attributes.
+ */
+public class DualRegistryAttributedLongUpDownCounter extends AttributedLongUpDownCounter {
+
+ private final AttributedLongUpDownCounter nodeUpDownCounter;
+
+ public DualRegistryAttributedLongUpDownCounter(
+ LongUpDownCounter coreUpDownCounter,
+ Attributes coreAttributes,
+ LongUpDownCounter nodeUpDownCounter,
+ Attributes nodeAttributes) {
+ super(coreUpDownCounter, coreAttributes);
+ assert coreUpDownCounter != nodeUpDownCounter;
+ this.nodeUpDownCounter = new AttributedLongUpDownCounter(nodeUpDownCounter, nodeAttributes);
+ }
+
+ @Override
+ public void add(Long value) {
+ super.add(value);
+ nodeUpDownCounter.add(value);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
index 0a027282c46..dc802afc2b7 100644
--- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
+++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
@@ -53,11 +53,9 @@ import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrConfig.UpdateHandlerInfo;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrInfoBean;
-import org.apache.solr.metrics.SolrDelegateRegistryMetricsContext;
-import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
+import org.apache.solr.metrics.otel.instruments.AttributedInstrumentFactory;
import org.apache.solr.metrics.otel.instruments.AttributedLongCounter;
import org.apache.solr.metrics.otel.instruments.AttributedLongUpDownCounter;
import org.apache.solr.request.LocalSolrQueryRequest;
@@ -227,84 +225,121 @@ public class DirectUpdateHandler2 extends UpdateHandler
@Override
public void initializeMetrics(
SolrMetricsContext parentContext, Attributes attributes, String scope) {
- if (core.getSolrConfig().getUpdateHandlerInfo().aggregateNodeLevelMetricsEnabled) {
- this.solrMetricsContext =
- new SolrDelegateRegistryMetricsContext(
- parentContext.getMetricManager(),
- parentContext.getRegistryName(),
- SolrMetricProducer.getUniqueMetricTag(this, parentContext.getTag()),
- SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));
- } else {
- this.solrMetricsContext = parentContext.getChildContext(this);
- }
- final List<AutoCloseable> observables = new ArrayList<>();
+ this.solrMetricsContext = parentContext.getChildContext(this);
var baseAttributes =
attributes.toBuilder()
.put(AttributeKey.stringKey("category"), getCategory().toString())
.build();
- var baseCommandsMetric =
- solrMetricsContext.longUpDownCounter(
- "solr_core_update_cumulative_ops",
- "Cumulative number of update commands processed. Cumulative can decrease from rollback command");
+ boolean aggregateNodeLevelMetricsEnabled =
+ core.getSolrConfig().getUpdateHandlerInfo().aggregateNodeLevelMetricsEnabled;
+
+ createMetrics(baseAttributes, aggregateNodeLevelMetricsEnabled);
+ }
+
+ private void createMetrics(Attributes baseAttributes, boolean aggregateToNodeRegistry) {
+ final List<AutoCloseable> observables = new ArrayList<>();
+
+ AttributedInstrumentFactory factory =
+ new AttributedInstrumentFactory(
+ solrMetricsContext, baseAttributes, aggregateToNodeRegistry);
addCommandsCumulative =
- new AttributedLongUpDownCounter(
- baseCommandsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build());
+ factory.attributedLongUpDownCounter(
+ "solr_core_update_cumulative_ops",
+ "Cumulative number of update commands processed. Cumulative can decrease from rollback command",
+ Attributes.of(OPERATION_ATTR, "adds"));
deleteByIdCommandsCumulative =
- new AttributedLongUpDownCounter(
- baseCommandsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_id").build());
+ factory.attributedLongUpDownCounter(
+ "solr_core_update_cumulative_ops",
+ "Cumulative number of update commands processed. Cumulative can decrease from rollback command",
+ Attributes.of(OPERATION_ATTR, "deletes_by_id"));
deleteByQueryCommandsCumulative =
- new AttributedLongUpDownCounter(
- baseCommandsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_query").build());
-
- var baseCommitOpsMetric =
- solrMetricsContext.longCounter(
- "solr_core_update_commit_ops", "Total number of commit operations");
+ factory.attributedLongUpDownCounter(
+ "solr_core_update_cumulative_ops",
+ "Cumulative number of update commands processed. Cumulative can decrease from rollback command",
+ Attributes.of(OPERATION_ATTR, "deletes_by_query"));
commitCommands =
- new AttributedLongCounter(
- baseCommitOpsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "commits").build());
+ factory.attributedLongCounter(
+ "solr_core_update_commit_ops",
+ "Total number of commit operations",
+ Attributes.of(OPERATION_ATTR, "commits"));
optimizeCommands =
- new AttributedLongCounter(
- baseCommitOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR, "optimize").build());
+ factory.attributedLongCounter(
+ "solr_core_update_commit_ops",
+ "Total number of commit operations",
+ Attributes.of(OPERATION_ATTR, "optimize"));
mergeIndexesCommands =
- new AttributedLongCounter(
- baseCommitOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR, "merge_indexes").build());
+ factory.attributedLongCounter(
+ "solr_core_update_commit_ops",
+ "Total number of commit operations",
+ Attributes.of(OPERATION_ATTR, "merge_indexes"));
expungeDeleteCommands =
- new AttributedLongCounter(
- baseCommitOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR, "expunge_deletes").build());
-
- var baseMaintenanceMetric =
- solrMetricsContext.longCounter(
- "solr_core_update_maintenance_ops", "Total number of maintenance operations");
+ factory.attributedLongCounter(
+ "solr_core_update_commit_ops",
+ "Total number of commit operations",
+ Attributes.of(OPERATION_ATTR, "expunge_deletes"));
rollbackCommands =
- new AttributedLongCounter(
- baseMaintenanceMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR, "rollback").build());
+ factory.attributedLongCounter(
+ "solr_core_update_maintenance_ops",
+ "Total number of maintenance operations",
+ Attributes.of(OPERATION_ATTR, "rollback"));
splitCommands =
- new AttributedLongCounter(
- baseMaintenanceMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "split").build());
-
- var baseErrorsMetric =
- solrMetricsContext.longCounter("solr_core_update_errors", "Total number of update errors");
+ factory.attributedLongCounter(
+ "solr_core_update_maintenance_ops",
+ "Total number of maintenance operations",
+ Attributes.of(OPERATION_ATTR, "split"));
numErrorsCumulative =
- new AttributedLongCounter(baseErrorsMetric, baseAttributes.toBuilder().build());
+ factory.attributedLongCounter(
+ "solr_core_update_errors", "Total number of update errors", Attributes.empty());
+
+ submittedAdds =
+ factory.attributedLongCounter(
+ "solr_core_update_submitted_ops",
+ "Total number of submitted update operations",
+ Attributes.of(OPERATION_ATTR, "adds"));
+
+ submittedDeleteById =
+ factory.attributedLongCounter(
+ "solr_core_update_submitted_ops",
+ "Total number of submitted update operations",
+ Attributes.of(OPERATION_ATTR, "deletes_by_id"));
+ submittedDeleteByQuery =
+ factory.attributedLongCounter(
+ "solr_core_update_submitted_ops",
+ "Total number of submitted update operations",
+ Attributes.of(OPERATION_ATTR, "deletes_by_query"));
+
+ committedAdds =
+ factory.attributedLongCounter(
+ "solr_core_update_committed_ops",
+ "Total number of committed update operations",
+ Attributes.of(OPERATION_ATTR, "adds"));
+
+ committedDeleteById =
+ factory.attributedLongCounter(
+ "solr_core_update_committed_ops",
+ "Total number of committed update operations",
+ Attributes.of(OPERATION_ATTR, "deletes_by_id"));
+
+ committedDeleteByQuery =
+ factory.attributedLongCounter(
+ "solr_core_update_committed_ops",
+ "Total number of committed update operations",
+ Attributes.of(OPERATION_ATTR, "deletes_by_query"));
+
+ // Create observable metrics only for core registry
observables.add(
solrMetricsContext.observableLongCounter(
"solr_core_update_auto_commits",
@@ -361,38 +396,6 @@ public class DirectUpdateHandler2 extends UpdateHandler
}));
this.toClose = Collections.unmodifiableList(observables);
-
- var baseSubmittedOpsMetric =
- solrMetricsContext.longCounter(
- "solr_core_update_submitted_ops", "Total number of submitted update operations");
-
- var baseCommittedOpsMetric =
- solrMetricsContext.longCounter(
- "solr_core_update_committed_ops", "Total number of committed update operations");
-
- submittedAdds =
- new AttributedLongCounter(
- baseSubmittedOpsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build());
- submittedDeleteById =
- new AttributedLongCounter(
- baseSubmittedOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_id").build());
- submittedDeleteByQuery =
- new AttributedLongCounter(
- baseSubmittedOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_query").build());
-
- committedAdds =
- new AttributedLongCounter(
- baseCommittedOpsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build());
- committedDeleteById =
- new AttributedLongCounter(
- baseCommittedOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_id").build());
- committedDeleteByQuery =
- new AttributedLongCounter(
- baseCommittedOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_query").build());
}
private void deleteAll() throws IOException {
diff --git a/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml
index f23456d060a..2b1586ae253 100644
--- a/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml
@@ -29,7 +29,7 @@
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
- <updateHandler class="solr.DirectUpdateHandler2">
+ <updateHandler class="solr.DirectUpdateHandler2" aggregateNodeLevelMetricsEnabled="true">
<commitWithin>
<softCommit>${solr.commitwithin.softcommit:true}</softCommit>
</commitWithin>
@@ -52,4 +52,4 @@
<indexConfig>
<mergeScheduler class="${solr.mscheduler:org.apache.lucene.index.ConcurrentMergeScheduler}"/>
</indexConfig>
-</config>
\ No newline at end of file
+</config>
diff --git a/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java b/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java
index 74d3d6b2510..2320b526b08 100644
--- a/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java
@@ -17,6 +17,7 @@
package org.apache.solr.handler;
+import static org.apache.solr.metrics.SolrMetricProducer.HANDLER_ATTR;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
@@ -196,6 +197,6 @@ public class RequestHandlerBaseTest extends SolrTestCaseJ4 {
when(metricsContext.longHistogram(any(), any())).thenReturn(mockLongHistogram);
return new RequestHandlerBase.HandlerMetrics(
- metricsContext, Attributes.of(AttributeKey.stringKey("/handler"), "/someBaseMetricPath"));
+ metricsContext, Attributes.of(HANDLER_ATTR, "/someBaseMetricPath"), false);
}
}
diff --git a/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java b/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java
index e9618297760..e338939b30c 100644
--- a/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java
@@ -16,19 +16,17 @@
*/
package org.apache.solr.handler;
+import io.prometheus.metrics.model.snapshots.CounterSnapshot;
+import io.prometheus.metrics.model.snapshots.Labels;
import java.io.IOException;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.util.SolrMetricTestUtils;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -54,10 +52,7 @@ public class RequestHandlerMetricsTest extends SolrCloudTestCase {
System.clearProperty("metricsEnabled");
}
- // TODO: Migrate aggregateNodeLevelMetricsEnabled for OTEL metrics
@Test
- @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17865")
- @SuppressWarnings({"unchecked"})
public void testAggregateNodeLevelMetrics() throws SolrServerException, IOException {
String collection1 = "testRequestHandlerMetrics1";
String collection2 = "testRequestHandlerMetrics2";
@@ -82,97 +77,83 @@ public class RequestHandlerMetricsTest extends SolrCloudTestCase {
cloudClient.query(collection1, solrQuery);
cloudClient.query(collection2, solrQuery);
- NamedList<Object> response =
- cloudClient.request(
- new GenericSolrRequest(
- SolrRequest.METHOD.GET, "/admin/metrics", SolrRequest.SolrRequestType.ADMIN));
-
- NamedList<Object> metrics = (NamedList<Object>) response.get("metrics");
-
- final double[] minQueryTime = {Double.MAX_VALUE};
- final double[] maxQueryTime = {-1.0};
- final double[] minUpdateTime = {Double.MAX_VALUE};
- final double[] maxUpdateTime = {-1.0};
- Set<NamedList<Object>> coreMetrics = new HashSet<>();
- metrics.forEach(
- (key, coreMetric) -> {
- if (key.startsWith("solr.core.testRequestHandlerMetrics")) {
- coreMetrics.add((NamedList<Object>) coreMetric);
- }
- });
- assertEquals(2, coreMetrics.size());
- coreMetrics.forEach(
- metric -> {
- assertEquals(
- 1L,
- ((Map<String, Number>) metric.get("QUERY./select.requestTimes"))
- .get("count")
- .longValue());
- minQueryTime[0] =
- Math.min(
- minQueryTime[0],
- ((Map<String, Number>) metric.get("QUERY./select.requestTimes"))
- .get("min_ms")
- .doubleValue());
- maxQueryTime[0] =
- Math.max(
- maxQueryTime[0],
- ((Map<String, Number>) metric.get("QUERY./select.requestTimes"))
- .get("max_ms")
- .doubleValue());
- assertEquals(
- 1L,
- ((Map<String, Number>) metric.get("UPDATE./update.requestTimes"))
- .get("count")
- .longValue());
- minUpdateTime[0] =
- Math.min(
- minUpdateTime[0],
- ((Map<String, Number>) metric.get("UPDATE./update.requestTimes"))
- .get("min_ms")
- .doubleValue());
- maxUpdateTime[0] =
- Math.max(
- maxUpdateTime[0],
- ((Map<String, Number>) metric.get("UPDATE./update.requestTimes"))
- .get("max_ms")
- .doubleValue());
- });
-
- NamedList<Object> nodeMetrics = (NamedList<Object>) metrics.get("solr.node");
- assertEquals(
- 2L,
- ((Map<String, Number>) nodeMetrics.get("QUERY./select.requestTimes"))
- .get("count")
- .longValue());
- assertEquals(
- minQueryTime[0],
- ((Map<String, Number>) nodeMetrics.get("QUERY./select.requestTimes"))
- .get("min_ms")
- .doubleValue(),
- 0.0);
- assertEquals(
- maxQueryTime[0],
- ((Map<String, Number>) nodeMetrics.get("QUERY./select.requestTimes"))
- .get("max_ms")
- .doubleValue(),
- 0.0);
- assertEquals(
- 2L,
- ((Map<String, Number>) nodeMetrics.get("UPDATE./update.requestTimes"))
- .get("count")
- .longValue());
- assertEquals(
- minUpdateTime[0],
- ((Map<String, Number>) nodeMetrics.get("UPDATE./update.requestTimes"))
- .get("min_ms")
- .doubleValue(),
- 0.0);
- assertEquals(
- maxUpdateTime[0],
- ((Map<String, Number>) nodeMetrics.get("UPDATE./update.requestTimes"))
- .get("max_ms")
- .doubleValue(),
- 0.0);
+ var coreContainer = cluster.getJettySolrRunners().get(0).getCoreContainer();
+
+ try (SolrCore core1 = coreContainer.getCore(coreContainer.getAllCoreNames().get(0));
+ SolrCore core2 = coreContainer.getCore(coreContainer.getAllCoreNames().get(1))) {
+
+ CounterSnapshot.CounterDataPointSnapshot actualCore1Selects =
+ SolrMetricTestUtils.newCloudSelectRequestsDatapoint(core1);
+ CounterSnapshot.CounterDataPointSnapshot actualCore1Updates =
+ SolrMetricTestUtils.newCloudUpdateRequestsDatapoint(core1);
+ CounterSnapshot.CounterDataPointSnapshot actualCore2Selects =
+ SolrMetricTestUtils.newCloudSelectRequestsDatapoint(core2);
+ CounterSnapshot.CounterDataPointSnapshot actualCore2Updates =
+ SolrMetricTestUtils.newCloudUpdateRequestsDatapoint(core2);
+ CounterSnapshot.CounterDataPointSnapshot actualCore1SubmittedOps =
+ SolrMetricTestUtils.getCounterDatapoint(
+ core1,
+ "solr_core_update_submitted_ops",
+ SolrMetricTestUtils.newCloudLabelsBuilder(core1)
+ .label("category", "UPDATE")
+ .label("ops", "adds")
+ .build());
+ CounterSnapshot.CounterDataPointSnapshot actualCore2SubmittedOps =
+ SolrMetricTestUtils.getCounterDatapoint(
+ core2,
+ "solr_core_update_submitted_ops",
+ SolrMetricTestUtils.newCloudLabelsBuilder(core2)
+ .label("category", "UPDATE")
+ .label("ops", "adds")
+ .build());
+
+ assertEquals(1.0, actualCore1Selects.getValue(), 0.0);
+ assertEquals(1.0, actualCore1Updates.getValue(), 0.0);
+ assertEquals(1.0, actualCore2Updates.getValue(), 0.0);
+ assertEquals(1.0, actualCore2Selects.getValue(), 0.0);
+ assertEquals(1.0, actualCore1SubmittedOps.getValue(), 0.0);
+ assertEquals(1.0, actualCore2SubmittedOps.getValue(), 0.0);
+
+ // Get node metrics and the select/update requests should be the sum of both cores requests
+ var nodeReader = SolrMetricTestUtils.getPrometheusMetricReader(coreContainer, "solr.node");
+
+ CounterSnapshot.CounterDataPointSnapshot nodeSelectRequests =
+ (CounterSnapshot.CounterDataPointSnapshot)
+ SolrMetricTestUtils.getDataPointSnapshot(
+ nodeReader,
+ "solr_node_requests",
+ Labels.builder()
+ .label("category", "QUERY")
+ .label("handler", "/select")
+ .label("otel_scope_name", "org.apache.solr")
+ .build());
+ CounterSnapshot.CounterDataPointSnapshot nodeUpdateRequests =
+ (CounterSnapshot.CounterDataPointSnapshot)
+ SolrMetricTestUtils.getDataPointSnapshot(
+ nodeReader,
+ "solr_node_requests",
+ Labels.builder()
+ .label("category", "UPDATE")
+ .label("handler", "/update")
+ .label("otel_scope_name", "org.apache.solr")
+ .build());
+ CounterSnapshot.CounterDataPointSnapshot nodeSubmittedOps =
+ (CounterSnapshot.CounterDataPointSnapshot)
+ SolrMetricTestUtils.getDataPointSnapshot(
+ nodeReader,
+ "solr_node_update_submitted_ops",
+ Labels.builder()
+ .label("category", "UPDATE")
+ .label("ops", "adds")
+ .label("otel_scope_name", "org.apache.solr")
+ .build());
+
+ assertNotNull("Node select requests should be recorded", nodeSelectRequests);
+ assertNotNull("Node update requests should be recorded", nodeUpdateRequests);
+ assertNotNull("Node submitted update operations should be recorded", nodeSubmittedOps);
+ assertEquals(2.0, nodeSelectRequests.getValue(), 0.0);
+ assertEquals(2.0, nodeUpdateRequests.getValue(), 0.0);
+ assertEquals(2.0, nodeSubmittedOps.getValue(), 0.0);
+ }
}
}