Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-18442: Fix OpenTelemetry Observable gauge memory leak

Chris Hostetter136d1bd1f22e
diff --git a/changelog/unreleased/SOLR-18442-observable-gauge-leak.yml b/changelog/unreleased/SOLR-18442-observable-gauge-leak.yml
new file mode 100644
index 00000000000..1479c33371b
--- /dev/null
+++ b/changelog/unreleased/SOLR-18442-observable-gauge-leak.yml
@@ -0,0 +1,13 @@
+title: >
+ Fix a memory leak where every SolrIndexSearcher ever opened was retained by the OpenTelemetry
+ metric registry via its "index.commit_size" observable gauge, along with its DirectoryReader and
+ segment live-docs bitsets. Nodes with a high commit cadence would eventually run out of heap.
+ Observable metrics registered through SolrMetricsContext with an explicit unit are now closed
+ together with the context.
+type: fixed
+authors:
+ - name: Jan Høydahl
+ - name: Mikhail Khludnev
+links:
+ - name: SOLR-18442
+ url: https://issues.apache.org/jira/browse/SOLR-18442
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java
index 720bbeb301d..45d7c591ef9 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java
@@ -160,9 +160,7 @@ public class SolrMetricsContext implements AutoCloseable {
public ObservableLongGauge observableLongGauge(
String metricName, String description, Consumer<ObservableLongMeasurement> callback) {
- var observableLongGauge = observableLongGauge(metricName, description, callback, null);
- closeables.add(observableLongGauge);
- return observableLongGauge;
+ return observableLongGauge(metricName, description, callback, null);
}
public ObservableLongGauge observableLongGauge(
@@ -170,14 +168,15 @@ public class SolrMetricsContext implements AutoCloseable {
String description,
Consumer<ObservableLongMeasurement> callback,
OtelUnit unit) {
- return metricManager.observableLongGauge(registryName, metricName, description, callback, unit);
+ var observableLongGauge =
+ metricManager.observableLongGauge(registryName, metricName, description, callback, unit);
+ closeables.add(observableLongGauge);
+ return observableLongGauge;
}
public ObservableDoubleGauge observableDoubleGauge(
String metricName, String description, Consumer<ObservableDoubleMeasurement> callback) {
- var observableDoubleGauge = observableDoubleGauge(metricName, description, callback, null);
- closeables.add(observableDoubleGauge);
- return observableDoubleGauge;
+ return observableDoubleGauge(metricName, description, callback, null);
}
public ObservableDoubleGauge observableDoubleGauge(
@@ -185,15 +184,15 @@ public class SolrMetricsContext implements AutoCloseable {
String description,
Consumer<ObservableDoubleMeasurement> callback,
OtelUnit unit) {
- return metricManager.observableDoubleGauge(
- registryName, metricName, description, callback, unit);
+ var observableDoubleGauge =
+ metricManager.observableDoubleGauge(registryName, metricName, description, callback, unit);
+ closeables.add(observableDoubleGauge);
+ return observableDoubleGauge;
}
public ObservableLongCounter observableLongCounter(
String metricName, String description, Consumer<ObservableLongMeasurement> callback) {
- var observableLongCounter = observableLongCounter(metricName, description, callback, null);
- closeables.add(observableLongCounter);
- return observableLongCounter;
+ return observableLongCounter(metricName, description, callback, null);
}
public ObservableLongCounter observableLongCounter(
@@ -201,15 +200,15 @@ public class SolrMetricsContext implements AutoCloseable {
String description,
Consumer<ObservableLongMeasurement> callback,
OtelUnit unit) {
- return metricManager.observableLongCounter(
- registryName, metricName, description, callback, unit);
+ var observableLongCounter =
+ metricManager.observableLongCounter(registryName, metricName, description, callback, unit);
+ closeables.add(observableLongCounter);
+ return observableLongCounter;
}
public ObservableDoubleCounter observableDoubleCounter(
String metricName, String description, Consumer<ObservableDoubleMeasurement> callback) {
- var observableDoubleCounter = observableDoubleCounter(metricName, description, callback, null);
- closeables.add(observableDoubleCounter);
- return observableDoubleCounter;
+ return observableDoubleCounter(metricName, description, callback, null);
}
public ObservableDoubleCounter observableDoubleCounter(
@@ -217,8 +216,11 @@ public class SolrMetricsContext implements AutoCloseable {
String description,
Consumer<ObservableDoubleMeasurement> callback,
OtelUnit unit) {
- return metricManager.observableDoubleCounter(
- registryName, metricName, description, callback, unit);
+ var observableDoubleCounter =
+ metricManager.observableDoubleCounter(
+ registryName, metricName, description, callback, unit);
+ closeables.add(observableDoubleCounter);
+ return observableDoubleCounter;
}
public ObservableLongMeasurement longGaugeMeasurement(String metricName, String description) {
diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrMetricsContextTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsContextTest.java
new file mode 100644
index 00000000000..048962f7a80
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsContextTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.metrics;
+
+import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;
+import io.opentelemetry.sdk.testing.exporter.InMemoryMetricExporter;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.metrics.otel.OtelUnit;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SolrMetricsContextTest extends SolrTestCase {
+ private static final String REGISTRY = "test_context_registry";
+ private SolrMetricManager metricManager;
+ private PrometheusMetricReader reader;
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ metricManager = new SolrMetricManager(InMemoryMetricExporter.create());
+ metricManager.meterProvider(REGISTRY);
+ reader = metricManager.getPrometheusMetricReader(REGISTRY);
+ }
+
+ @After
+ @Override
+ public void tearDown() throws Exception {
+ metricManager.closeAllRegistries();
+ super.tearDown();
+ }
+
+ /** Callbacks registered through a context must stop firing once the context is closed. */
+ @Test
+ public void testCloseUnregistersObservableCallbacks() {
+ SolrMetricsContext ctx = new SolrMetricsContext(metricManager, REGISTRY);
+ AtomicInteger invocations = new AtomicInteger();
+
+ ctx.observableLongGauge("long_gauge", "d", m -> invocations.incrementAndGet());
+ ctx.observableLongGauge(
+ "long_gauge_unit", "d", m -> invocations.incrementAndGet(), OtelUnit.BYTES);
+ ctx.observableDoubleGauge("double_gauge", "d", m -> invocations.incrementAndGet());
+ ctx.observableDoubleGauge(
+ "double_gauge_unit", "d", m -> invocations.incrementAndGet(), OtelUnit.BYTES);
+ ctx.observableLongCounter("long_counter", "d", m -> invocations.incrementAndGet());
+ ctx.observableLongCounter(
+ "long_counter_unit", "d", m -> invocations.incrementAndGet(), OtelUnit.BYTES);
+ ctx.observableDoubleCounter("double_counter", "d", m -> invocations.incrementAndGet());
+ ctx.observableDoubleCounter(
+ "double_counter_unit", "d", m -> invocations.incrementAndGet(), OtelUnit.BYTES);
+
+ reader.collect();
+ assertEquals("all 8 callbacks should fire while the context is open", 8, invocations.get());
+
+ ctx.close();
+ invocations.set(0);
+ reader.collect();
+ assertEquals("no callback should fire after the context is closed", 0, invocations.get());
+ }
+}