Apache Solr open-source search software
SOLR-18442: Fix OpenTelemetry Observable gauge memory leak
Chris Hostetter136d1bd1f22e
diff --git a/changelog/unreleased/SOLR-18442-observable-gauge-leak.yml b/changelog/unreleased/SOLR-18442-observable-gauge-leak.ymlnew file mode 100644index 00000000000..1479c33371b--- /dev/null+++ b/changelog/unreleased/SOLR-18442-observable-gauge-leak.yml@@ -0,0 +1,13 @@+title: >+ Fix a memory leak where every SolrIndexSearcher ever opened was retained by the OpenTelemetry+ metric registry via its "index.commit_size" observable gauge, along with its DirectoryReader and+ segment live-docs bitsets. Nodes with a high commit cadence would eventually run out of heap.+ Observable metrics registered through SolrMetricsContext with an explicit unit are now closed+ together with the context.+type: fixed+authors:+ - name: Jan Høydahl+ - name: Mikhail Khludnev+links:+ - name: SOLR-18442+ url: https://issues.apache.org/jira/browse/SOLR-18442diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.javaindex 720bbeb301d..45d7c591ef9 100644--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java@@ -160,9 +160,7 @@ public class SolrMetricsContext implements AutoCloseable {public ObservableLongGauge observableLongGauge(String metricName, String description, Consumer<ObservableLongMeasurement> callback) {- var observableLongGauge = observableLongGauge(metricName, description, callback, null);- closeables.add(observableLongGauge);- return observableLongGauge;+ return observableLongGauge(metricName, description, callback, null);}public ObservableLongGauge observableLongGauge(@@ -170,14 +168,15 @@ public class SolrMetricsContext implements AutoCloseable {String description,Consumer<ObservableLongMeasurement> callback,OtelUnit unit) {- return metricManager.observableLongGauge(registryName, metricName, description, callback, unit);+ var observableLongGauge =+ metricManager.observableLongGauge(registryName, metricName, description, callback, unit);+ closeables.add(observableLongGauge);+ return observableLongGauge;}public ObservableDoubleGauge observableDoubleGauge(String metricName, String description, Consumer<ObservableDoubleMeasurement> callback) {- var observableDoubleGauge = observableDoubleGauge(metricName, description, callback, null);- closeables.add(observableDoubleGauge);- return observableDoubleGauge;+ return observableDoubleGauge(metricName, description, callback, null);}public ObservableDoubleGauge observableDoubleGauge(@@ -185,15 +184,15 @@ public class SolrMetricsContext implements AutoCloseable {String description,Consumer<ObservableDoubleMeasurement> callback,OtelUnit unit) {- return metricManager.observableDoubleGauge(- registryName, metricName, description, callback, unit);+ var observableDoubleGauge =+ metricManager.observableDoubleGauge(registryName, metricName, description, callback, unit);+ closeables.add(observableDoubleGauge);+ return observableDoubleGauge;}public ObservableLongCounter observableLongCounter(String metricName, String description, Consumer<ObservableLongMeasurement> callback) {- var observableLongCounter = observableLongCounter(metricName, description, callback, null);- closeables.add(observableLongCounter);- return observableLongCounter;+ return observableLongCounter(metricName, description, callback, null);}public ObservableLongCounter observableLongCounter(@@ -201,15 +200,15 @@ public class SolrMetricsContext implements AutoCloseable {String description,Consumer<ObservableLongMeasurement> callback,OtelUnit unit) {- return metricManager.observableLongCounter(- registryName, metricName, description, callback, unit);+ var observableLongCounter =+ metricManager.observableLongCounter(registryName, metricName, description, callback, unit);+ closeables.add(observableLongCounter);+ return observableLongCounter;}public ObservableDoubleCounter observableDoubleCounter(String metricName, String description, Consumer<ObservableDoubleMeasurement> callback) {- var observableDoubleCounter = observableDoubleCounter(metricName, description, callback, null);- closeables.add(observableDoubleCounter);- return observableDoubleCounter;+ return observableDoubleCounter(metricName, description, callback, null);}public ObservableDoubleCounter observableDoubleCounter(@@ -217,8 +216,11 @@ public class SolrMetricsContext implements AutoCloseable {String description,Consumer<ObservableDoubleMeasurement> callback,OtelUnit unit) {- return metricManager.observableDoubleCounter(- registryName, metricName, description, callback, unit);+ var observableDoubleCounter =+ metricManager.observableDoubleCounter(+ registryName, metricName, description, callback, unit);+ closeables.add(observableDoubleCounter);+ return observableDoubleCounter;}public ObservableLongMeasurement longGaugeMeasurement(String metricName, String description) {diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrMetricsContextTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsContextTest.javanew file mode 100644index 00000000000..048962f7a80--- /dev/null+++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsContextTest.java@@ -0,0 +1,77 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.metrics;++import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;+import io.opentelemetry.sdk.testing.exporter.InMemoryMetricExporter;+import java.util.concurrent.atomic.AtomicInteger;+import org.apache.solr.SolrTestCase;+import org.apache.solr.metrics.otel.OtelUnit;+import org.junit.After;+import org.junit.Before;+import org.junit.Test;++public class SolrMetricsContextTest extends SolrTestCase {+ private static final String REGISTRY = "test_context_registry";+ private SolrMetricManager metricManager;+ private PrometheusMetricReader reader;++ @Before+ @Override+ public void setUp() throws Exception {+ super.setUp();+ metricManager = new SolrMetricManager(InMemoryMetricExporter.create());+ metricManager.meterProvider(REGISTRY);+ reader = metricManager.getPrometheusMetricReader(REGISTRY);+ }++ @After+ @Override+ public void tearDown() throws Exception {+ metricManager.closeAllRegistries();+ super.tearDown();+ }++ /** Callbacks registered through a context must stop firing once the context is closed. */+ @Test+ public void testCloseUnregistersObservableCallbacks() {+ SolrMetricsContext ctx = new SolrMetricsContext(metricManager, REGISTRY);+ AtomicInteger invocations = new AtomicInteger();++ ctx.observableLongGauge("long_gauge", "d", m -> invocations.incrementAndGet());+ ctx.observableLongGauge(+ "long_gauge_unit", "d", m -> invocations.incrementAndGet(), OtelUnit.BYTES);+ ctx.observableDoubleGauge("double_gauge", "d", m -> invocations.incrementAndGet());+ ctx.observableDoubleGauge(+ "double_gauge_unit", "d", m -> invocations.incrementAndGet(), OtelUnit.BYTES);+ ctx.observableLongCounter("long_counter", "d", m -> invocations.incrementAndGet());+ ctx.observableLongCounter(+ "long_counter_unit", "d", m -> invocations.incrementAndGet(), OtelUnit.BYTES);+ ctx.observableDoubleCounter("double_counter", "d", m -> invocations.incrementAndGet());+ ctx.observableDoubleCounter(+ "double_counter_unit", "d", m -> invocations.incrementAndGet(), OtelUnit.BYTES);++ reader.collect();+ assertEquals("all 8 callbacks should fire while the context is open", 8, invocations.get());++ ctx.close();+ invocations.set(0);+ reader.collect();+ assertEquals("no callback should fire after the context is closed", 0, invocations.get());+ }+}