Apache Solr open-source search software
SOLR-16273: Prometheus Exporter is very slow when collecting large amounts of sample data (#1627)
Matthew Biscocho4362568f5fe0
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex c74f49c09b8..0d61771348b 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -183,6 +183,8 @@ Optimizationsrequest. SolrJ will now form the URL directly to the intended replica of the collection (and not the collection itself).(Pierre Salagnac, Ivan Djurasevic, Jason Baik)+* SOLR-16273: Improve the performance of the Prometheus Metric Exporter scraping metrics (Matthew Biscocho via David Smiley & Christine Poerschke+Bug Fixes---------------------diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/MetricSamples.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/MetricSamples.javaindex 9fae6ce23ed..328d76d054c 100644--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/MetricSamples.java+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/MetricSamples.java@@ -19,24 +19,35 @@ package org.apache.solr.prometheus.collector;import io.prometheus.client.Collector;import java.util.HashMap;+import java.util.HashSet;import java.util.List;import java.util.Map;+import java.util.Set;import java.util.stream.Collectors;public class MetricSamples {private final Map<String, Collector.MetricFamilySamples> samplesByMetricName;+ private final Set<Collector.MetricFamilySamples.Sample> seenSamples;+public MetricSamples(Map<String, Collector.MetricFamilySamples> input) {samplesByMetricName = input;+ seenSamples = new HashSet<>();+ for (Collector.MetricFamilySamples metricFamilySamples : input.values()) {+ seenSamples.addAll(metricFamilySamples.samples);+ }}public MetricSamples() {this(new HashMap<>());}- public void addSamplesIfNotPresent(String metricName, Collector.MetricFamilySamples samples) {- samplesByMetricName.putIfAbsent(metricName, samples);+ public void addSamplesIfNotPresent(+ String metricName, Collector.MetricFamilySamples metricFamilySamples) {+ if (samplesByMetricName.putIfAbsent(metricName, metricFamilySamples) == null) {+ seenSamples.addAll(metricFamilySamples.samples);+ }}public void addSampleIfMetricExists(@@ -47,7 +58,7 @@ public class MetricSamples {return;}- if (!sampleFamily.samples.contains(sample)) {+ if (seenSamples.add(sample)) {sampleFamily.samples.add(sample);}}@@ -62,6 +73,7 @@ public class MetricSamples {}} else {this.samplesByMetricName.put(key, entry.getValue());+ seenSamples.addAll(entry.getValue().samples);}}}diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/collector/MetricSamplesTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/collector/MetricSamplesTest.javaindex de2d754c8f6..8707e4fc7b3 100644--- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/collector/MetricSamplesTest.java+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/collector/MetricSamplesTest.java@@ -90,8 +90,13 @@ public class MetricSamplesTest extends SolrTestCase {Map.of("same",samples(- "test1", Collector.Type.GAUGE, sample("test1", 3.0), sample("test1", 4.0)),+ "test1",+ Collector.Type.GAUGE,+ sample("test1", 3.0),+ sample("test1", 4.0),+ sample("same", 1.0)),"diff2", samples("diff2", Collector.Type.GAUGE, sample("diff2", 1.0))));+lhs.addAll(rhs);List<Collector.MetricFamilySamples> output = lhs.asList();@@ -100,4 +105,29 @@ public class MetricSamplesTest extends SolrTestCase {validateMetricSamples(output, "diff1", Collections.singletonList(1.0));validateMetricSamples(output, "diff2", Collections.singletonList(1.0));}++ @Test+ public void addSamplesIfNotPresent() {++ MetricSamples testMetricSamples =+ new MetricSamples(+ new HashMap<>(+ Map.of("same", samples("same", Collector.Type.GAUGE, sample("same", 1.0)))));++ Collector.MetricFamilySamples sameSamples =+ samples("same", Collector.Type.GAUGE, sample("same", 1.0));+ Collector.MetricFamilySamples newSamples =+ samples("new", Collector.Type.GAUGE, sample("new", 2.0), sample("new", 3.0));+ Collector.MetricFamilySamples alreadyPresentSamples =+ samples("new", Collector.Type.GAUGE, sample("new", 4.0));++ testMetricSamples.addSamplesIfNotPresent("same", sameSamples);+ testMetricSamples.addSamplesIfNotPresent("new", newSamples);+ testMetricSamples.addSamplesIfNotPresent("new", alreadyPresentSamples);++ List<Collector.MetricFamilySamples> output = testMetricSamples.asList();++ validateMetricSamples(output, "same", Collections.singletonList(1.0));+ validateMetricSamples(output, "new", Arrays.asList(2.0, 3.0));+ }}