Skip to content

Apache Lucene open-source search software

Add DistinctValuesCollectorManager to enable concurrent search (#16217)

Luca Cavanna737382fc5dd3
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index dcf582e93cc..97b76b3e982 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -400,6 +400,8 @@ Improvements
* GITHUB#16229: BooleanWeight.explain prints failing optional clauses when query fails due to insufficient SHOULD matches. (Jakub Slowinski)
+* GITHUB#16217: Introduce DistinctValuesCollectorManager to parallelize search when using DistinctValuesCollector. (Luca Cavanna)
+
Optimizations
---------------------
* GITHUB#16222: MultiTermQuery constant-score wrapper now defers term collection to ScorerSupplier#get()
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/DistinctValuesCollectorManager.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DistinctValuesCollectorManager.java
new file mode 100644
index 00000000000..7d13335cee9
--- /dev/null
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DistinctValuesCollectorManager.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.grouping;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Supplier;
+import org.apache.lucene.search.CollectorManager;
+
+/**
+ * A {@link CollectorManager} implementation for {@link DistinctValuesCollector} that supports
+ * parallel collection and merges results by taking the union of distinct values per group across
+ * segments.
+ *
+ * @lucene.experimental
+ */
+public class DistinctValuesCollectorManager<T, R>
+ implements CollectorManager<
+ DistinctValuesCollector<T, R>, List<DistinctValuesCollector.GroupCount<T, R>>> {
+
+ private final Supplier<GroupSelector<T>> groupSelectorFactory;
+ private final Collection<SearchGroup<T>> searchGroups;
+ private final Supplier<GroupSelector<R>> valueSelectorFactory;
+
+ /**
+ * Creates a new DistinctValuesCollectorManager.
+ *
+ * @param groupSelectorFactory factory to create group selectors for each collector
+ * @param searchGroups the search groups from the first pass
+ * @param valueSelectorFactory factory to create value selectors for each collector
+ */
+ public DistinctValuesCollectorManager(
+ Supplier<GroupSelector<T>> groupSelectorFactory,
+ Collection<SearchGroup<T>> searchGroups,
+ Supplier<GroupSelector<R>> valueSelectorFactory) {
+ this.groupSelectorFactory = groupSelectorFactory;
+ this.searchGroups = searchGroups;
+ this.valueSelectorFactory = valueSelectorFactory;
+ }
+
+ @Override
+ public DistinctValuesCollector<T, R> newCollector() throws IOException {
+ return new DistinctValuesCollector<>(
+ groupSelectorFactory.get(), searchGroups, valueSelectorFactory.get());
+ }
+
+ @Override
+ public List<DistinctValuesCollector.GroupCount<T, R>> reduce(
+ Collection<DistinctValuesCollector<T, R>> collectors) {
+ List<List<DistinctValuesCollector.GroupCount<T, R>>> allGroupsList = new ArrayList<>();
+ for (DistinctValuesCollector<T, R> collector : collectors) {
+ allGroupsList.add(collector.getGroups());
+ }
+ if (allGroupsList.isEmpty()) {
+ return List.of();
+ }
+ // Merge by taking the union of uniqueValues for each group across all collectors.
+ // All collectors share the same searchGroups so position j in each list is the same group.
+ List<DistinctValuesCollector.GroupCount<T, R>> first = allGroupsList.get(0);
+ List<DistinctValuesCollector.GroupCount<T, R>> merged = new ArrayList<>(first.size());
+ for (int j = 0; j < first.size(); j++) {
+ Set<R> union = new HashSet<>(first.get(j).uniqueValues);
+ for (int i = 1; i < allGroupsList.size(); i++) {
+ assert allGroupsList.get(i).size() == first.size();
+ union.addAll(allGroupsList.get(i).get(j).uniqueValues);
+ }
+ merged.add(new DistinctValuesCollector.GroupCount<>(first.get(j).groupValue, union));
+ }
+ return merged;
+ }
+}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java
index ffa2cef0ee9..bbc64239a7b 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java
@@ -16,7 +16,6 @@
*/
package org.apache.lucene.search.grouping;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -29,6 +28,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.function.Supplier;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
@@ -134,16 +134,22 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {
}
};
- // === Search for content:random
- FirstPassGroupingCollector<Comparable<Object>> firstCollector =
- createRandomFirstPassCollector(new Sort(), GROUP_FIELD, 10);
- indexSearcher.search(new TermQuery(new Term("content", "random")), firstCollector);
- DistinctValuesCollector<Comparable<Object>, Comparable<Object>> distinctValuesCollector =
- createDistinctCountCollector(firstCollector, COUNT_FIELD);
- indexSearcher.search(new TermQuery(new Term("content", "random")), distinctValuesCollector);
+ boolean useValueSource = random.nextBoolean();
+ Supplier<GroupSelector<Comparable<Object>>> groupSelectorFactory =
+ createGroupSelectorFactory(GROUP_FIELD, useValueSource);
+ Supplier<GroupSelector<Comparable<Object>>> valueSelectorFactory =
+ createGroupSelectorFactory(COUNT_FIELD, useValueSource);
+ // === Search for content:random
+ Collection<SearchGroup<Comparable<Object>>> searchGroups =
+ indexSearcher.search(
+ new TermQuery(new Term("content", "random")),
+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, new Sort(), 0, 10));
List<DistinctValuesCollector.GroupCount<Comparable<Object>, Comparable<Object>>> gcs =
- distinctValuesCollector.getGroups();
+ indexSearcher.search(
+ new TermQuery(new Term("content", "random")),
+ new DistinctValuesCollectorManager<>(
+ groupSelectorFactory, searchGroups, valueSelectorFactory));
Collections.sort(gcs, cmp);
assertEquals(4, gcs.size());
@@ -170,12 +176,15 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {
compare("1", countValues.get(0));
// === Search for content:some
- firstCollector = createRandomFirstPassCollector(new Sort(), GROUP_FIELD, 10);
- indexSearcher.search(new TermQuery(new Term("content", "some")), firstCollector);
- distinctValuesCollector = createDistinctCountCollector(firstCollector, COUNT_FIELD);
- indexSearcher.search(new TermQuery(new Term("content", "some")), distinctValuesCollector);
-
- gcs = distinctValuesCollector.getGroups();
+ searchGroups =
+ indexSearcher.search(
+ new TermQuery(new Term("content", "some")),
+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, new Sort(), 0, 10));
+ gcs =
+ indexSearcher.search(
+ new TermQuery(new Term("content", "some")),
+ new DistinctValuesCollectorManager<>(
+ groupSelectorFactory, searchGroups, valueSelectorFactory));
Collections.sort(gcs, cmp);
assertEquals(3, gcs.size());
@@ -197,12 +206,15 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {
compare("1", countValues.get(0));
// === Search for content:blob
- firstCollector = createRandomFirstPassCollector(new Sort(), GROUP_FIELD, 10);
- indexSearcher.search(new TermQuery(new Term("content", "blob")), firstCollector);
- distinctValuesCollector = createDistinctCountCollector(firstCollector, COUNT_FIELD);
- indexSearcher.search(new TermQuery(new Term("content", "blob")), distinctValuesCollector);
-
- gcs = distinctValuesCollector.getGroups();
+ searchGroups =
+ indexSearcher.search(
+ new TermQuery(new Term("content", "blob")),
+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, new Sort(), 0, 10));
+ gcs =
+ indexSearcher.search(
+ new TermQuery(new Term("content", "blob")),
+ new DistinctValuesCollectorManager<>(
+ groupSelectorFactory, searchGroups, valueSelectorFactory));
Collections.sort(gcs, cmp);
assertEquals(2, gcs.size());
@@ -235,25 +247,36 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {
List<DistinctValuesCollector.GroupCount<Comparable<Object>, Comparable<Object>>>
expectedResult = createExpectedResult(context, term, groupSort, topN);
- FirstPassGroupingCollector<Comparable<Object>> firstCollector =
- createRandomFirstPassCollector(groupSort, GROUP_FIELD, topN);
- searcher.search(new TermQuery(new Term("content", term)), firstCollector);
- DistinctValuesCollector<Comparable<Object>, Comparable<Object>> distinctValuesCollector =
- createDistinctCountCollector(firstCollector, COUNT_FIELD);
- searcher.search(new TermQuery(new Term("content", term)), distinctValuesCollector);
+ boolean useValueSource = random.nextBoolean();
+ Supplier<GroupSelector<Comparable<Object>>> groupSelectorFactory =
+ createGroupSelectorFactory(GROUP_FIELD, useValueSource);
+ Supplier<GroupSelector<Comparable<Object>>> valueSelectorFactory =
+ createGroupSelectorFactory(COUNT_FIELD, useValueSource);
+
+ Collection<SearchGroup<Comparable<Object>>> searchGroups =
+ searcher.search(
+ new TermQuery(new Term("content", term)),
+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, groupSort, 0, topN));
+
+ if (searchGroups == null || searchGroups.isEmpty()) {
+ assertEquals(0, expectedResult.size());
+ continue;
+ }
+
@SuppressWarnings("unchecked")
List<DistinctValuesCollector.GroupCount<Comparable<Object>, Comparable<Object>>>
- actualResult = distinctValuesCollector.getGroups();
+ actualResult =
+ searcher.search(
+ new TermQuery(new Term("content", term)),
+ new DistinctValuesCollectorManager<>(
+ groupSelectorFactory, searchGroups, valueSelectorFactory));
if (VERBOSE) {
System.out.println("Index iter=" + indexIter);
System.out.println("Search iter=" + searchIter);
- System.out.println(
- "1st pass collector class name=" + firstCollector.getClass().getName());
- System.out.println(
- "2nd pass collector class name=" + distinctValuesCollector.getClass().getName());
+ System.out.println("useValueSource=" + useValueSource);
System.out.println("Search term=" + term);
- System.out.println("1st pass groups=" + firstCollector.getTopGroups(0));
+ System.out.println("1st pass groups=" + searchGroups);
System.out.println("Expected:");
printGroups(expectedResult);
System.out.println("Actual:");
@@ -350,37 +373,15 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {
doc.add(new SortedDocValuesField(field, new BytesRef(value)));
}
- @SuppressWarnings({"unchecked", "rawtypes"})
- private <T extends Comparable<Object>, R extends Comparable<Object>>
- DistinctValuesCollector<T, R> createDistinctCountCollector(
- FirstPassGroupingCollector<T> firstPassGroupingCollector, String countField)
- throws IOException {
- Collection<SearchGroup<T>> searchGroups = firstPassGroupingCollector.getTopGroups(0);
- GroupSelector<T> selector = firstPassGroupingCollector.getGroupSelector();
- if (ValueSourceGroupSelector.class.isAssignableFrom(selector.getClass())) {
- GroupSelector gs =
- new ValueSourceGroupSelector(new BytesRefFieldSource(countField), new HashMap<>());
- return new DistinctValuesCollector<>(selector, searchGroups, gs);
- } else {
- GroupSelector ts = new TermGroupSelector(countField);
- return new DistinctValuesCollector<>(selector, searchGroups, ts);
- }
- }
-
- @SuppressWarnings({"unchecked", "rawtypes"})
- private <T> FirstPassGroupingCollector<T> createRandomFirstPassCollector(
- Sort groupSort, String groupField, int topNGroups) throws IOException {
- Random random = random();
- if (random.nextBoolean()) {
- return (FirstPassGroupingCollector<T>)
- new FirstPassGroupingCollector<>(
- new ValueSourceGroupSelector(new BytesRefFieldSource(groupField), new HashMap<>()),
- groupSort,
- topNGroups);
+ @SuppressWarnings({"unchecked"})
+ private static <T> Supplier<GroupSelector<T>> createGroupSelectorFactory(
+ String field, boolean useValueSource) {
+ if (useValueSource) {
+ return () ->
+ (GroupSelector<T>)
+ new ValueSourceGroupSelector(new BytesRefFieldSource(field), new HashMap<>());
} else {
- return (FirstPassGroupingCollector<T>)
- new FirstPassGroupingCollector<>(
- new TermGroupSelector(groupField), groupSort, topNGroups);
+ return () -> (GroupSelector<T>) new TermGroupSelector(field);
}
}