Apache Lucene open-source search software
Add DistinctValuesCollectorManager to enable concurrent search (#16217)
Luca Cavanna737382fc5dd3
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex dcf582e93cc..97b76b3e982 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -400,6 +400,8 @@ Improvements* GITHUB#16229: BooleanWeight.explain prints failing optional clauses when query fails due to insufficient SHOULD matches. (Jakub Slowinski)+* GITHUB#16217: Introduce DistinctValuesCollectorManager to parallelize search when using DistinctValuesCollector. (Luca Cavanna)+Optimizations---------------------* GITHUB#16222: MultiTermQuery constant-score wrapper now defers term collection to ScorerSupplier#get()diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/DistinctValuesCollectorManager.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DistinctValuesCollectorManager.javanew file mode 100644index 00000000000..7d13335cee9--- /dev/null+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DistinctValuesCollectorManager.java@@ -0,0 +1,89 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search.grouping;++import java.io.IOException;+import java.util.ArrayList;+import java.util.Collection;+import java.util.HashSet;+import java.util.List;+import java.util.Set;+import java.util.function.Supplier;+import org.apache.lucene.search.CollectorManager;++/**+ * A {@link CollectorManager} implementation for {@link DistinctValuesCollector} that supports+ * parallel collection and merges results by taking the union of distinct values per group across+ * segments.+ *+ * @lucene.experimental+ */+public class DistinctValuesCollectorManager<T, R>+ implements CollectorManager<+ DistinctValuesCollector<T, R>, List<DistinctValuesCollector.GroupCount<T, R>>> {++ private final Supplier<GroupSelector<T>> groupSelectorFactory;+ private final Collection<SearchGroup<T>> searchGroups;+ private final Supplier<GroupSelector<R>> valueSelectorFactory;++ /**+ * Creates a new DistinctValuesCollectorManager.+ *+ * @param groupSelectorFactory factory to create group selectors for each collector+ * @param searchGroups the search groups from the first pass+ * @param valueSelectorFactory factory to create value selectors for each collector+ */+ public DistinctValuesCollectorManager(+ Supplier<GroupSelector<T>> groupSelectorFactory,+ Collection<SearchGroup<T>> searchGroups,+ Supplier<GroupSelector<R>> valueSelectorFactory) {+ this.groupSelectorFactory = groupSelectorFactory;+ this.searchGroups = searchGroups;+ this.valueSelectorFactory = valueSelectorFactory;+ }++ @Override+ public DistinctValuesCollector<T, R> newCollector() throws IOException {+ return new DistinctValuesCollector<>(+ groupSelectorFactory.get(), searchGroups, valueSelectorFactory.get());+ }++ @Override+ public List<DistinctValuesCollector.GroupCount<T, R>> reduce(+ Collection<DistinctValuesCollector<T, R>> collectors) {+ List<List<DistinctValuesCollector.GroupCount<T, R>>> allGroupsList = new ArrayList<>();+ for (DistinctValuesCollector<T, R> collector : collectors) {+ allGroupsList.add(collector.getGroups());+ }+ if (allGroupsList.isEmpty()) {+ return List.of();+ }+ // Merge by taking the union of uniqueValues for each group across all collectors.+ // All collectors share the same searchGroups so position j in each list is the same group.+ List<DistinctValuesCollector.GroupCount<T, R>> first = allGroupsList.get(0);+ List<DistinctValuesCollector.GroupCount<T, R>> merged = new ArrayList<>(first.size());+ for (int j = 0; j < first.size(); j++) {+ Set<R> union = new HashSet<>(first.get(j).uniqueValues);+ for (int i = 1; i < allGroupsList.size(); i++) {+ assert allGroupsList.get(i).size() == first.size();+ union.addAll(allGroupsList.get(i).get(j).uniqueValues);+ }+ merged.add(new DistinctValuesCollector.GroupCount<>(first.get(j).groupValue, union));+ }+ return merged;+ }+}diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.javaindex ffa2cef0ee9..bbc64239a7b 100644--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java@@ -16,7 +16,6 @@*/package org.apache.lucene.search.grouping;-import java.io.IOException;import java.util.ArrayList;import java.util.Collection;import java.util.Collections;@@ -29,6 +28,7 @@ import java.util.Locale;import java.util.Map;import java.util.Random;import java.util.Set;+import java.util.function.Supplier;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.SortedDocValuesField;@@ -134,16 +134,22 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {}};- // === Search for content:random- FirstPassGroupingCollector<Comparable<Object>> firstCollector =- createRandomFirstPassCollector(new Sort(), GROUP_FIELD, 10);- indexSearcher.search(new TermQuery(new Term("content", "random")), firstCollector);- DistinctValuesCollector<Comparable<Object>, Comparable<Object>> distinctValuesCollector =- createDistinctCountCollector(firstCollector, COUNT_FIELD);- indexSearcher.search(new TermQuery(new Term("content", "random")), distinctValuesCollector);+ boolean useValueSource = random.nextBoolean();+ Supplier<GroupSelector<Comparable<Object>>> groupSelectorFactory =+ createGroupSelectorFactory(GROUP_FIELD, useValueSource);+ Supplier<GroupSelector<Comparable<Object>>> valueSelectorFactory =+ createGroupSelectorFactory(COUNT_FIELD, useValueSource);+ // === Search for content:random+ Collection<SearchGroup<Comparable<Object>>> searchGroups =+ indexSearcher.search(+ new TermQuery(new Term("content", "random")),+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, new Sort(), 0, 10));List<DistinctValuesCollector.GroupCount<Comparable<Object>, Comparable<Object>>> gcs =- distinctValuesCollector.getGroups();+ indexSearcher.search(+ new TermQuery(new Term("content", "random")),+ new DistinctValuesCollectorManager<>(+ groupSelectorFactory, searchGroups, valueSelectorFactory));Collections.sort(gcs, cmp);assertEquals(4, gcs.size());@@ -170,12 +176,15 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {compare("1", countValues.get(0));// === Search for content:some- firstCollector = createRandomFirstPassCollector(new Sort(), GROUP_FIELD, 10);- indexSearcher.search(new TermQuery(new Term("content", "some")), firstCollector);- distinctValuesCollector = createDistinctCountCollector(firstCollector, COUNT_FIELD);- indexSearcher.search(new TermQuery(new Term("content", "some")), distinctValuesCollector);-- gcs = distinctValuesCollector.getGroups();+ searchGroups =+ indexSearcher.search(+ new TermQuery(new Term("content", "some")),+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, new Sort(), 0, 10));+ gcs =+ indexSearcher.search(+ new TermQuery(new Term("content", "some")),+ new DistinctValuesCollectorManager<>(+ groupSelectorFactory, searchGroups, valueSelectorFactory));Collections.sort(gcs, cmp);assertEquals(3, gcs.size());@@ -197,12 +206,15 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {compare("1", countValues.get(0));// === Search for content:blob- firstCollector = createRandomFirstPassCollector(new Sort(), GROUP_FIELD, 10);- indexSearcher.search(new TermQuery(new Term("content", "blob")), firstCollector);- distinctValuesCollector = createDistinctCountCollector(firstCollector, COUNT_FIELD);- indexSearcher.search(new TermQuery(new Term("content", "blob")), distinctValuesCollector);-- gcs = distinctValuesCollector.getGroups();+ searchGroups =+ indexSearcher.search(+ new TermQuery(new Term("content", "blob")),+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, new Sort(), 0, 10));+ gcs =+ indexSearcher.search(+ new TermQuery(new Term("content", "blob")),+ new DistinctValuesCollectorManager<>(+ groupSelectorFactory, searchGroups, valueSelectorFactory));Collections.sort(gcs, cmp);assertEquals(2, gcs.size());@@ -235,25 +247,36 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {List<DistinctValuesCollector.GroupCount<Comparable<Object>, Comparable<Object>>>expectedResult = createExpectedResult(context, term, groupSort, topN);- FirstPassGroupingCollector<Comparable<Object>> firstCollector =- createRandomFirstPassCollector(groupSort, GROUP_FIELD, topN);- searcher.search(new TermQuery(new Term("content", term)), firstCollector);- DistinctValuesCollector<Comparable<Object>, Comparable<Object>> distinctValuesCollector =- createDistinctCountCollector(firstCollector, COUNT_FIELD);- searcher.search(new TermQuery(new Term("content", term)), distinctValuesCollector);+ boolean useValueSource = random.nextBoolean();+ Supplier<GroupSelector<Comparable<Object>>> groupSelectorFactory =+ createGroupSelectorFactory(GROUP_FIELD, useValueSource);+ Supplier<GroupSelector<Comparable<Object>>> valueSelectorFactory =+ createGroupSelectorFactory(COUNT_FIELD, useValueSource);++ Collection<SearchGroup<Comparable<Object>>> searchGroups =+ searcher.search(+ new TermQuery(new Term("content", term)),+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, groupSort, 0, topN));++ if (searchGroups == null || searchGroups.isEmpty()) {+ assertEquals(0, expectedResult.size());+ continue;+ }+@SuppressWarnings("unchecked")List<DistinctValuesCollector.GroupCount<Comparable<Object>, Comparable<Object>>>- actualResult = distinctValuesCollector.getGroups();+ actualResult =+ searcher.search(+ new TermQuery(new Term("content", term)),+ new DistinctValuesCollectorManager<>(+ groupSelectorFactory, searchGroups, valueSelectorFactory));if (VERBOSE) {System.out.println("Index iter=" + indexIter);System.out.println("Search iter=" + searchIter);- System.out.println(- "1st pass collector class name=" + firstCollector.getClass().getName());- System.out.println(- "2nd pass collector class name=" + distinctValuesCollector.getClass().getName());+ System.out.println("useValueSource=" + useValueSource);System.out.println("Search term=" + term);- System.out.println("1st pass groups=" + firstCollector.getTopGroups(0));+ System.out.println("1st pass groups=" + searchGroups);System.out.println("Expected:");printGroups(expectedResult);System.out.println("Actual:");@@ -350,37 +373,15 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {doc.add(new SortedDocValuesField(field, new BytesRef(value)));}- @SuppressWarnings({"unchecked", "rawtypes"})- private <T extends Comparable<Object>, R extends Comparable<Object>>- DistinctValuesCollector<T, R> createDistinctCountCollector(- FirstPassGroupingCollector<T> firstPassGroupingCollector, String countField)- throws IOException {- Collection<SearchGroup<T>> searchGroups = firstPassGroupingCollector.getTopGroups(0);- GroupSelector<T> selector = firstPassGroupingCollector.getGroupSelector();- if (ValueSourceGroupSelector.class.isAssignableFrom(selector.getClass())) {- GroupSelector gs =- new ValueSourceGroupSelector(new BytesRefFieldSource(countField), new HashMap<>());- return new DistinctValuesCollector<>(selector, searchGroups, gs);- } else {- GroupSelector ts = new TermGroupSelector(countField);- return new DistinctValuesCollector<>(selector, searchGroups, ts);- }- }-- @SuppressWarnings({"unchecked", "rawtypes"})- private <T> FirstPassGroupingCollector<T> createRandomFirstPassCollector(- Sort groupSort, String groupField, int topNGroups) throws IOException {- Random random = random();- if (random.nextBoolean()) {- return (FirstPassGroupingCollector<T>)- new FirstPassGroupingCollector<>(- new ValueSourceGroupSelector(new BytesRefFieldSource(groupField), new HashMap<>()),- groupSort,- topNGroups);+ @SuppressWarnings({"unchecked"})+ private static <T> Supplier<GroupSelector<T>> createGroupSelectorFactory(+ String field, boolean useValueSource) {+ if (useValueSource) {+ return () ->+ (GroupSelector<T>)+ new ValueSourceGroupSelector(new BytesRefFieldSource(field), new HashMap<>());} else {- return (FirstPassGroupingCollector<T>)- new FirstPassGroupingCollector<>(- new TermGroupSelector(groupField), groupSort, topNGroups);+ return () -> (GroupSelector<T>) new TermGroupSelector(field);}}