Apache Lucene open-source search software
Introduce AllGroupHeadsCollectorManager (#15565)
gaobinlong11232def9896
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex f556fd58861..5b0bf2fabde 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -338,6 +338,8 @@ Improvements* GITHUB#15574, GITHUB#15995: Introduce TopGroupsCollectorManager to parallelize search when using TopGroupsCollector. (Binlong Gao)+* GITHUB#15565: Introduce AllGroupHeadsCollectorManager to parallelize search when using AllGroupHeadsCollector. (Binlong Gao)+* GITHUB#15989: DocValuesRangeIterator always tries to use skipper-based block iterationas its approximation. (Alan Woodward)diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollector.javaindex 4ff5e16c96d..013f83b3f3c 100644--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollector.java+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollector.java@@ -232,6 +232,20 @@ public abstract class AllGroupHeadsCollector<T> extends SimpleCollector {* @throws IOException If I/O related errors occur*/protected abstract void updateDocHead(int doc) throws IOException;++ /**+ * Returns the sort values for this group head.+ *+ * @return the sort values, or null if not stored+ */+ protected abstract Object[] getSortValues();++ /**+ * Returns the field comparators used to determine the group head ordering.+ *+ * @return the comparators, one per sort field+ */+ protected abstract FieldComparator[] getComparators();}/** General implementation using a {@link FieldComparator} to select the group head */@@ -252,6 +266,7 @@ public abstract class AllGroupHeadsCollector<T> extends SimpleCollector {final FieldComparator[] comparators;final LeafFieldComparator[] leafComparators;+ final Object[] sortValues;protected SortingGroupHead(Sort sort, T groupValue, int doc, LeafReaderContext context, Scorable scorer)@@ -260,12 +275,14 @@ public abstract class AllGroupHeadsCollector<T> extends SimpleCollector {final SortField[] sortFields = sort.getSort();comparators = new FieldComparator[sortFields.length];leafComparators = new LeafFieldComparator[sortFields.length];+ sortValues = new Object[sortFields.length];for (int i = 0; i < sortFields.length; i++) {comparators[i] = sortFields[i].getComparator(1, Pruning.NONE);leafComparators[i] = comparators[i].getLeafComparator(context);leafComparators[i].setScorer(scorer);leafComparators[i].copy(0, doc);leafComparators[i].setBottom(0);+ sortValues[i] = comparators[i].value(0);}}@@ -291,12 +308,23 @@ public abstract class AllGroupHeadsCollector<T> extends SimpleCollector {@Overridepublic void updateDocHead(int doc) throws IOException {- for (LeafFieldComparator comparator : leafComparators) {- comparator.copy(0, doc);- comparator.setBottom(0);+ for (int i = 0; i < leafComparators.length; i++) {+ leafComparators[i].copy(0, doc);+ leafComparators[i].setBottom(0);+ sortValues[i] = comparators[i].value(0);}this.doc = doc + docBase;}++ @Override+ protected Object[] getSortValues() {+ return sortValues;+ }++ @Override+ protected FieldComparator[] getComparators() {+ return comparators;+ }}/** Specialized implementation for sorting by score */@@ -317,12 +345,14 @@ public abstract class AllGroupHeadsCollector<T> extends SimpleCollector {private Scorable scorer;private float topScore;+ private final Object[] sortValues;protected ScoringGroupHead(Scorable scorer, T groupValue, int doc, int docBase)throws IOException {super(groupValue, doc, docBase);this.scorer = scorer;this.topScore = scorer.score();+ this.sortValues = new Object[] {topScore};}@Override@@ -344,6 +374,17 @@ public abstract class AllGroupHeadsCollector<T> extends SimpleCollector {@Overrideprotected void updateDocHead(int doc) throws IOException {this.doc = doc + docBase;+ sortValues[0] = topScore;+ }++ @Override+ protected Object[] getSortValues() {+ return sortValues;+ }++ @Override+ protected FieldComparator[] getComparators() {+ return null;}}}diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollectorManager.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollectorManager.javanew file mode 100644index 00000000000..2eae8b2171b--- /dev/null+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollectorManager.java@@ -0,0 +1,163 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search.grouping;++import java.io.IOException;+import java.util.Collection;+import java.util.HashMap;+import java.util.Map;+import java.util.function.Supplier;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.search.CollectorManager;+import org.apache.lucene.search.FieldComparator;+import org.apache.lucene.search.Sort;+import org.apache.lucene.search.SortField;+import org.apache.lucene.util.Bits;+import org.apache.lucene.util.FixedBitSet;++/**+ * A {@link CollectorManager} implementation for {@link AllGroupHeadsCollector} that collects the+ * most relevant document (group head) for each group across multiple segments and merges the+ * per-segment results into a single {@link GroupHeadsResult}.+ *+ * <p>Example usage:+ *+ * <pre class="prettyprint">+ * IndexSearcher searcher = ...; // your IndexSearcher+ * AllGroupHeadsCollectorManager<BytesRef> manager =+ * new AllGroupHeadsCollectorManager<>(+ * () -> new TermGroupSelector("category"), Sort.RELEVANCE);+ * GroupHeadsResult result = searcher.search(new MatchAllDocsQuery(), manager);+ * Bits groupHeadsBits = result.retrieveGroupHeads(searcher.getIndexReader().maxDoc());+ * </pre>+ *+ * @param <T> the type of the group value+ * @lucene.experimental+ */+public class AllGroupHeadsCollectorManager<T>+ implements CollectorManager<+ AllGroupHeadsCollector<T>, AllGroupHeadsCollectorManager.GroupHeadsResult> {++ /** Holds the merged group heads and provides access as an {@code int[]} or {@link Bits}. */+ public static class GroupHeadsResult {+ private final int[] groupHeads;++ private GroupHeadsResult(int[] groupHeads) {+ this.groupHeads = groupHeads;+ }++ /** Returns the group head document IDs as an array. */+ public int[] retrieveGroupHeads() {+ return groupHeads;+ }++ /**+ * Returns the group head document IDs as a {@link Bits} set of size {@code maxDoc}, suitable+ * for use as a filter.+ *+ * @param maxDoc The maxDoc of the top level {@link IndexReader}.+ */+ public Bits retrieveGroupHeads(int maxDoc) {+ FixedBitSet result = new FixedBitSet(maxDoc);+ for (int docId : groupHeads) {+ result.set(docId);+ }+ return result;+ }+ }++ private static final class GroupHeadWithValues {+ int doc;+ final Object[] sortValues;++ GroupHeadWithValues(int doc, Object[] sortValues) {+ this.doc = doc;+ this.sortValues = sortValues;+ }+ }++ private final Supplier<GroupSelector<T>> groupSelectorFactory;+ private final Sort sortWithinGroup;++ /**+ * Creates a new AllGroupHeadsCollectorManager.+ *+ * @param groupSelectorFactory factory to create group selectors for each collector+ * @param sortWithinGroup the sort to use within each group to determine the group head+ */+ public AllGroupHeadsCollectorManager(+ Supplier<GroupSelector<T>> groupSelectorFactory, Sort sortWithinGroup) {+ this.groupSelectorFactory = groupSelectorFactory;+ this.sortWithinGroup = sortWithinGroup;+ }++ @Override+ public AllGroupHeadsCollector<T> newCollector() throws IOException {+ return AllGroupHeadsCollector.newCollector(groupSelectorFactory.get(), sortWithinGroup);+ }++ @Override+ public GroupHeadsResult reduce(Collection<AllGroupHeadsCollector<T>> collectors) {+ Map<T, GroupHeadWithValues> mergedHeads = new HashMap<>();+ SortField[] sortFields = sortWithinGroup.getSort();++ for (AllGroupHeadsCollector<T> collector : collectors) {+ mergeCollectorHeads(collector, mergedHeads, sortFields);+ }++ return new GroupHeadsResult(mergedHeads.values().stream().mapToInt(h -> h.doc).toArray());+ }++ private void mergeCollectorHeads(+ AllGroupHeadsCollector<T> collector,+ Map<T, GroupHeadWithValues> mergedHeads,+ SortField[] sortFields) {+ for (AllGroupHeadsCollector.GroupHead<T> head : collector.getCollectedGroupHeads()) {+ Object[] sortValues = head.getSortValues();+ GroupHeadWithValues existing = mergedHeads.get(head.groupValue);+ if (existing == null || isCompetitive(head, sortValues, existing, sortFields)) {+ mergedHeads.put(head.groupValue, new GroupHeadWithValues(head.doc, sortValues));+ }+ }+ }++ @SuppressWarnings({"rawtypes"})+ private boolean isCompetitive(+ AllGroupHeadsCollector.GroupHead<T> head,+ Object[] sortValues,+ GroupHeadWithValues existing,+ SortField[] sortFields) {+ FieldComparator[] comparators = head.getComparators();+ int cmp;+ if (sortWithinGroup.equals(Sort.RELEVANCE)) {+ cmp = Float.compare((float) sortValues[0], (float) existing.sortValues[0]);+ return cmp > 0 || (cmp == 0 && head.doc < existing.doc);+ } else {+ cmp = 0;+ for (int i = 0; i < sortFields.length; i++) {+ @SuppressWarnings({"unchecked"})+ int c = comparators[i].compareValues(sortValues[i], existing.sortValues[i]);+ c = sortFields[i].getReverse() ? -c : c;+ if (c != 0) {+ cmp = c;+ break;+ }+ }+ return cmp < 0 || (cmp == 0 && head.doc < existing.doc);+ }+ }+}diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestAllGroupHeadsCollector.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestAllGroupHeadsCollector.javaindex a7aa05c1418..baa2c479233 100644--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestAllGroupHeadsCollector.java+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestAllGroupHeadsCollector.java@@ -136,45 +136,62 @@ public class TestAllGroupHeadsCollector extends LuceneTestCase {int maxDoc = reader.maxDoc();Sort sortWithinGroup = new Sort(new SortField("id_1", SortField.Type.INT, true));- AllGroupHeadsCollector<?> allGroupHeadsCollector =- createRandomCollector(groupField, sortWithinGroup);- indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector);- assertTrue(arrayContains(new int[] {2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads()));+ AllGroupHeadsCollectorManager<?> allGroupHeadsCollectorManager =+ createRandomCollectorManager(groupField, sortWithinGroup);+ AllGroupHeadsCollectorManager.GroupHeadsResult groupHeadsResult =+ indexSearcher.search(+ new TermQuery(new Term("content", "random")), allGroupHeadsCollectorManager);+ assertTrue(arrayContains(new int[] {2, 3, 5, 7}, groupHeadsResult.retrieveGroupHeads()));assertTrue(openBitSetContains(- new int[] {2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));+ new int[] {2, 3, 5, 7}, groupHeadsResult.retrieveGroupHeads(maxDoc), maxDoc));- allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup);- indexSearcher.search(new TermQuery(new Term("content", "some")), allGroupHeadsCollector);- assertTrue(arrayContains(new int[] {2, 3, 4}, allGroupHeadsCollector.retrieveGroupHeads()));+ allGroupHeadsCollectorManager = createRandomCollectorManager(groupField, sortWithinGroup);+ groupHeadsResult =+ indexSearcher.search(+ new TermQuery(new Term("content", "some")), allGroupHeadsCollectorManager);+ assertTrue(arrayContains(new int[] {2, 3, 4}, groupHeadsResult.retrieveGroupHeads()));assertTrue(openBitSetContains(- new int[] {2, 3, 4}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));+ new int[] {2, 3, 4}, groupHeadsResult.retrieveGroupHeads(maxDoc), maxDoc));- allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup);- indexSearcher.search(new TermQuery(new Term("content", "blob")), allGroupHeadsCollector);- assertTrue(arrayContains(new int[] {1, 5}, allGroupHeadsCollector.retrieveGroupHeads()));+ allGroupHeadsCollectorManager = createRandomCollectorManager(groupField, sortWithinGroup);+ groupHeadsResult =+ indexSearcher.search(+ new TermQuery(new Term("content", "blob")), allGroupHeadsCollectorManager);+ assertTrue(arrayContains(new int[] {1, 5}, groupHeadsResult.retrieveGroupHeads()));assertTrue(- openBitSetContains(- new int[] {1, 5}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));+ openBitSetContains(new int[] {1, 5}, groupHeadsResult.retrieveGroupHeads(maxDoc), maxDoc));// STRING sort type triggers different implementationSort sortWithinGroup2 = new Sort(new SortField("id_2", SortField.Type.STRING, true));- allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup2);- indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector);- assertTrue(arrayContains(new int[] {2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads()));+ allGroupHeadsCollectorManager = createRandomCollectorManager(groupField, sortWithinGroup2);+ groupHeadsResult =+ indexSearcher.search(+ new TermQuery(new Term("content", "random")), allGroupHeadsCollectorManager);+ assertTrue(arrayContains(new int[] {2, 3, 5, 7}, groupHeadsResult.retrieveGroupHeads()));assertTrue(openBitSetContains(- new int[] {2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));+ new int[] {2, 3, 5, 7}, groupHeadsResult.retrieveGroupHeads(maxDoc), maxDoc));Sort sortWithinGroup3 = new Sort(new SortField("id_2", SortField.Type.STRING, false));- allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup3);- indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector);+ allGroupHeadsCollectorManager = createRandomCollectorManager(groupField, sortWithinGroup3);+ groupHeadsResult =+ indexSearcher.search(+ new TermQuery(new Term("content", "random")), allGroupHeadsCollectorManager);// 7 b/c higher doc id wins, even if order of field is in not in reverse.- assertTrue(arrayContains(new int[] {0, 3, 4, 6}, allGroupHeadsCollector.retrieveGroupHeads()));+ assertTrue(arrayContains(new int[] {0, 3, 4, 6}, groupHeadsResult.retrieveGroupHeads()));assertTrue(openBitSetContains(- new int[] {0, 3, 4, 6}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));+ new int[] {0, 3, 4, 6}, groupHeadsResult.retrieveGroupHeads(maxDoc), maxDoc));++ // query that matches no documents should return empty group heads+ allGroupHeadsCollectorManager = createRandomCollectorManager(groupField, sortWithinGroup);+ groupHeadsResult =+ indexSearcher.search(+ new TermQuery(new Term("content", "nonexistent")), allGroupHeadsCollectorManager);+ assertEquals(0, groupHeadsResult.retrieveGroupHeads().length);+ assertEquals(0, ((FixedBitSet) groupHeadsResult.retrieveGroupHeads(maxDoc)).cardinality());indexSearcher.getIndexReader().close();dir.close();@@ -345,13 +362,14 @@ public class TestAllGroupHeadsCollector extends LuceneTestCase {final String searchTerm = "real" + random().nextInt(3);boolean sortByScoreOnly = random().nextBoolean();Sort sortWithinGroup = getRandomSort(sortByScoreOnly);- AllGroupHeadsCollector<?> allGroupHeadsCollector =- createRandomCollector("group", sortWithinGroup);- s.search(new TermQuery(new Term("content", searchTerm)), allGroupHeadsCollector);+ AllGroupHeadsCollectorManager<?> allGroupHeadsCollectorManager =+ createRandomCollectorManager("group", sortWithinGroup);+ AllGroupHeadsCollectorManager.GroupHeadsResult groupHeadsResult =+ s.search(new TermQuery(new Term("content", searchTerm)), allGroupHeadsCollectorManager);int[] expectedGroupHeads =createExpectedGroupHeads(searchTerm, groupDocs, sortWithinGroup, sortByScoreOnly, fieldIdToDocID);- int[] actualGroupHeads = allGroupHeadsCollector.retrieveGroupHeads();+ int[] actualGroupHeads = groupHeadsResult.retrieveGroupHeads();// The actual group heads contains Lucene ids. Need to change them into our id value.for (int i = 0; i < actualGroupHeads.length; i++) {actualGroupHeads[i] = docIDToFieldId[actualGroupHeads[i]];@@ -361,7 +379,8 @@ public class TestAllGroupHeadsCollector extends LuceneTestCase {Arrays.sort(actualGroupHeads);if (VERBOSE) {- System.out.println("Collector: " + allGroupHeadsCollector.getClass().getSimpleName());+ System.out.println(+ "CollectorManager: " + allGroupHeadsCollectorManager.getClass().getSimpleName());System.out.println("Sort within group: " + sortWithinGroup);System.out.println("Num group: " + numGroups);System.out.println("Num doc: " + numDocs);@@ -548,14 +567,15 @@ public class TestAllGroupHeadsCollector extends LuceneTestCase {};}- private AllGroupHeadsCollector<?> createRandomCollector(String groupField, Sort sortWithinGroup) {+ private AllGroupHeadsCollectorManager<?> createRandomCollectorManager(+ String groupField, Sort sortWithinGroup) {if (random().nextBoolean()) {ValueSource vs = new BytesRefFieldSource(groupField);- return AllGroupHeadsCollector.newCollector(- new ValueSourceGroupSelector(vs, new HashMap<>()), sortWithinGroup);+ return new AllGroupHeadsCollectorManager<>(+ () -> new ValueSourceGroupSelector(vs, new HashMap<>()), sortWithinGroup);} else {- return AllGroupHeadsCollector.newCollector(- new TermGroupSelector(groupField), sortWithinGroup);+ return new AllGroupHeadsCollectorManager<>(+ () -> new TermGroupSelector(groupField), sortWithinGroup);}}