Skip to content

Apache Lucene open-source search software

Introduce CachingCollectorManager to parallelize search when using CachingCollector (#16247)

gaobinlong3afffeef5a8d
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 20c0e582f18..02113d05d85 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -312,7 +312,12 @@ New Features
Improvements
---------------------
-* GITHUB#16170: Alter TopGroups.merge() to not return null if no groups. (Binlong Gao)
+
+* GITHUB#16247: Introduce CachingCollectorManager to parallelize search when using CachingCollector.
+ Remove experimental GroupingSearch constructor that takes a GroupSelector as argument
+ in favour of providing a GroupSelector supplier. (Binlong Gao)
+
+* GITHUB#16305: Alter TopGroups.merge() to not return null if no groups. (Binlong Gao)
Optimizations
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java b/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java
index 07aa3581e71..57311aa601d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java
@@ -329,7 +329,8 @@ public abstract class CachingCollector extends FilterCollector {
: new NoScoreCachingCollector(other, maxDocsToCache);
}
- private boolean cached;
+ // visible for other threads in concurrent search mode
+ private volatile boolean cached;
private CachingCollector(Collector in) {
super(in);
diff --git a/lucene/core/src/java/org/apache/lucene/search/CachingCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/CachingCollectorManager.java
new file mode 100644
index 00000000000..09a3be7b363
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/CachingCollectorManager.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * A {@link CollectorManager} that wraps a delegate {@link CollectorManager} and caches all
+ * collected documents (and optionally scores) per slice, so they can be replayed to a second-pass
+ * {@link CollectorManager} without re-running the query.
+ *
+ * <p>One {@link CachingCollector} is created per slice. During {@link #replay}, each cached slice
+ * is replayed into a fresh second-pass collector, and all second-pass collectors are reduced
+ * together. This works correctly with both sequential and concurrent search.
+ *
+ * <p>Example usage:
+ *
+ * <pre class="prettyprint">
+ * CachingCollectorManager&lt;C1, R1&gt; caching = new CachingCollectorManager&lt;&gt;(
+ * firstPassManager, cacheScores, maxRAMMB, null);
+ * R1 firstResult = searcher.search(query, caching);
+ *
+ * if (caching.isCached()) {
+ * R2 secondResult = caching.replay(secondPassManager);
+ * } else {
+ * // cache overflowed — re-run the query
+ * R2 secondResult = searcher.search(query, secondPassManager);
+ * }
+ * </pre>
+ *
+ * @lucene.experimental
+ */
+public class CachingCollectorManager<C extends Collector, R>
+ implements CollectorManager<CachingCollector, R> {
+
+ private final CollectorManager<C, R> delegate;
+ private final boolean cacheScores;
+ private final Double maxRAMMB;
+ private final Integer maxDocsToCache;
+
+ // One CachingCollector per slice
+ private final List<CachingCollector> cachingCollectors = new ArrayList<>();
+
+ /**
+ * @param delegate the first-pass {@link CollectorManager}
+ * @param cacheScores whether to cache scores in addition to document IDs
+ * @param maxRAMMB the maximum RAM in MB to use per slice cache, or null if using maxDocsToCache
+ * @param maxDocsToCache the maximum number of documents to cache per slice, or null if using
+ * maxRAMMB
+ */
+ public CachingCollectorManager(
+ CollectorManager<C, R> delegate,
+ boolean cacheScores,
+ Double maxRAMMB,
+ Integer maxDocsToCache) {
+ if (maxRAMMB == null && maxDocsToCache == null || maxRAMMB != null && maxDocsToCache != null) {
+ throw new IllegalArgumentException("Exactly one of maxRAMMB or maxDocsToCache must be set");
+ }
+ this.delegate = delegate;
+ this.cacheScores = cacheScores;
+ this.maxRAMMB = maxRAMMB;
+ this.maxDocsToCache = maxDocsToCache;
+ }
+
+ @Override
+ public CachingCollector newCollector() throws IOException {
+ C collector = delegate.newCollector();
+ CachingCollector cache =
+ maxDocsToCache != null
+ ? CachingCollector.create(collector, cacheScores, maxDocsToCache)
+ : CachingCollector.create(collector, cacheScores, maxRAMMB);
+ cachingCollectors.add(cache);
+ return cache;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public R reduce(Collection<CachingCollector> collectors) throws IOException {
+ List<C> originals = new ArrayList<>(collectors.size());
+ for (CachingCollector cache : collectors) {
+ originals.add((C) cache.in);
+ }
+ return delegate.reduce(originals);
+ }
+
+ /**
+ * Returns {@code true} if the search has been run and all per-slice caches are intact (none
+ * overflowed their RAM/doc budget). Returns {@code false} if the search has not yet been run or
+ * any cache overflowed.
+ */
+ public boolean isCached() {
+ return !cachingCollectors.isEmpty()
+ && cachingCollectors.stream().allMatch(CachingCollector::isCached);
+ }
+
+ /**
+ * Replays each per-slice cache into a fresh second-pass collector, then reduces all results.
+ *
+ * @throws IllegalStateException if {@link #isCached()} returns {@code false}
+ */
+ public <C2 extends Collector, R2> R2 replay(CollectorManager<C2, R2> secondPassManager)
+ throws IOException {
+ if (!isCached()) {
+ throw new IllegalStateException("cache is not available; re-run the query instead");
+ }
+ List<C2> secondCollectors = new ArrayList<>(cachingCollectors.size());
+ for (CachingCollector cache : cachingCollectors) {
+ C2 secondCollector = secondPassManager.newCollector();
+ cache.replay(secondCollector);
+ secondCollectors.add(secondCollector);
+ }
+ return secondPassManager.reduce(secondCollectors);
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestCachingCollectorManager.java b/lucene/core/src/test/org/apache/lucene/search/TestCachingCollectorManager.java
new file mode 100644
index 00000000000..530d64bab1f
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestCachingCollectorManager.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+public class TestCachingCollectorManager extends LuceneTestCase {
+
+ public void testCacheOverflow() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ for (int i = 0; i < atLeast(10); i++) {
+ iw.addDocument(new Document());
+ }
+ IndexSearcher searcher = newSearcher(iw.getReader());
+ iw.close();
+
+ CachingCollectorManager<TopScoreDocCollector, TopDocs> caching =
+ new CachingCollectorManager<>(
+ new TopScoreDocCollectorManager(10, Integer.MAX_VALUE), false, null, 0);
+
+ searcher.search(MatchAllDocsQuery.INSTANCE, caching);
+ assertFalse(caching.isCached());
+ assertThrows(
+ IllegalStateException.class,
+ () -> caching.replay(new TopScoreDocCollectorManager(10, Integer.MAX_VALUE)));
+
+ searcher.getIndexReader().close();
+ dir.close();
+ }
+
+ public void testNotCachedBeforeSearch() {
+ CachingCollectorManager<TopScoreDocCollector, TopDocs> caching =
+ new CachingCollectorManager<>(
+ new TopScoreDocCollectorManager(10, Integer.MAX_VALUE), false, null, Integer.MAX_VALUE);
+ assertFalse(caching.isCached());
+
+ assertThrows(
+ IllegalStateException.class,
+ () -> caching.replay(new TopScoreDocCollectorManager(10, Integer.MAX_VALUE)));
+ }
+
+ public void testBasic() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ for (int i = 0; i < 10; i++) {
+ iw.addDocument(new Document());
+ }
+ IndexSearcher searcher = newSearcher(iw.getReader());
+ iw.close();
+
+ CachingCollectorManager<TopScoreDocCollector, TopDocs> caching =
+ new CachingCollectorManager<>(
+ new TopScoreDocCollectorManager(10, Integer.MAX_VALUE), true, null, Integer.MAX_VALUE);
+
+ TopDocs firstResult = searcher.search(MatchAllDocsQuery.INSTANCE, caching);
+ assertTrue(caching.isCached());
+ assertEquals(10, firstResult.totalHits.value());
+
+ TopDocs replayResult = caching.replay(new TopScoreDocCollectorManager(10, Integer.MAX_VALUE));
+ assertEquals(firstResult.totalHits.value(), replayResult.totalHits.value());
+ assertEquals(firstResult.scoreDocs.length, replayResult.scoreDocs.length);
+
+ searcher.getIndexReader().close();
+ dir.close();
+ }
+
+ public void testConstructor() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ new CachingCollectorManager<>(
+ new TopScoreDocCollectorManager(10, Integer.MAX_VALUE), false, null, null));
+
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ new CachingCollectorManager<>(
+ new TopScoreDocCollectorManager(10, Integer.MAX_VALUE), false, 1.0, 1));
+ }
+}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollector.java
index 013f83b3f3c..04c80c727a3 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupHeadsCollector.java
@@ -173,6 +173,7 @@ public abstract class AllGroupHeadsCollector<T> extends SimpleCollector {
@Override
public void setScorer(Scorable scorer) throws IOException {
this.scorer = scorer;
+ groupSelector.setScorer(scorer);
for (GroupHead<T> head : heads.values()) {
head.setScorer(scorer);
}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupsCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupsCollector.java
index d1b700cef1c..4aa5301a0af 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupsCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AllGroupsCollector.java
@@ -70,7 +70,9 @@ public class AllGroupsCollector<T> extends SimpleCollector {
}
@Override
- public void setScorer(Scorable scorer) throws IOException {}
+ public void setScorer(Scorable scorer) throws IOException {
+ groupSelector.setScorer(scorer);
+ }
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java
index 630ffde1404..88b21b50686 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java
@@ -17,14 +17,18 @@
package org.apache.lucene.search.grouping;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
+import java.util.function.Supplier;
import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.CachingCollector;
+import org.apache.lucene.search.CachingCollectorManager;
import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.CollectorManager;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MultiCollector;
+import org.apache.lucene.search.MultiCollectorManager;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Sort;
@@ -41,7 +45,7 @@ import org.apache.lucene.util.mutable.MutableValue;
*/
public class GroupingSearch {
- private final GroupSelector<?> grouper;
+ private final Supplier<GroupSelector<?>> grouperFactory;
private final Query groupEndDocs;
private Sort groupSort = Sort.RELEVANCE;
@@ -69,17 +73,17 @@ public class GroupingSearch {
* @param groupField The name of the field to group by.
*/
public GroupingSearch(String groupField) {
- this(new TermGroupSelector(groupField), null);
+ this(() -> new TermGroupSelector(groupField), null);
}
/**
* Constructs a <code>GroupingSearch</code> instance that groups documents using a {@link
- * GroupSelector}
+ * GroupSelector} factory.
*
- * @param groupSelector a {@link GroupSelector} that defines groups for this GroupingSearch
+ * @param grouperFactory a factory that creates fresh {@link GroupSelector} instances
*/
- public GroupingSearch(GroupSelector<?> groupSelector) {
- this(groupSelector, null);
+ public GroupingSearch(Supplier<GroupSelector<?>> grouperFactory) {
+ this(grouperFactory, null);
}
/**
@@ -90,7 +94,7 @@ public class GroupingSearch {
* @param valueSourceContext The context of the specified groupFunction
*/
public GroupingSearch(ValueSource groupFunction, Map<Object, Object> valueSourceContext) {
- this(new ValueSourceGroupSelector(groupFunction, valueSourceContext), null);
+ this(() -> new ValueSourceGroupSelector(groupFunction, valueSourceContext), null);
}
/**
@@ -103,8 +107,8 @@ public class GroupingSearch {
this(null, groupEndDocs);
}
- private GroupingSearch(GroupSelector<?> grouper, Query groupEndDocs) {
- this.grouper = grouper;
+ private GroupingSearch(Supplier<GroupSelector<?>> grouperFactory, Query groupEndDocs) {
+ this.grouperFactory = grouperFactory;
this.groupEndDocs = groupEndDocs;
}
@@ -123,7 +127,7 @@ public class GroupingSearch {
@SuppressWarnings("unchecked")
public <T> TopGroups<T> search(
IndexSearcher searcher, Query query, int groupOffset, int groupLimit) throws IOException {
- if (grouper != null) {
+ if (grouperFactory != null) {
return groupByFieldOrFunction(searcher, query, groupOffset, groupLimit);
} else if (groupEndDocs != null) {
return groupByDocBlock(searcher, query, groupOffset, groupLimit);
@@ -134,59 +138,83 @@ public class GroupingSearch {
}
@SuppressWarnings({"unchecked", "rawtypes"})
- protected TopGroups groupByFieldOrFunction(
+ protected <T> TopGroups<T> groupByFieldOrFunction(
IndexSearcher searcher, Query query, int groupOffset, int groupLimit) throws IOException {
- int topN = groupOffset + groupLimit;
+ @SuppressWarnings("unchecked")
+ Supplier<GroupSelector<T>> typedGrouperFactory =
+ (Supplier<GroupSelector<T>>) (Supplier<?>) grouperFactory;
+ FirstPassGroupingCollectorManager<T> firstPassManager =
+ new FirstPassGroupingCollectorManager<>(
+ typedGrouperFactory, groupSort, groupOffset, groupLimit, ignoreDocsWithoutGroupField);
+ List<CollectorManager<? extends Collector, ?>> firstRoundManagers = new ArrayList<>();
+ firstRoundManagers.add(firstPassManager);
+ AllGroupsCollectorManager<T> allGroupsManager;
+ if (allGroups) {
+ allGroupsManager = new AllGroupsCollectorManager<>(typedGrouperFactory);
+ firstRoundManagers.add(allGroupsManager);
+ }
- final FirstPassGroupingCollector firstPassCollector =
- new FirstPassGroupingCollector(grouper, groupSort, topN, ignoreDocsWithoutGroupField);
- final AllGroupsCollector allGroupsCollector =
- allGroups ? new AllGroupsCollector(grouper) : null;
- final AllGroupHeadsCollector allGroupHeadsCollector =
- allGroupHeads ? AllGroupHeadsCollector.newCollector(grouper, sortWithinGroup) : null;
+ AllGroupHeadsCollectorManager<T> allGroupHeadsManager;
+ if (allGroupHeads) {
+ allGroupHeadsManager =
+ new AllGroupHeadsCollectorManager<>(typedGrouperFactory, sortWithinGroup);
+ firstRoundManagers.add(allGroupHeadsManager);
+ }
- final Collector firstRound =
- MultiCollector.wrap(firstPassCollector, allGroupsCollector, allGroupHeadsCollector);
+ CollectorManager<?, Object[]> firstRoundManager =
+ new MultiCollectorManager(firstRoundManagers.toArray(CollectorManager[]::new));
- CachingCollector cachedCollector = null;
+ CachingCollectorManager<?, Object[]> cachingManager = null;
+ Object[] firstRoundResults;
if (maxCacheRAMMB != null || maxDocsToCache != null) {
- if (maxCacheRAMMB != null) {
- cachedCollector = CachingCollector.create(firstRound, cacheScores, maxCacheRAMMB);
- } else {
- cachedCollector = CachingCollector.create(firstRound, cacheScores, maxDocsToCache);
- }
- searcher.search(query, cachedCollector);
+ cachingManager =
+ new CachingCollectorManager<>(
+ firstRoundManager, cacheScores, maxCacheRAMMB, maxDocsToCache);
+ firstRoundResults = searcher.search(query, cachingManager);
} else {
- searcher.search(query, firstRound);
+ firstRoundResults = searcher.search(query, firstRoundManager);
}
- matchingGroups = allGroups ? allGroupsCollector.getGroups() : Collections.emptyList();
- matchingGroupHeads =
- allGroupHeads
- ? allGroupHeadsCollector.retrieveGroupHeads(searcher.getIndexReader().maxDoc())
- : new Bits.MatchNoBits(searcher.getIndexReader().maxDoc());
+ int resultIdx = 0;
+ Collection<SearchGroup<T>> topSearchGroups =
+ (Collection<SearchGroup<T>>) firstRoundResults[resultIdx++];
+
+ matchingGroups =
+ allGroups ? (Collection<?>) firstRoundResults[resultIdx++] : Collections.emptyList();
+
+ if (allGroupHeads) {
+ AllGroupHeadsCollectorManager.GroupHeadsResult headsResult =
+ (AllGroupHeadsCollectorManager.GroupHeadsResult) firstRoundResults[resultIdx];
+ matchingGroupHeads = headsResult.retrieveGroupHeads(searcher.getIndexReader().maxDoc());
+ } else {
+ matchingGroupHeads = new Bits.MatchNoBits(searcher.getIndexReader().maxDoc());
+ }
- Collection<SearchGroup> topSearchGroups = firstPassCollector.getTopGroups(groupOffset);
- if (topSearchGroups == null) {
- return new TopGroups(new SortField[0], new SortField[0], 0, 0, new GroupDocs[0], Float.NaN);
+ if (topSearchGroups.isEmpty()) {
+ return new TopGroups<>(new SortField[0], new SortField[0], 0, 0, new GroupDocs[0], Float.NaN);
}
- int topNInsideGroup = groupDocsOffset + groupDocsLimit;
- TopGroupsCollector secondPassCollector =
- new TopGroupsCollector(
- grouper, topSearchGroups, groupSort, sortWithinGroup, topNInsideGroup, includeMaxScore);
+ TopGroupsCollectorManager<T> secondPassManager =
+ new TopGroupsCollectorManager<>(
+ typedGrouperFactory,
+ topSearchGroups,
+ groupSort,
+ sortWithinGroup,
+ groupDocsOffset,
+ groupDocsLimit,
+ includeMaxScore);
- if (cachedCollector != null && cachedCollector.isCached()) {
- cachedCollector.replay(secondPassCollector);
+ TopGroups<T> secondResult;
+ if (cachingManager != null && cachingManager.isCached()) {
+ secondResult = cachingManager.replay(secondPassManager);
} else {
- searcher.search(query, secondPassCollector);
+ secondResult = searcher.search(query, secondPassManager);
}
if (allGroups) {
- return new TopGroups(
- secondPassCollector.getTopGroups(groupDocsOffset), matchingGroups.size());
+ return new TopGroups<>(secondResult, matchingGroups.size());
} else {
- return secondPassCollector.getTopGroups(groupDocsOffset);
+ return secondResult;
}
}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/BaseGroupSelectorTestCase.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/BaseGroupSelectorTestCase.java
index f82da65c0c1..820ef11dca3 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/BaseGroupSelectorTestCase.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/BaseGroupSelectorTestCase.java
@@ -32,6 +32,7 @@ import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
@@ -57,7 +58,7 @@ public abstract class BaseGroupSelectorTestCase<T> extends AbstractGroupingTestC
Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));
IndexSearcher searcher = shard.getIndexSearcher();
- GroupingSearch grouper = new GroupingSearch(getGroupSelector());
+ GroupingSearch grouper = new GroupingSearch(this::getGroupSelector);
grouper.setGroupDocsLimit(10);
TopGroups<T> topGroups = grouper.search(searcher, topLevel, 0, 5);
TopDocs topDoc = searcher.search(topLevel, 1);
@@ -70,9 +71,8 @@ public abstract class BaseGroupSelectorTestCase<T> extends AbstractGroupingTestC
.add(filterQuery(topGroups.groups[i].groupValue()), BooleanClause.Occur.FILTER)
.build();
TopDocs td = searcher.search(filtered, 10);
- assertScoreDocsEquals(topGroups.groups[i].scoreDocs(), td.scoreDocs);
+ assertScoresEquals(topGroups.groups[i].scoreDocs(), td.scoreDocs);
if (i == 0) {
- assertEquals(td.scoreDocs[0].doc, topDoc.scoreDocs[0].doc);
assertEquals(td.scoreDocs[0].score, topDoc.scoreDocs[0].score, 0);
}
}
@@ -89,7 +89,7 @@ public abstract class BaseGroupSelectorTestCase<T> extends AbstractGroupingTestC
String[] query = new String[] {"foo", "bar", "baz"};
Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));
- GroupingSearch grouper = new GroupingSearch(getGroupSelector());
+ GroupingSearch grouper = new GroupingSearch(this::getGroupSelector);
grouper.setGroupDocsLimit(10);
Sort sort =
new Sort(
@@ -108,7 +108,7 @@ public abstract class BaseGroupSelectorTestCase<T> extends AbstractGroupingTestC
.add(filterQuery(topGroups.groups[i].groupValue()), BooleanClause.Occur.FILTER)
.build();
TopDocs td = searcher.search(filtered, 10);
- assertScoreDocsEquals(topGroups.groups[i].scoreDocs(), td.scoreDocs);
+ assertScoresEquals(topGroups.groups[i].scoreDocs(), td.scoreDocs);
// The top group should have sort values equal to the sort values of the top doc of
// a top-level search sorted by the same Sort; subsequent groups should have sort values
// that compare lower than their predecessor.
@@ -132,7 +132,7 @@ public abstract class BaseGroupSelectorTestCase<T> extends AbstractGroupingTestC
String[] query = new String[] {"foo", "bar", "baz"};
Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));
- GroupingSearch grouper = new GroupingSearch(getGroupSelector());
+ GroupingSearch grouper = new GroupingSearch(this::getGroupSelector);
grouper.setGroupDocsLimit(10);
Sort sort =
new Sort(
@@ -160,7 +160,7 @@ public abstract class BaseGroupSelectorTestCase<T> extends AbstractGroupingTestC
.add(filterQuery(topGroups.groups[i].groupValue()), BooleanClause.Occur.FILTER)
.build();
TopDocs td = searcher.search(filtered, 10, sort);
- assertScoreDocsEquals(td.scoreDocs, topGroups.groups[i].scoreDocs());
+ assertScoresEquals(td.scoreDocs, topGroups.groups[i].scoreDocs());
}
shard.close();
@@ -175,8 +175,7 @@ public abstract class BaseGroupSelectorTestCase<T> extends AbstractGroupingTestC
String[] query = new String[] {"foo", "bar", "baz"};
Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));
- GroupSelector<T> groupSelector = getGroupSelector();
- GroupingSearch grouping = new GroupingSearch(groupSelector);
+ GroupingSearch grouping = new GroupingSearch(this::getGroupSelector);
grouping.setAllGroups(true);
grouping.setAllGroupHeads(true);
@@ -235,8 +234,7 @@ public abstract class BaseGroupSelectorTestCase<T> extends AbstractGroupingTestC
new Sort(
new SortField("sort1", SortField.Type.STRING),
new SortField("sort2", SortField.Type.LONG));
- GroupSelector<T> groupSelector = getGroupSelector();
- GroupingSearch grouping = new GroupingSearch(groupSelector);
+ GroupingSearch grouping = new GroupingSearch(this::getGroupSelector);
grouping.setAllGroups(true);
grouping.setAllGroupHeads(true);
grouping.setSortWithinGroup(sort);
@@ -417,12 +415,12 @@ public abstract class BaseGroupSelectorTestCase<T> extends AbstractGroupingTestC
Query query = new TermQuery(new Term("text", "foo"));
// Test default behavior (include null group)
- GroupingSearch grouping1 = new GroupingSearch(getGroupSelector());
+ GroupingSearch grouping1 = new GroupingSearch(this::getGroupSelector);
TopGroups<T> groups1 = grouping1.search(searcher, query, 0, 10);
int defaultGroupCount = groups1.groups.length;
// Test ignoring docs without group field
- GroupingSearch grouping2 = new GroupingSearch(getGroupSelector());
+ GroupingSearch grouping2 = new GroupingSearch(this::getGroupSelector);
grouping2.setIgnoreDocsWithoutGroupField(true);
TopGroups<T> groups2 = grouping2.search(searcher, query, 0, 10);
int ignoreGroupCount = groups2.groups.length;
@@ -437,6 +435,13 @@ public abstract class BaseGroupSelectorTestCase<T> extends AbstractGroupingTestC
shard.close();
}
+ private void assertScoresEquals(ScoreDoc[] expected, ScoreDoc[] actual) {
+ assertEquals(expected.length, actual.length);
+ for (int i = 0; i < expected.length; i++) {
+ assertEquals(expected[i].score, actual[i].score, 0);
+ }
+ }
+
private void assertSortsBefore(GroupDocs<T> first, GroupDocs<T> second) {
Object[] groupSortValues = second.groupSortValues();
Object[] prevSortValues = first.groupSortValues();
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGroupingSearch.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGroupingSearch.java
index 07de8a18d7d..53ec2f70ed9 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGroupingSearch.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGroupingSearch.java
@@ -36,6 +36,7 @@ import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.mutable.MutableValueStr;
@@ -218,6 +219,48 @@ public class TestGroupingSearch extends AbstractGroupingTestCase {
return groupingSearch;
}
+ public void testEmptyTopSearchGroups() throws Exception {
+ final String groupField = "author";
+
+ Shard shard = new Shard();
+ RandomIndexWriter w = shard.writer;
+
+ Document doc = new Document();
+ doc.add(new TextField(groupField, "author1", Field.Store.YES));
+ doc.add(new SortedDocValuesField(groupField, new BytesRef("author1")));
+ doc.add(new TextField("content", "hello world", Field.Store.YES));
+ w.addDocument(doc);
+
+ IndexSearcher indexSearcher = shard.getIndexSearcher();
+ w.close();
+
+ GroupingSearch gs = new GroupingSearch(groupField);
+ gs.setGroupDocsLimit(5);
+ gs.setAllGroups(true);
+ gs.setAllGroupHeads(true);
+ TopGroups<?> groups =
+ gs.search(indexSearcher, new TermQuery(new Term("content", "hello")), 10, 10);
+
+ assertNotNull(groups);
+ assertEquals(0, groups.totalHitCount);
+ assertEquals(0, groups.groups.length);
+
+ assertNotNull(
+ "getAllMatchingGroups() should not be null when no groups are found",
+ gs.getAllMatchingGroups());
+ assertEquals(1, gs.getAllMatchingGroups().size());
+
+ assertNotNull(
+ "getAllGroupHeads() should not be null when no groups are found", gs.getAllGroupHeads());
+ int count = 0;
+ Bits bits = gs.getAllGroupHeads();
+ for (int i = 0; i < bits.length(); i++) {
+ if (bits.get(i)) count++;
+ }
+ assertEquals(1, count);
+ shard.close();
+ }
+
public void testSetAllGroups() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w =