Skip to content

Apache Lucene open-source search software

Introduce LargeNumHitsTopDocsCollectorManager (#15660)

gaobinlong3fe701ceaa5f
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5dbdc67c4ff..f556fd58861 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -343,6 +343,8 @@ Improvements
* GITHUB#16000: Clarify that Accountable#ramBytesUsed() reports JVM heap memory only. (Luca Cavanna)
+* GITHUB#15660: Introduce LargeNumHitsTopDocsCollectorManager to parallelize search when using LargeNumHitsTopDocsCollector. (Binlong Gao)
+
Optimizations
---------------------
* GITHUB#15861: Optimise PhraseScorer by short circuiting non competitive documents in TOP_SCORES mode. (Prithvi S)
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/LargeNumHitsTopDocsCollectorManager.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/LargeNumHitsTopDocsCollectorManager.java
new file mode 100644
index 00000000000..e874b5894c3
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/LargeNumHitsTopDocsCollectorManager.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.search;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.lucene.search.CollectorManager;
+import org.apache.lucene.search.TopDocs;
+
+/**
+ * CollectorManager for {@link LargeNumHitsTopDocsCollector} that enables concurrent collection of
+ * top docs across multiple segments.
+ *
+ * <p>Example usage:
+ *
+ * <pre class="prettyprint">
+ * IndexSearcher searcher = ...; // your IndexSearcher
+ * TopDocs topDocs = searcher.search(new MatchAllDocsQuery(),
+ * new LargeNumHitsTopDocsCollectorManager(100_000));
+ * System.out.println("Total hits: " + topDocs.totalHits.value());
+ * </pre>
+ */
+public class LargeNumHitsTopDocsCollectorManager
+ implements CollectorManager<LargeNumHitsTopDocsCollector, TopDocs> {
+ private final int numHits;
+ private final List<LargeNumHitsTopDocsCollector> collectors;
+
+ /**
+ * Creates a new {@link LargeNumHitsTopDocsCollectorManager} given the number of hits to collect.
+ *
+ * @param numHits the number of results to collect.
+ */
+ public LargeNumHitsTopDocsCollectorManager(int numHits) {
+ if (numHits <= 0) {
+ throw new IllegalArgumentException("numHits must be > 0, got " + numHits);
+ }
+ this.numHits = numHits;
+ this.collectors = new ArrayList<>();
+ }
+
+ @Override
+ public LargeNumHitsTopDocsCollector newCollector() {
+ LargeNumHitsTopDocsCollector collector = new LargeNumHitsTopDocsCollector(numHits);
+ this.collectors.add(collector);
+ return collector;
+ }
+
+ @Override
+ public TopDocs reduce(Collection<LargeNumHitsTopDocsCollector> collectors) throws IOException {
+ final TopDocs[] topDocs =
+ collectors.stream().map(LargeNumHitsTopDocsCollector::topDocs).toArray(TopDocs[]::new);
+ return TopDocs.merge(0, numHits, topDocs);
+ }
+
+ List<LargeNumHitsTopDocsCollector> getCollectors() {
+ return this.collectors;
+ }
+}
diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java
index bef8a07590c..7df2723e862 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java
@@ -20,6 +20,7 @@ package org.apache.lucene.sandbox.search;
import static org.apache.lucene.search.TopDocsCollector.EMPTY_TOPDOCS;
import java.io.IOException;
+import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@@ -90,20 +91,18 @@ public class TestLargeNumHitsTopDocsCollector extends LuceneTestCase {
}
}
- public void testTopDocs() throws IOException {
+ public void testHowMany() throws IOException {
IndexSearcher searcher = newSearcher(reader);
- LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(15);
- TopScoreDocCollectorManager regularCollectorManager =
- new TopScoreDocCollectorManager(15, Integer.MAX_VALUE);
-
- searcher.search(testQuery, largeCollector);
- TopDocs topDocs = searcher.search(testQuery, regularCollectorManager);
+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =
+ new LargeNumHitsTopDocsCollectorManager(15);
- assertEquals(largeCollector.totalHits, topDocs.totalHits.value());
+ searcher.search(testQuery, largeCollectorManager);
+ List<LargeNumHitsTopDocsCollector> collectorList = largeCollectorManager.getCollectors();
+ LargeNumHitsTopDocsCollector largeCollector =
+ collectorList.get(random().nextInt(0, collectorList.size()));
IllegalArgumentException expected =
expectThrows(IllegalArgumentException.class, () -> largeCollector.topDocs(-1));
-
assertTrue(expected.getMessage().contains("Number of hits requested must not be negative"));
assertEquals(EMPTY_TOPDOCS, largeCollector.topDocs(0));
@@ -112,72 +111,103 @@ public class TestLargeNumHitsTopDocsCollector extends LuceneTestCase {
public void testNoPQBuild() throws IOException {
IndexSearcher searcher = newSearcher(reader);
- LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(250_000);
+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =
+ new LargeNumHitsTopDocsCollectorManager(250_000);
TopScoreDocCollectorManager regularCollectorManager =
new TopScoreDocCollectorManager(reader.numDocs(), Integer.MAX_VALUE);
- searcher.search(testQuery, largeCollector);
- TopDocs topDocs = searcher.search(testQuery, regularCollectorManager);
-
- assertEquals(largeCollector.totalHits, topDocs.totalHits.value());
+ TopDocs largeTopDocs = searcher.search(testQuery, largeCollectorManager);
+ TopDocs regularTopDocs = searcher.search(testQuery, regularCollectorManager);
+ assertEquals(largeTopDocs.totalHits.value(), regularTopDocs.totalHits.value());
- assertNull(largeCollector.pq);
- assertNull(largeCollector.pqTop);
+ for (LargeNumHitsTopDocsCollector largeCollector : largeCollectorManager.getCollectors()) {
+ assertNull(largeCollector.pq);
+ assertNull(largeCollector.pqTop);
+ }
}
public void testPQBuild() throws IOException {
IndexSearcher searcher = newSearcher(reader);
- LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(50);
+ int requestedHitCount = random().nextInt(1, 1000);
+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =
+ new LargeNumHitsTopDocsCollectorManager(requestedHitCount);
TopScoreDocCollectorManager regularCollectorManager =
- new TopScoreDocCollectorManager(50, Integer.MAX_VALUE);
-
- searcher.search(testQuery, largeCollector);
- TopDocs topDocs = searcher.search(testQuery, regularCollectorManager);
+ new TopScoreDocCollectorManager(requestedHitCount, Integer.MAX_VALUE);
- assertEquals(largeCollector.totalHits, topDocs.totalHits.value());
+ TopDocs largeTopDocs = searcher.search(testQuery, largeCollectorManager);
+ TopDocs regularTopDocs = searcher.search(testQuery, regularCollectorManager);
+ assertEquals(largeTopDocs.totalHits.value(), regularTopDocs.totalHits.value());
- assertNotNull(largeCollector.pq);
- assertNotNull(largeCollector.pqTop);
+ for (LargeNumHitsTopDocsCollector largeCollector : largeCollectorManager.getCollectors()) {
+ if (largeCollector.totalHits > requestedHitCount) {
+ assertNotNull(largeCollector.pq);
+ assertNotNull(largeCollector.pqTop);
+ }
+ }
}
public void testNoPQHitsOrder() throws IOException {
IndexSearcher searcher = newSearcher(reader);
- LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(250_000);
+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =
+ new LargeNumHitsTopDocsCollectorManager(250_000);
TopScoreDocCollectorManager regularCollectorManager =
new TopScoreDocCollectorManager(reader.numDocs(), Integer.MAX_VALUE);
- searcher.search(testQuery, largeCollector);
- TopDocs topDocs = searcher.search(testQuery, regularCollectorManager);
-
- assertEquals(largeCollector.totalHits, topDocs.totalHits.value());
+ TopDocs largeTopDocs = searcher.search(testQuery, largeCollectorManager);
+ TopDocs regularTopDocs = searcher.search(testQuery, regularCollectorManager);
+ assertEquals(largeTopDocs.totalHits.value(), regularTopDocs.totalHits.value());
- assertNull(largeCollector.pq);
- assertNull(largeCollector.pqTop);
-
- topDocs = largeCollector.topDocs();
+ for (LargeNumHitsTopDocsCollector largeCollector : largeCollectorManager.getCollectors()) {
+ assertNull(largeCollector.pq);
+ assertNull(largeCollector.pqTop);
+ }
- if (topDocs.scoreDocs.length > 0) {
- float preScore = topDocs.scoreDocs[0].score;
- for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+ if (largeTopDocs.scoreDocs.length > 0) {
+ float preScore = largeTopDocs.scoreDocs[0].score;
+ for (ScoreDoc scoreDoc : largeTopDocs.scoreDocs) {
assert scoreDoc.score <= preScore;
preScore = scoreDoc.score;
}
}
}
+ public void testInvalidNumHitsInManagerConstructor() {
+ for (int n : new int[] {0, -1}) {
+ IllegalArgumentException e =
+ expectThrows(
+ IllegalArgumentException.class, () -> new LargeNumHitsTopDocsCollectorManager(n));
+ assertTrue(e.getMessage().contains("numHits must be > 0"));
+ }
+ }
+
+ public void testReduceWithZeroHitCollector() throws IOException {
+ Query termQuery = new TermQuery(new Term("field", "no-hit"));
+
+ IndexSearcher searcher = newSearcher(reader);
+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =
+ new LargeNumHitsTopDocsCollectorManager(50);
+ TopScoreDocCollectorManager regularCollectorManager =
+ new TopScoreDocCollectorManager(50, Integer.MAX_VALUE);
+
+ TopDocs largeTopDocs = searcher.search(termQuery, largeCollectorManager);
+ TopDocs regularTopDocs = searcher.search(termQuery, regularCollectorManager);
+
+ assertEquals(largeTopDocs.totalHits.value(), regularTopDocs.totalHits.value());
+ assertEquals(0, largeTopDocs.totalHits.value());
+ }
+
private void runNumHits(int numHits) throws IOException {
IndexSearcher searcher = newSearcher(reader);
- LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(numHits);
+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =
+ new LargeNumHitsTopDocsCollectorManager(numHits);
TopScoreDocCollectorManager regularCollectorManager =
new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE);
- searcher.search(testQuery, largeCollector);
-
- TopDocs firstTopDocs = largeCollector.topDocs();
- TopDocs secondTopDocs = searcher.search(testQuery, regularCollectorManager);
+ TopDocs largeTopDocs = searcher.search(testQuery, largeCollectorManager);
+ TopDocs regularTopDocs = searcher.search(testQuery, regularCollectorManager);
- assertEquals(largeCollector.totalHits, secondTopDocs.totalHits.value());
- assertEquals(firstTopDocs.scoreDocs.length, secondTopDocs.scoreDocs.length);
- CheckHits.checkEqual(testQuery, firstTopDocs.scoreDocs, secondTopDocs.scoreDocs);
+ assertEquals(largeTopDocs.totalHits.value(), regularTopDocs.totalHits.value());
+ assertEquals(largeTopDocs.scoreDocs.length, regularTopDocs.scoreDocs.length);
+ CheckHits.checkEqual(testQuery, largeTopDocs.scoreDocs, regularTopDocs.scoreDocs);
}
}