Apache Lucene open-source search software
Introduce LargeNumHitsTopDocsCollectorManager (#15660)
gaobinlong3fe701ceaa5f
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 5dbdc67c4ff..f556fd58861 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -343,6 +343,8 @@ Improvements* GITHUB#16000: Clarify that Accountable#ramBytesUsed() reports JVM heap memory only. (Luca Cavanna)+* GITHUB#15660: Introduce LargeNumHitsTopDocsCollectorManager to parallelize search when using LargeNumHitsTopDocsCollector. (Binlong Gao)+Optimizations---------------------* GITHUB#15861: Optimise PhraseScorer by short circuiting non competitive documents in TOP_SCORES mode. (Prithvi S)diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/LargeNumHitsTopDocsCollectorManager.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/LargeNumHitsTopDocsCollectorManager.javanew file mode 100644index 00000000000..e874b5894c3--- /dev/null+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/LargeNumHitsTopDocsCollectorManager.java@@ -0,0 +1,74 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.sandbox.search;++import java.io.IOException;+import java.util.ArrayList;+import java.util.Collection;+import java.util.List;+import org.apache.lucene.search.CollectorManager;+import org.apache.lucene.search.TopDocs;++/**+ * CollectorManager for {@link LargeNumHitsTopDocsCollector} that enables concurrent collection of+ * top docs across multiple segments.+ *+ * <p>Example usage:+ *+ * <pre class="prettyprint">+ * IndexSearcher searcher = ...; // your IndexSearcher+ * TopDocs topDocs = searcher.search(new MatchAllDocsQuery(),+ * new LargeNumHitsTopDocsCollectorManager(100_000));+ * System.out.println("Total hits: " + topDocs.totalHits.value());+ * </pre>+ */+public class LargeNumHitsTopDocsCollectorManager+ implements CollectorManager<LargeNumHitsTopDocsCollector, TopDocs> {+ private final int numHits;+ private final List<LargeNumHitsTopDocsCollector> collectors;++ /**+ * Creates a new {@link LargeNumHitsTopDocsCollectorManager} given the number of hits to collect.+ *+ * @param numHits the number of results to collect.+ */+ public LargeNumHitsTopDocsCollectorManager(int numHits) {+ if (numHits <= 0) {+ throw new IllegalArgumentException("numHits must be > 0, got " + numHits);+ }+ this.numHits = numHits;+ this.collectors = new ArrayList<>();+ }++ @Override+ public LargeNumHitsTopDocsCollector newCollector() {+ LargeNumHitsTopDocsCollector collector = new LargeNumHitsTopDocsCollector(numHits);+ this.collectors.add(collector);+ return collector;+ }++ @Override+ public TopDocs reduce(Collection<LargeNumHitsTopDocsCollector> collectors) throws IOException {+ final TopDocs[] topDocs =+ collectors.stream().map(LargeNumHitsTopDocsCollector::topDocs).toArray(TopDocs[]::new);+ return TopDocs.merge(0, numHits, topDocs);+ }++ List<LargeNumHitsTopDocsCollector> getCollectors() {+ return this.collectors;+ }+}diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.javaindex bef8a07590c..7df2723e862 100644--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java@@ -20,6 +20,7 @@ package org.apache.lucene.sandbox.search;import static org.apache.lucene.search.TopDocsCollector.EMPTY_TOPDOCS;import java.io.IOException;+import java.util.List;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.index.IndexReader;@@ -90,20 +91,18 @@ public class TestLargeNumHitsTopDocsCollector extends LuceneTestCase {}}- public void testTopDocs() throws IOException {+ public void testHowMany() throws IOException {IndexSearcher searcher = newSearcher(reader);- LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(15);- TopScoreDocCollectorManager regularCollectorManager =- new TopScoreDocCollectorManager(15, Integer.MAX_VALUE);-- searcher.search(testQuery, largeCollector);- TopDocs topDocs = searcher.search(testQuery, regularCollectorManager);+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =+ new LargeNumHitsTopDocsCollectorManager(15);- assertEquals(largeCollector.totalHits, topDocs.totalHits.value());+ searcher.search(testQuery, largeCollectorManager);+ List<LargeNumHitsTopDocsCollector> collectorList = largeCollectorManager.getCollectors();+ LargeNumHitsTopDocsCollector largeCollector =+ collectorList.get(random().nextInt(0, collectorList.size()));IllegalArgumentException expected =expectThrows(IllegalArgumentException.class, () -> largeCollector.topDocs(-1));-assertTrue(expected.getMessage().contains("Number of hits requested must not be negative"));assertEquals(EMPTY_TOPDOCS, largeCollector.topDocs(0));@@ -112,72 +111,103 @@ public class TestLargeNumHitsTopDocsCollector extends LuceneTestCase {public void testNoPQBuild() throws IOException {IndexSearcher searcher = newSearcher(reader);- LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(250_000);+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =+ new LargeNumHitsTopDocsCollectorManager(250_000);TopScoreDocCollectorManager regularCollectorManager =new TopScoreDocCollectorManager(reader.numDocs(), Integer.MAX_VALUE);- searcher.search(testQuery, largeCollector);- TopDocs topDocs = searcher.search(testQuery, regularCollectorManager);-- assertEquals(largeCollector.totalHits, topDocs.totalHits.value());+ TopDocs largeTopDocs = searcher.search(testQuery, largeCollectorManager);+ TopDocs regularTopDocs = searcher.search(testQuery, regularCollectorManager);+ assertEquals(largeTopDocs.totalHits.value(), regularTopDocs.totalHits.value());- assertNull(largeCollector.pq);- assertNull(largeCollector.pqTop);+ for (LargeNumHitsTopDocsCollector largeCollector : largeCollectorManager.getCollectors()) {+ assertNull(largeCollector.pq);+ assertNull(largeCollector.pqTop);+ }}public void testPQBuild() throws IOException {IndexSearcher searcher = newSearcher(reader);- LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(50);+ int requestedHitCount = random().nextInt(1, 1000);+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =+ new LargeNumHitsTopDocsCollectorManager(requestedHitCount);TopScoreDocCollectorManager regularCollectorManager =- new TopScoreDocCollectorManager(50, Integer.MAX_VALUE);-- searcher.search(testQuery, largeCollector);- TopDocs topDocs = searcher.search(testQuery, regularCollectorManager);+ new TopScoreDocCollectorManager(requestedHitCount, Integer.MAX_VALUE);- assertEquals(largeCollector.totalHits, topDocs.totalHits.value());+ TopDocs largeTopDocs = searcher.search(testQuery, largeCollectorManager);+ TopDocs regularTopDocs = searcher.search(testQuery, regularCollectorManager);+ assertEquals(largeTopDocs.totalHits.value(), regularTopDocs.totalHits.value());- assertNotNull(largeCollector.pq);- assertNotNull(largeCollector.pqTop);+ for (LargeNumHitsTopDocsCollector largeCollector : largeCollectorManager.getCollectors()) {+ if (largeCollector.totalHits > requestedHitCount) {+ assertNotNull(largeCollector.pq);+ assertNotNull(largeCollector.pqTop);+ }+ }}public void testNoPQHitsOrder() throws IOException {IndexSearcher searcher = newSearcher(reader);- LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(250_000);+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =+ new LargeNumHitsTopDocsCollectorManager(250_000);TopScoreDocCollectorManager regularCollectorManager =new TopScoreDocCollectorManager(reader.numDocs(), Integer.MAX_VALUE);- searcher.search(testQuery, largeCollector);- TopDocs topDocs = searcher.search(testQuery, regularCollectorManager);-- assertEquals(largeCollector.totalHits, topDocs.totalHits.value());+ TopDocs largeTopDocs = searcher.search(testQuery, largeCollectorManager);+ TopDocs regularTopDocs = searcher.search(testQuery, regularCollectorManager);+ assertEquals(largeTopDocs.totalHits.value(), regularTopDocs.totalHits.value());- assertNull(largeCollector.pq);- assertNull(largeCollector.pqTop);-- topDocs = largeCollector.topDocs();+ for (LargeNumHitsTopDocsCollector largeCollector : largeCollectorManager.getCollectors()) {+ assertNull(largeCollector.pq);+ assertNull(largeCollector.pqTop);+ }- if (topDocs.scoreDocs.length > 0) {- float preScore = topDocs.scoreDocs[0].score;- for (ScoreDoc scoreDoc : topDocs.scoreDocs) {+ if (largeTopDocs.scoreDocs.length > 0) {+ float preScore = largeTopDocs.scoreDocs[0].score;+ for (ScoreDoc scoreDoc : largeTopDocs.scoreDocs) {assert scoreDoc.score <= preScore;preScore = scoreDoc.score;}}}+ public void testInvalidNumHitsInManagerConstructor() {+ for (int n : new int[] {0, -1}) {+ IllegalArgumentException e =+ expectThrows(+ IllegalArgumentException.class, () -> new LargeNumHitsTopDocsCollectorManager(n));+ assertTrue(e.getMessage().contains("numHits must be > 0"));+ }+ }++ public void testReduceWithZeroHitCollector() throws IOException {+ Query termQuery = new TermQuery(new Term("field", "no-hit"));++ IndexSearcher searcher = newSearcher(reader);+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =+ new LargeNumHitsTopDocsCollectorManager(50);+ TopScoreDocCollectorManager regularCollectorManager =+ new TopScoreDocCollectorManager(50, Integer.MAX_VALUE);++ TopDocs largeTopDocs = searcher.search(termQuery, largeCollectorManager);+ TopDocs regularTopDocs = searcher.search(termQuery, regularCollectorManager);++ assertEquals(largeTopDocs.totalHits.value(), regularTopDocs.totalHits.value());+ assertEquals(0, largeTopDocs.totalHits.value());+ }+private void runNumHits(int numHits) throws IOException {IndexSearcher searcher = newSearcher(reader);- LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(numHits);+ LargeNumHitsTopDocsCollectorManager largeCollectorManager =+ new LargeNumHitsTopDocsCollectorManager(numHits);TopScoreDocCollectorManager regularCollectorManager =new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE);- searcher.search(testQuery, largeCollector);-- TopDocs firstTopDocs = largeCollector.topDocs();- TopDocs secondTopDocs = searcher.search(testQuery, regularCollectorManager);+ TopDocs largeTopDocs = searcher.search(testQuery, largeCollectorManager);+ TopDocs regularTopDocs = searcher.search(testQuery, regularCollectorManager);- assertEquals(largeCollector.totalHits, secondTopDocs.totalHits.value());- assertEquals(firstTopDocs.scoreDocs.length, secondTopDocs.scoreDocs.length);- CheckHits.checkEqual(testQuery, firstTopDocs.scoreDocs, secondTopDocs.scoreDocs);+ assertEquals(largeTopDocs.totalHits.value(), regularTopDocs.totalHits.value());+ assertEquals(largeTopDocs.scoreDocs.length, regularTopDocs.scoreDocs.length);+ CheckHits.checkEqual(testQuery, largeTopDocs.scoreDocs, regularTopDocs.scoreDocs);}}