Apache Lucene open-source search software
LUCENE-10484: Add support for concurrent facets random sampling (#765)
Luca Cavanna7ed0f3d7adb5
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex b08b45d298e..9e10dc2d171 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -33,6 +33,9 @@ Improvements* LUCENE-10416: Update Korean Dictionary to mecab-ko-dic-2.1.1-20180720 for Nori.(Uihyun Kim)+* LUCENE-10484: Add support for concurrent random sampling by calling+ RandomSamplingFacetsCollector#createManager. (Luca Cavanna)+Optimizations---------------------(No changes)diff --git a/lucene/facet/src/java/org/apache/lucene/facet/RandomSamplingFacetsCollector.java b/lucene/facet/src/java/org/apache/lucene/facet/RandomSamplingFacetsCollector.javaindex 92a0c3133b1..de7c0e27323 100644--- a/lucene/facet/src/java/org/apache/lucene/facet/RandomSamplingFacetsCollector.java+++ b/lucene/facet/src/java/org/apache/lucene/facet/RandomSamplingFacetsCollector.java@@ -18,10 +18,12 @@ package org.apache.lucene.facet;import java.io.IOException;import java.util.ArrayList;+import java.util.Collection;import java.util.List;import org.apache.lucene.facet.FacetsConfig.DimConfig;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.Term;+import org.apache.lucene.search.CollectorManager;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.util.BitDocIdSet;@@ -255,4 +257,40 @@ public class RandomSamplingFacetsCollector extends FacetsCollector {public double getSamplingRate() {return samplingRate;}++ /**+ * Creates a {@link CollectorManager} for concurrent random sampling through {@link+ * RandomSamplingFacetsCollector}+ */+ public static CollectorManager<RandomSamplingFacetsCollector, RandomSamplingFacetsCollector>+ createManager(int sampleSize, long seed) {+ return new CollectorManager<>() {+ @Override+ public RandomSamplingFacetsCollector newCollector() {+ return new RandomSamplingFacetsCollector(sampleSize, seed);+ }++ @Override+ public RandomSamplingFacetsCollector reduce(+ Collection<RandomSamplingFacetsCollector> collectors) {+ if (collectors == null || collectors.size() == 0) {+ return new RandomSamplingFacetsCollector(sampleSize, seed);+ }+ if (collectors.size() == 1) {+ return collectors.iterator().next();+ }+ return new ReducedRandomSamplingFacetsCollector(sampleSize, seed, collectors);+ }+ };+ }++ private static class ReducedRandomSamplingFacetsCollector extends RandomSamplingFacetsCollector {+ ReducedRandomSamplingFacetsCollector(+ int sampleSize, long seed, Collection<RandomSamplingFacetsCollector> facetsCollectors) {+ super(sampleSize, seed);+ facetsCollectors.forEach(+ facetsCollector ->+ getOriginalMatchingDocs().addAll(facetsCollector.getOriginalMatchingDocs()));+ }+ }}diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestRandomSamplingFacetsCollector.java b/lucene/facet/src/test/org/apache/lucene/facet/TestRandomSamplingFacetsCollector.javaindex 26d80fc784f..df51e0afc38 100644--- a/lucene/facet/src/test/org/apache/lucene/facet/TestRandomSamplingFacetsCollector.java+++ b/lucene/facet/src/test/org/apache/lucene/facet/TestRandomSamplingFacetsCollector.java@@ -27,9 +27,9 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader;import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;import org.apache.lucene.index.Term;+import org.apache.lucene.search.CollectorManager;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.IndexSearcher;-import org.apache.lucene.search.MultiCollector;import org.apache.lucene.search.TermQuery;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.index.RandomIndexWriter;@@ -74,11 +74,11 @@ public class TestRandomSamplingFacetsCollector extends FacetTestCase {IOUtils.close(writer, taxoWriter);// Test empty results- RandomSamplingFacetsCollector collectRandomZeroResults =- new RandomSamplingFacetsCollector(numDocs / 10, random.nextLong());-+ CollectorManager<RandomSamplingFacetsCollector, RandomSamplingFacetsCollector> fcm =+ RandomSamplingFacetsCollector.createManager(numDocs / 10, random.nextLong());// There should be no divisions by zero- searcher.search(new TermQuery(new Term("EvenOdd", "NeverMatches")), collectRandomZeroResults);+ RandomSamplingFacetsCollector collectRandomZeroResults =+ searcher.search(new TermQuery(new Term("EvenOdd", "NeverMatches")), fcm);// There should be no divisions by zero and no null resultassertNotNull(collectRandomZeroResults.getMatchingDocs());@@ -93,13 +93,9 @@ public class TestRandomSamplingFacetsCollector extends FacetTestCase {// Use a query to select half of the documents.TermQuery query = new TermQuery(new Term("EvenOdd", "even"));- RandomSamplingFacetsCollector random10Percent =- new RandomSamplingFacetsCollector(- numDocs / 10, random.nextLong()); // 10% of total docs, 20% of the hits-- FacetsCollector fc = new FacetsCollector();-- searcher.search(query, MultiCollector.wrap(fc, random10Percent));+ // 10% of total docs, 20% of the hits+ fcm = RandomSamplingFacetsCollector.createManager(numDocs / 10, random.nextLong());+ RandomSamplingFacetsCollector random10Percent = searcher.search(query, fcm);final List<MatchingDocs> matchingDocs = random10Percent.getMatchingDocs();