Apache Solr open-source search software
SOLR-14185: introduce DocSet.iterator(LeafReaderContext), replacing Filter where possible (#2)
Michael Gibneybec4ed890b02
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 3b1eb261bb1..179e18c4182 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -116,6 +116,10 @@ when told to. The admin UI now tells it to. (Nazerke Seidan, David Smiley)* SOLR-15322: Solr releases now contain everything needed to build runable docker images (Houston Putman, hossman)+* SOLR-14185: Added DocSet.iterator(LeafReaderContext) and some related changes that may add+ a minor performance boost to some cases (e.g. interval facets). Reduced need for Filter.java.+ (Michael Gibney, David Smiley)+Other Changes----------------------* SOLR-14656: Autoscaling framework removed (Ishan Chattopadhyaya, noble, Ilan Ginzburg)diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.javaindex 70dbab46eeb..3c6859c88d3 100644--- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java+++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java@@ -57,7 +57,6 @@ import org.apache.solr.search.DocSetBuilder;import org.apache.solr.search.DocSetProducer;import org.apache.solr.search.DocSetUtil;import org.apache.solr.search.ExtendedQueryBase;-import org.apache.solr.search.Filter;import org.apache.solr.search.SolrIndexSearcher;/** @lucene.experimental */@@ -333,6 +332,22 @@ public final class SolrRangeQuery extends ExtendedQueryBase implements DocSetPro}}+ private static class SegmentDocIdSet extends DocIdSet {+ private final DocSet docs;+ private final LeafReaderContext ctx;+ private SegmentDocIdSet(DocSet docs, LeafReaderContext ctx) {+ this.docs = docs;+ this.ctx = ctx;+ }+ @Override+ public long ramBytesUsed() {+ return docs.ramBytesUsed();+ }+ @Override+ public DocIdSetIterator iterator() throws IOException {+ return docs.iterator(ctx);+ }+ }// adapted from MultiTermQueryConstantScoreWrapperclass ConstWeight extends ConstantScoreWeight {@@ -341,7 +356,7 @@ public final class SolrRangeQuery extends ExtendedQueryBase implements DocSetProfinal IndexSearcher searcher;final ScoreMode scoreMode;boolean checkedFilterCache;- Filter filter;+ DocSet answer;final SegState[] segStates;@@ -400,17 +415,14 @@ public final class SolrRangeQuery extends ExtendedQueryBase implements DocSetProdoCheck = false;} else {solrSearcher = (SolrIndexSearcher)searcher;- DocSet answer = solrSearcher.getFilterCache().get(SolrRangeQuery.this);- if (answer != null) {- filter = answer.getTopFilter();- }+ answer = solrSearcher.getFilterCache().get(SolrRangeQuery.this);}} else {doCheck = false;}- if (filter != null) {- return segStates[context.ord] = new SegState(filter.getDocIdSet(context, null));+ if (answer != null) {+ return segStates[context.ord] = new SegState(new SegmentDocIdSet(answer, context));}final Terms terms = context.reader().terms(SolrRangeQuery.this.getField());@@ -439,10 +451,9 @@ public final class SolrRangeQuery extends ExtendedQueryBase implements DocSetPro// Too many terms for boolean query...if (doCheck) {- DocSet answer = createDocSet(solrSearcher, count);+ answer = createDocSet(solrSearcher, count);solrSearcher.getFilterCache().put(SolrRangeQuery.this, answer);- filter = answer.getTopFilter();- return segStates[context.ord] = new SegState(filter.getDocIdSet(context, null));+ return segStates[context.ord] = new SegState(new SegmentDocIdSet(answer, context));}/* FUTURE: reuse term states in the future to help build DocSet, use collected count so far...diff --git a/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java b/solr/core/src/java/org/apache/solr/request/DocValuesFacets.javaindex 7a002294a06..c140a450cd3 100644--- a/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java+++ b/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java@@ -27,7 +27,6 @@ import org.apache.lucene.index.MultiDocValues;import org.apache.lucene.index.OrdinalMap;import org.apache.lucene.index.SortedDocValues;import org.apache.lucene.index.SortedSetDocValues;-import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.BytesRefBuilder;@@ -39,7 +38,6 @@ import org.apache.solr.common.util.NamedList;import org.apache.solr.schema.FieldType;import org.apache.solr.schema.SchemaField;import org.apache.solr.search.DocSet;-import org.apache.solr.search.Filter;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.search.facet.FacetDebugInfo;import org.apache.solr.util.LongPriorityQueue;@@ -129,15 +127,10 @@ public class DocValuesFacets {fdebug.putInfoItem("numBuckets", nTerms);}- Filter filter = docs.getTopFilter();List<LeafReaderContext> leaves = searcher.getTopReaderContext().leaves();for (int subIndex = 0; subIndex < leaves.size(); subIndex++) {LeafReaderContext leaf = leaves.get(subIndex);- DocIdSet dis = filter.getDocIdSet(leaf, null); // solr docsets already exclude any deleted docs- DocIdSetIterator disi = null;- if (dis != null) {- disi = dis.iterator();- }+ final DocIdSetIterator disi = docs.iterator(leaf); // solr docsets already exclude any deleted docsif (disi != null) {if (multiValued) {SortedSetDocValues sub = leaf.reader().getSortedSetDocValues(fieldName);diff --git a/solr/core/src/java/org/apache/solr/request/DocValuesStats.java b/solr/core/src/java/org/apache/solr/request/DocValuesStats.javaindex dae9943fb7d..ed4e9da1d11 100644--- a/solr/core/src/java/org/apache/solr/request/DocValuesStats.java+++ b/solr/core/src/java/org/apache/solr/request/DocValuesStats.java@@ -27,7 +27,6 @@ import org.apache.lucene.index.MultiDocValues;import org.apache.lucene.index.OrdinalMap;import org.apache.lucene.index.SortedDocValues;import org.apache.lucene.index.SortedSetDocValues;-import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.LongValues;@@ -39,7 +38,6 @@ import org.apache.solr.handler.component.StatsValuesFactory;import org.apache.solr.schema.FieldType;import org.apache.solr.schema.SchemaField;import org.apache.solr.search.DocSet;-import org.apache.solr.search.Filter;import org.apache.solr.search.SolrIndexSearcher;/**@@ -108,17 +106,11 @@ public class DocValuesStats {// going to collect counts for.final int[] counts = new int[nTerms];- Filter filter = docs.getTopFilter();List<LeafReaderContext> leaves = searcher.getTopReaderContext().leaves();for (int subIndex = 0; subIndex < leaves.size(); subIndex++) {LeafReaderContext leaf = leaves.get(subIndex);- DocIdSet dis = filter.getDocIdSet(leaf, null); // solr docsets already exclude any deleted docs- DocIdSetIterator disi = null;-- if (dis != null) {- disi = dis.iterator();- }+ final DocIdSetIterator disi = docs.iterator(leaf); // solr docsets already exclude any deleted docsif (disi != null) {int docBase = leaf.docBase;diff --git a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java b/solr/core/src/java/org/apache/solr/request/IntervalFacets.javaindex 188c07a61cd..1e66d770e30 100644--- a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java+++ b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java@@ -32,9 +32,7 @@ import org.apache.lucene.index.NumericDocValues;import org.apache.lucene.index.SortedDocValues;import org.apache.lucene.index.SortedNumericDocValues;import org.apache.lucene.index.SortedSetDocValues;-import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.DocIdSetIterator;-import org.apache.lucene.util.Bits;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.NumericUtils;import org.apache.solr.common.SolrException;@@ -47,7 +45,6 @@ import org.apache.solr.schema.PointField;import org.apache.solr.schema.SchemaField;import org.apache.solr.search.DocIterator;import org.apache.solr.search.DocSet;-import org.apache.solr.search.Filter;import org.apache.solr.search.QueryParsing;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.search.SyntaxError;@@ -274,15 +271,10 @@ public class IntervalFacets implements Iterable<FacetInterval> {}private void getCountString() throws IOException {- Filter filter = docs.getTopFilter();List<LeafReaderContext> leaves = searcher.getTopReaderContext().leaves();for (int subIndex = 0; subIndex < leaves.size(); subIndex++) {LeafReaderContext leaf = leaves.get(subIndex);- DocIdSet dis = filter.getDocIdSet(leaf, null); // solr docsets already exclude any deleted docs- if (dis == null) {- continue;- }- DocIdSetIterator disi = dis.iterator();+ final DocIdSetIterator disi = docs.iterator(leaf); // solr docsets already exclude any deleted docsif (disi != null) {if (schemaField.multiValued()) {SortedSetDocValues sub = leaf.reader().getSortedSetDocValues(schemaField.getName());@@ -292,16 +284,16 @@ public class IntervalFacets implements Iterable<FacetInterval> {final SortedDocValues singleton = DocValues.unwrapSingleton(sub);if (singleton != null) {// some codecs may optimize SORTED_SET storage for single-valued fields- accumIntervalsSingle(singleton, disi, dis.bits());+ accumIntervalsSingle(singleton, disi);} else {- accumIntervalsMulti(sub, disi, dis.bits());+ accumIntervalsMulti(sub, disi);}} else {SortedDocValues sub = leaf.reader().getSortedDocValues(schemaField.getName());if (sub == null) {continue;}- accumIntervalsSingle(sub, disi, dis.bits());+ accumIntervalsSingle(sub, disi);}}}@@ -346,8 +338,7 @@ public class IntervalFacets implements Iterable<FacetInterval> {}}- private void accumIntervalsMulti(SortedSetDocValues ssdv,- DocIdSetIterator disi, Bits bits) throws IOException {+ private void accumIntervalsMulti(SortedSetDocValues ssdv, DocIdSetIterator disi) throws IOException {// First update the ordinals in the intervals for this segmentfor (FacetInterval interval : intervals) {interval.updateContext(ssdv);@@ -355,9 +346,6 @@ public class IntervalFacets implements Iterable<FacetInterval> {int doc;while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {- if (bits != null && bits.get(doc) == false) {- continue;- }if (doc > ssdv.docID()) {ssdv.advance(doc);}@@ -397,16 +385,13 @@ public class IntervalFacets implements Iterable<FacetInterval> {}}- private void accumIntervalsSingle(SortedDocValues sdv, DocIdSetIterator disi, Bits bits) throws IOException {+ private void accumIntervalsSingle(SortedDocValues sdv, DocIdSetIterator disi) throws IOException {// First update the ordinals in the intervals to this segmentfor (FacetInterval interval : intervals) {interval.updateContext(sdv);}int doc;while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {- if (bits != null && bits.get(doc) == false) {- continue;- }if (doc > sdv.docID()) {sdv.advance(doc);}diff --git a/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java b/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.javaindex 4ff769f96dd..cb4be5fcc5c 100644--- a/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java+++ b/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java@@ -31,7 +31,6 @@ import org.apache.lucene.index.DocValues;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.SortedDocValues;import org.apache.lucene.index.TermsEnum;-import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.BytesRefBuilder;@@ -43,7 +42,6 @@ import org.apache.solr.common.params.FacetParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.schema.FieldType;import org.apache.solr.search.DocSet;-import org.apache.solr.search.Filter;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.util.BoundedTreeSet;@@ -65,8 +63,6 @@ class PerSegmentSingleValuedFaceting {private final Predicate<BytesRef> termFilter;- Filter baseSet;-int nThreads;public PerSegmentSingleValuedFaceting(SolrIndexSearcher searcher, DocSet docs, String fieldName, int offset, int limit, int mincount, boolean missing, String sort, String prefix, String contains, boolean ignoreCase) {@@ -95,9 +91,6 @@ class PerSegmentSingleValuedFaceting {CompletionService<SegFacet> completionService = new ExecutorCompletionService<>(executor);- // reuse the translation logic to go from top level set to per-segment set- baseSet = docs.getTopFilter();-final List<LeafReaderContext> leaves = searcher.getTopReaderContext().leaves();// The list of pending tasks that aren't immediately submitted// TODO: Is there a completion service, or a delegating executor that can@@ -301,8 +294,12 @@ class PerSegmentSingleValuedFaceting {// count collection array only needs to be as big as the number of terms we are// going to collect counts for.final int[] counts = this.counts = new int[nTerms];- DocIdSet idSet = baseSet.getDocIdSet(context, null); // this set only includes live docs- DocIdSetIterator iter = idSet.iterator();+ final DocIdSetIterator iter = docs.iterator(context); // this set only includes live docs++ if (iter == null) {+ // `DocIdSet.iterator()` can return null instead of empty DocIdSetIterator+ return;+ }if (prefix==null) {// specialized version when collecting counts for all termsdiff --git a/solr/core/src/java/org/apache/solr/search/BitDocSet.java b/solr/core/src/java/org/apache/solr/search/BitDocSet.javaindex 0644a729ec5..6597659e2e5 100644--- a/solr/core/src/java/org/apache/solr/search/BitDocSet.java+++ b/solr/core/src/java/org/apache/solr/search/BitDocSet.java@@ -20,12 +20,10 @@ import java.util.Collection;import java.util.Collections;import java.util.Objects;-import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.util.Accountable;-import org.apache.lucene.util.BitDocIdSet;import org.apache.lucene.util.BitSetIterator;import org.apache.lucene.util.Bits;import org.apache.lucene.util.FixedBitSet;@@ -237,6 +235,82 @@ public class BitDocSet extends DocSet {return new BitDocSet(bits.clone(), size);}+ @Override+ public DocIdSetIterator iterator(LeafReaderContext context) {+ if (context.isTopLevel) {+ switch (size) {+ case 0:+ return null;+ default:+ // we have an explicit size; use it+ return new BitSetIterator(bits, size);+ case -1:+ // size has not been computed; use bits.length() as an upper bound on cost+ final int maxSize = bits.length();+ if (maxSize < 1) {+ return null;+ } else {+ return new BitSetIterator(bits, maxSize);+ }+ }+ }++ final int maxDoc = context.reader().maxDoc();+ if (maxDoc < 1) {+ // entirely empty segment; verified this actually happens+ return null;+ }++ final int base = context.docBase;+ final int max = base + maxDoc; // one past the max doc in this segment.+ final FixedBitSet bs = bits;++ return new DocIdSetIterator() {+ int pos = base - 1;+ int adjustedDoc = -1;++ @Override+ public int docID() {+ return adjustedDoc;+ }++ @Override+ public int nextDoc() {+ int next = pos + 1;+ if (next >= max) {+ return adjustedDoc = NO_MORE_DOCS;+ } else {+ pos = bs.nextSetBit(next);+ return adjustedDoc = pos < max ? pos - base : NO_MORE_DOCS;+ }+ }++ @Override+ public int advance(int target) {+ if (target == NO_MORE_DOCS) return adjustedDoc = NO_MORE_DOCS;+ int adjusted = target + base;+ if (adjusted >= max) {+ return adjustedDoc = NO_MORE_DOCS;+ } else {+ pos = bs.nextSetBit(adjusted);+ return adjustedDoc = pos < max ? pos - base : NO_MORE_DOCS;+ }+ }++ @Override+ public long cost() {+ // we don't want to actually compute cardinality, but+ // if it's already been computed, we use it (pro-rated for the segment)+ int maxDoc = max - base;+ if (size != -1) {+ return (long) (size * ((FixedBitSet.bits2words(maxDoc) << 6) / (float) bs.length()));+ } else {+ return maxDoc;+ }+ }+ };+ }+@Overridepublic Filter getTopFilter() {// TODO: if cardinality isn't cached, do a quick measure of sparseness@@ -247,73 +321,30 @@ public class BitDocSet extends DocSet {@Overridepublic DocIdSet getDocIdSet(final LeafReaderContext context, final Bits acceptDocs) {- LeafReader reader = context.reader();// all Solr DocSets that are used as filters only include live docs- final Bits acceptDocs2 = acceptDocs == null ? null : (reader.getLiveDocs() == acceptDocs ? null : acceptDocs);-- if (context.isTopLevel) {- return BitsFilteredDocIdSet.wrap(new BitDocIdSet(bs), acceptDocs);- }-- final int base = context.docBase;- final int max = base + reader.maxDoc(); // one past the max doc in this segment.+ final Bits acceptDocs2 = acceptDocs == null ? null : (context.reader().getLiveDocs() == acceptDocs ? null : acceptDocs);return BitsFilteredDocIdSet.wrap(new DocIdSet() {@Overridepublic DocIdSetIterator iterator() {- return new DocIdSetIterator() {- int pos = base - 1;- int adjustedDoc = -1;-- @Override- public int docID() {- return adjustedDoc;- }-- @Override- public int nextDoc() {- int next = pos+1;- if (next >= max) {- return adjustedDoc = NO_MORE_DOCS;- } else {- pos = bs.nextSetBit(next);- return adjustedDoc = pos < max ? pos - base : NO_MORE_DOCS;- }- }-- @Override- public int advance(int target) {- if (target == NO_MORE_DOCS) return adjustedDoc = NO_MORE_DOCS;- int adjusted = target + base;- if (adjusted >= max) {- return adjustedDoc = NO_MORE_DOCS;- } else {- pos = bs.nextSetBit(adjusted);- return adjustedDoc = pos < max ? pos - base : NO_MORE_DOCS;- }- }-- @Override- public long cost() {- // we don't want to actually compute cardinality, but- // if it's already been computed, we use it (pro-rated for the segment)- int maxDoc = max-base;- if (size != -1) {- return (long)(size * ((FixedBitSet.bits2words(maxDoc)<<6) / (float)bs.length()));- } else {- return maxDoc;- }- }- };+ return BitDocSet.this.iterator(context);}@Overridepublic long ramBytesUsed() {- return bs.ramBytesUsed();+ return BitDocSet.this.ramBytesUsed();}@Overridepublic Bits bits() {+ if (context.isTopLevel) {+ return bits;+ }++ final int base = context.docBase;+ final int length = context.reader().maxDoc();+ final FixedBitSet bs = bits;+return new Bits() {@Overridepublic boolean get(int index) {@@ -322,7 +353,7 @@ public class BitDocSet extends DocSet {@Overridepublic int length() {- return max-base;+ return length;}};}diff --git a/solr/core/src/java/org/apache/solr/search/DocSet.java b/solr/core/src/java/org/apache/solr/search/DocSet.javaindex 1ebb9462a31..157f67f0240 100644--- a/solr/core/src/java/org/apache/solr/search/DocSet.java+++ b/solr/core/src/java/org/apache/solr/search/DocSet.java@@ -16,6 +16,8 @@*/package org.apache.solr.search;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.util.Accountable;import org.apache.lucene.util.Bits;import org.apache.lucene.util.FixedBitSet;@@ -63,6 +65,12 @@ public abstract class DocSet implements Accountable, Cloneable /* extends Collec//TODO switch to DocIdSetIterator in Solr 9?public abstract DocIterator iterator();+ /**+ * Returns an ordered iterator of the documents in the set for the specified {@link LeafReaderContext}.+ * <b>NOTE:</b> may return null if there are no matching documents for this leaf.+ */+ public abstract DocIdSetIterator iterator(LeafReaderContext ctx);+/*** Returns the intersection of this set with another set. Neither set is modified - a new DocSet is* created and returned.diff --git a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.javaindex c66e079170d..2d3573dd894 100644--- a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java+++ b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java@@ -561,21 +561,16 @@ abstract class PointSetQuery extends Query implements DocSetProducer, Accountabl@Overridepublic final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {return new ConstantScoreWeight(this, boost) {- Filter filter;+ DocSet docs;@Overridepublic Scorer scorer(LeafReaderContext context) throws IOException {- if (filter == null) {- DocSet set = getDocSet(searcher);- filter = set.getTopFilter();+ if (docs == null) {+ docs = getDocSet(searcher);}// Although this set only includes live docs, other filters can be pushed down to queries.- DocIdSet readerSet = filter.getDocIdSet(context, null);- if (readerSet == null) {- return null;- }- DocIdSetIterator readerSetIterator = readerSet.iterator();+ DocIdSetIterator readerSetIterator = docs.iterator(context);if (readerSetIterator == null) {return null;}diff --git a/solr/core/src/java/org/apache/solr/search/JoinQuery.java b/solr/core/src/java/org/apache/solr/search/JoinQuery.javaindex 7959a4c1d69..e179aeb565d 100644--- a/solr/core/src/java/org/apache/solr/search/JoinQuery.java+++ b/solr/core/src/java/org/apache/solr/search/JoinQuery.java@@ -33,7 +33,6 @@ import org.apache.lucene.index.Terms;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.search.ConstantScoreScorer;import org.apache.lucene.search.ConstantScoreWeight;-import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Query;@@ -161,12 +160,10 @@ class JoinQuery extends Query {}DocSet resultSet;- Filter filter;-@Overridepublic Scorer scorer(LeafReaderContext context) throws IOException {- if (filter == null) {+ if (resultSet == null) {boolean debug = rb != null && rb.isDebug();RTimer timer = (debug ? new RTimer() : null);resultSet = getDocSet();@@ -192,16 +189,10 @@ class JoinQuery extends Query {// TODO: perhaps synchronize addDebug in the future...rb.addDebug(dbg, "join", JoinQuery.this.toString());}-- filter = resultSet.getTopFilter();}// Although this set only includes live docs, other filters can be pushed down to queries.- DocIdSet readerSet = filter.getDocIdSet(context, null);- if (readerSet == null) {- return null;- }- DocIdSetIterator readerSetIterator = readerSet.iterator();+ DocIdSetIterator readerSetIterator = resultSet.iterator(context);if (readerSetIterator == null) {return null;}diff --git a/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java b/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.javaindex f65ee17e33d..94e3d6f5270 100644--- a/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java+++ b/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java@@ -16,12 +16,15 @@*/package org.apache.solr.search;+import java.util.Arrays;import java.util.Collection;import java.util.Collections;+import java.util.List;import com.carrotsearch.hppc.IntHashSet;-import org.apache.lucene.index.LeafReader;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.ReaderUtil;import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.util.Accountable;@@ -404,6 +407,9 @@ public class SortedIntDocSet extends DocSet {int doc = docs[i];if (other.exists(doc)) arr[icount++] = doc;}+ if (icount == docs.length) {+ return this; // no change+ }return new SortedIntDocSet(arr,icount);}@@ -411,6 +417,9 @@ public class SortedIntDocSet extends DocSet {int maxsz = Math.min(docs.length, otherDocs.length);int[] arr = new int[maxsz];int sz = intersection(docs, docs.length, otherDocs, otherDocs.length, arr);+ if (sz == docs.length) {+ return this; // no change+ }return new SortedIntDocSet(arr,sz);}@@ -532,12 +541,18 @@ public class SortedIntDocSet extends DocSet {int doc = docs[i];if (!other.exists(doc)) arr[count++] = doc;}+ if (count == docs.length) {+ return this; // no change+ }return new SortedIntDocSet(arr,count);}int[] otherDocs = ((SortedIntDocSet)other).docs;int[] arr = new int[docs.length];int sz = andNot(docs, docs.length, otherDocs, otherDocs.length, arr);+ if (sz == docs.length) {+ return this; // no change+ }return new SortedIntDocSet(arr,sz);}@@ -643,25 +658,6 @@ public class SortedIntDocSet extends DocSet {return bitSet;}- public static int findIndex(int[] arr, int value, int low, int high) {- // binary search- while (low <= high) {- int mid = (low+high) >>> 1;- int found = arr[mid];-- if (found < value) {- low = mid+1;- }- else if (found > value) {- high = mid-1;- }- else {- return mid;- }- }- return low;- }-@Overridepublic DocSet union(DocSet other) {// TODO could be more efficient if both are SortedIntDocSet@@ -671,107 +667,112 @@ public class SortedIntDocSet extends DocSet {return new BitDocSet(newbits);}+ private volatile int[] cachedOrdIdxMap; // idx of first doc _beyond_ the corresponding seg+ private int[] getOrdIdxMap(LeafReaderContext ctx) {+ final int[] cached = cachedOrdIdxMap;+ if (cached != null) {+ return cached;+ } else {+ List<LeafReaderContext> leaves = ReaderUtil.getTopLevelContext(ctx).leaves();+ final int[] ret = new int[leaves.size()];+ int lastLimit = 0;+ int lastLimitDoc = docs[0]; // docs.length != 0+ for (LeafReaderContext lrc : leaves) {+ // sanity check that initial `lastLimit*` values are valid (and consequently that our initial+ // setting of `startIdx` for context.ord==0 won't inadvertently include invalid docs).+ assert lrc.ord != 0 || lastLimitDoc >= lrc.docBase;+ final int max = lrc.docBase + lrc.reader().maxDoc(); // one past the max doc in this segment.+ if (lastLimitDoc >= max) {+ ret[lrc.ord] = lastLimit;+ continue;+ }+ assert lastLimitDoc >= lrc.docBase;+ final int nextLimit = Arrays.binarySearch(docs, lastLimit + 1, docs.length, max);+ lastLimit = nextLimit < 0 ? ~nextLimit : nextLimit;+ lastLimitDoc = lastLimit < docs.length ? docs[lastLimit] : DocIdSetIterator.NO_MORE_DOCS;+ ret[lrc.ord] = lastLimit;+ }+ return cachedOrdIdxMap = ret; // set/replace atomically after building+ }+ }+@Override- public Filter getTopFilter() {- return new Filter() {- int lastEndIdx = 0;+ public DocIdSetIterator iterator(LeafReaderContext context) {++ if (docs.length == 0 || context.reader().maxDoc() < 1) {+ // empty docset or entirely empty segment (verified that the latter actually happens)+ // NOTE: wrt the "empty docset" case, this is not just an optimization; this shortcircuits also+ // to prevent the static DocSet.EmptyLazyHolder.INSTANCE from having cachedOrdIdxMap initiated+ // across different IndexReaders.+ return null;+ }++ final int startIdx;+ final int limitIdx;+ if (context.isTopLevel) {+ startIdx = 0;+ limitIdx = docs.length;+ } else {+ int[] ordIdxMap = getOrdIdxMap(context);+ startIdx = context.ord == 0 ? 0 : ordIdxMap[context.ord - 1];+ limitIdx = ordIdxMap[context.ord];++ if (startIdx >= limitIdx) {+ return null; // verified this does happen+ }+ }+ final int base = context.docBase;++ return new DocIdSetIterator() {+ int idx = startIdx - 1;+ int adjustedDoc = -1;@Override- public DocIdSet getDocIdSet(final LeafReaderContext context, final Bits acceptDocs) {- LeafReader reader = context.reader();- // all Solr DocSets that are used as filters only include live docs- final Bits acceptDocs2 = acceptDocs == null ? null : (reader.getLiveDocs() == acceptDocs ? null : acceptDocs);-- final int base = context.docBase;- final int maxDoc = reader.maxDoc();- final int max = base + maxDoc; // one past the max doc in this segment.- int sidx = Math.max(0,lastEndIdx);-- if (sidx > 0 && docs[sidx-1] >= base) {- // oops, the lastEndIdx isn't correct... we must have been used- // in a multi-threaded context, or the indexreaders are being- // used out-of-order. start at 0.- sidx = 0;- }- if (sidx < docs.length && docs[sidx] < base) {- // if docs[sidx] is < base, we need to seek to find the real start.- sidx = findIndex(docs, base, sidx, docs.length-1);- }+ public int docID() {+ return adjustedDoc;+ }- final int startIdx = sidx;+ @Override+ public int nextDoc() {+ return adjustedDoc = (++idx >= limitIdx) ? NO_MORE_DOCS : (docs[idx] - base);+ }- // Largest possible end index is limited to the start index- // plus the number of docs contained in the segment. Subtract 1 since- // the end index is inclusive.- int eidx = Math.min(docs.length, startIdx + maxDoc) - 1;+ @Override+ public int advance(int target) {+ if (++idx >= limitIdx || target==NO_MORE_DOCS) return adjustedDoc=NO_MORE_DOCS;+ target += base;- // find the real end- eidx = findIndex(docs, max, startIdx, eidx) - 1;+ // probe next+ int rawDoc = docs[idx];+ if (rawDoc >= target) return adjustedDoc=rawDoc-base;- final int endIdx = eidx;- lastEndIdx = endIdx;+ // TODO: probe more before resorting to binary search?+ final int findIdx = Arrays.binarySearch(docs, idx + 1, limitIdx, target);+ idx = findIdx < 0 ? ~findIdx : findIdx;+ return adjustedDoc = idx < limitIdx ? docs[idx] - base : NO_MORE_DOCS;+ }++ @Override+ public long cost() {+ return limitIdx - startIdx;+ }+ };+ }++ @Override+ public Filter getTopFilter() {+ return new Filter() {++ @Override+ public DocIdSet getDocIdSet(final LeafReaderContext context, final Bits acceptDocs) {+ // all Solr DocSets that are used as filters only include live docs+ final Bits acceptDocs2 = acceptDocs == null ? null : (context.reader().getLiveDocs() == acceptDocs ? null : acceptDocs);return BitsFilteredDocIdSet.wrap(new DocIdSet() {@Overridepublic DocIdSetIterator iterator() {- return new DocIdSetIterator() {- int idx = startIdx;- int adjustedDoc = -1;-- @Override- public int docID() {- return adjustedDoc;- }-- @Override- public int nextDoc() {- return adjustedDoc = (idx > endIdx) ? NO_MORE_DOCS : (docs[idx++] - base);- }-- @Override- public int advance(int target) {- if (idx > endIdx || target==NO_MORE_DOCS) return adjustedDoc=NO_MORE_DOCS;- target += base;-- // probe next- int rawDoc = docs[idx++];- if (rawDoc >= target) return adjustedDoc=rawDoc-base;-- int high = endIdx;-- // TODO: probe more before resorting to binary search?-- // binary search- while (idx <= high) {- int mid = (idx+high) >>> 1;- rawDoc = docs[mid];-- if (rawDoc < target) {- idx = mid+1;- }- else if (rawDoc > target) {- high = mid-1;- }- else {- idx=mid+1;- return adjustedDoc=rawDoc - base;- }- }-- // low is on the insertion point...- if (idx <= endIdx) {- return adjustedDoc = docs[idx++] - base;- } else {- return adjustedDoc=NO_MORE_DOCS;- }- }-- @Override- public long cost() {- return docs.length;- }- };+ return SortedIntDocSet.this.iterator(context);}@Overridediff --git a/solr/core/src/java/org/apache/solr/search/facet/SweepCountAware.java b/solr/core/src/java/org/apache/solr/search/facet/SweepCountAware.javaindex bfc583f009f..ab5b32379bb 100644--- a/solr/core/src/java/org/apache/solr/search/facet/SweepCountAware.java+++ b/solr/core/src/java/org/apache/solr/search/facet/SweepCountAware.java@@ -165,7 +165,10 @@ interface SweepCountAware {*/public void register(CountSlotAcc[] countAccs, LongValues toGlobal, int maxSegOrd) {int segOrd = maxSegOrd;- final int maxIdx = countAccs.length - 1;+ // NOTE: the `countAccs` array may be oversized (e.g., in the event that one or more of the domains supplies a <code>null</code>+ // DocIdSetIterator -- matches no docs -- for a given segment). But the relevant contents of `countAccs` are re-initialized+ // per-segment, so we can (and must!) safely set `maxIdx` based on `activeSegCounts.length` (as opposed to `countAccs.length`).+ final int maxIdx = activeSegCounts.length - 1;for (;;) {if (seen[segOrd]) {int i = maxIdx;diff --git a/solr/core/src/java/org/apache/solr/search/facet/SweepDISI.java b/solr/core/src/java/org/apache/solr/search/facet/SweepDISI.javaindex c083c5db0ec..122b0a4c9e9 100644--- a/solr/core/src/java/org/apache/solr/search/facet/SweepDISI.java+++ b/solr/core/src/java/org/apache/solr/search/facet/SweepDISI.java@@ -20,7 +20,6 @@ import java.io.IOException;import java.util.List;import org.apache.lucene.index.LeafReaderContext;-import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.DocIdSetIterator;import org.apache.solr.search.facet.SlotAcc.CountSlotAcc;import org.apache.solr.search.facet.SlotAcc.SweepCountAccStruct;@@ -36,8 +35,7 @@ abstract class SweepDISI extends DocIdSetIterator implements SweepCountAware {}private static boolean addAcc(SweepCountAccStruct entry, DocIdSetIterator[] subIterators, CountSlotAcc[] activeCountAccs, LeafReaderContext subCtx, int idx) throws IOException {- final DocIdSet docIdSet = entry.docSet.getTopFilter().getDocIdSet(subCtx, null);- if (docIdSet == null || (subIterators[idx] = docIdSet.iterator()) == null) {+ if ((subIterators[idx] = entry.docSet.iterator(subCtx)) == null) {return false;}activeCountAccs[idx] = entry.countAcc;diff --git a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.javaindex 16bf6e29aa4..adf5dd08b8b 100644--- a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java+++ b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java@@ -29,7 +29,6 @@ import org.apache.lucene.index.Terms;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.search.ConstantScoreScorer;import org.apache.lucene.search.ConstantScoreWeight;-import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Query;@@ -61,7 +60,6 @@ import org.apache.solr.schema.FieldType;import org.apache.solr.search.BitDocSet;import org.apache.solr.search.DocSet;import org.apache.solr.search.DocSetUtil;-import org.apache.solr.search.Filter;import org.apache.solr.search.SolrIndexSearcher;public class CrossCollectionJoinQuery extends Query {@@ -178,7 +176,7 @@ public class CrossCollectionJoinQuery extends Query {private SolrIndexSearcher searcher;private ScoreMode scoreMode;- private Filter filter;+ private DocSet docs;public CrossCollectionJoinQueryWeight(SolrIndexSearcher searcher, ScoreMode scoreMode, float score) {super(CrossCollectionJoinQuery.this, score);@@ -307,15 +305,11 @@ public class CrossCollectionJoinQuery extends Query {@Overridepublic Scorer scorer(LeafReaderContext context) throws IOException {- if (filter == null) {- filter = getDocSet().getTopFilter();+ if (docs == null) {+ docs = getDocSet();}- DocIdSet readerSet = filter.getDocIdSet(context, null);- if (readerSet == null) {- return null;- }- DocIdSetIterator readerSetIterator = readerSet.iterator();+ DocIdSetIterator readerSetIterator = docs.iterator(context);if (readerSetIterator == null) {return null;}diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.javaindex c25679b9b4c..f35d9578e3e 100644--- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java+++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java@@ -46,7 +46,6 @@ import org.apache.lucene.util.automaton.DaciukMihovAutomatonBuilder;import org.apache.solr.schema.SchemaField;import org.apache.solr.search.BitDocSet;import org.apache.solr.search.DocSet;-import org.apache.solr.search.Filter;import org.apache.solr.search.SolrIndexSearcher;/**@@ -133,7 +132,6 @@ public class GraphQuery extends Query {final SolrIndexSearcher fromSearcher;private int currentDepth = -1;- private Filter filter;private DocSet resultSet;SchemaField collectSchemaField; // the field to collect values fromSchemaField matchSchemaField; // the field to match those values@@ -268,13 +266,12 @@ public class GraphQuery extends Query {@Overridepublic Scorer scorer(LeafReaderContext context) throws IOException {- if (filter == null) {+ if (resultSet == null) {resultSet = getDocSet();- filter = resultSet.getTopFilter();}- DocIdSet readerSet = filter.getDocIdSet(context,context.reader().getLiveDocs());+ DocIdSetIterator disi = resultSet.iterator(context);// create a scrorer on the result set, if results from right query are empty, use empty iterator.- return new GraphScorer(this, readerSet == null ? DocIdSetIterator.empty() : readerSet.iterator(), 1);+ return new GraphScorer(this, disi == null ? DocIdSetIterator.empty() : disi, 1);}@Overridediff --git a/solr/core/src/test/org/apache/solr/search/TestDocSet.java b/solr/core/src/test/org/apache/solr/search/TestDocSet.javaindex 822830f58c4..0f3e999b586 100644--- a/solr/core/src/test/org/apache/solr/search/TestDocSet.java+++ b/solr/core/src/test/org/apache/solr/search/TestDocSet.java@@ -20,6 +20,8 @@ import java.io.IOException;import java.util.Arrays;import java.util.List;import java.util.Random;+import java.util.function.Supplier;+import java.util.function.ToIntFunction;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.FieldInfos;@@ -390,43 +392,147 @@ public class TestDocSet extends SolrTestCase {return mr;}- public void doTestIteratorEqual(DocIdSet a, DocIdSet b) throws IOException {- DocIdSetIterator ia = a.iterator();- DocIdSetIterator ib = b.iterator();+ private static boolean checkNullOrEmpty(DocIdSetIterator[] disis) throws IOException {+ for (DocIdSetIterator disi : disis) {+ if (disi == null) {+ for (DocIdSetIterator check : disis) {+ if (check != null) {+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, check.nextDoc());+ }+ }+ return true;+ }+ }+ return false;+ }++ private static Supplier<DocIdSetIterator> disiSupplier(final DocIdSet docs) {+ return () -> {+ try {+ return docs.iterator();+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ };+ }++ @SafeVarargs+ private static void populateDisis(NoThrowDocIdSetIterator[] disis, Supplier<DocIdSetIterator>... suppliers) {+ for (int i = 0; i < suppliers.length; i++) {+ DocIdSetIterator disi = suppliers[i].get();+ disis[i] = disi == null ? null : new NoThrowDocIdSetIterator(disi);+ }+ }++ private static void populateDocs(NoThrowDocIdSetIterator[] disis, int[] docs, ToIntFunction<NoThrowDocIdSetIterator> toDocId) throws IOException {+ for (int i = 0; i < docs.length; i++) {+ docs[i] = toDocId.applyAsInt(disis[i]);+ }+ }++ private static void assertAll(int expected, int[] docs) {+ for (int doc : docs) {+ assertEquals(expected, doc);+ }+ }++ /**+ * By wrapping exceptions (which we don't expect to have thrown in this context anyway), we allow for+ * more transparent/readable inline functions.+ */+ private static class NoThrowDocIdSetIterator extends DocIdSetIterator {+ private final DocIdSetIterator backing;+ private NoThrowDocIdSetIterator(DocIdSetIterator backing) {+ this.backing = backing;+ }+ @Override+ public int advance(int target) {+ try {+ return backing.advance(target);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ @Override+ public long cost() {+ return backing.cost();+ }+ @Override+ public int docID() {+ return backing.docID();+ }+ @Override+ public int nextDoc() {+ try {+ return backing.nextDoc();+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ }++ @SafeVarargs+ private void doTestIteratorEqual(Bits bits, Supplier<DocIdSetIterator>... disiSuppliers) throws IOException {+ NoThrowDocIdSetIterator[] disis = new NoThrowDocIdSetIterator[disiSuppliers.length];+ int[] docs = new int[disiSuppliers.length];+ populateDisis(disis, disiSuppliers);+ if (checkNullOrEmpty(disis)) {+ // both iterators are empty or null (equivalent), so there's nothing more to check+ return;+ }// test for next() equivalence+ final int bitsLength = bits == null ? -1 : bits.length();+ int bitsDoc = -1;for(;;) {- int da = ia.nextDoc();- int db = ib.nextDoc();- assertEquals(da, db);- assertEquals(ia.docID(), ib.docID());- if (da==DocIdSetIterator.NO_MORE_DOCS) break;+ populateDocs(disis, docs, (disi) -> disi.nextDoc());+ final int expected = docs[0]; // arbitrarily pick the first as "expected"+ assertAll(expected, docs);+ populateDocs(disis, docs, (disi) -> disi.docID());+ assertAll(expected, docs);+ while (++bitsDoc < expected && bitsDoc < bitsLength) {+ assertFalse(bits.get(bitsDoc));+ }+ if (expected==DocIdSetIterator.NO_MORE_DOCS) break;+ assertTrue(bits.get(expected));}for (int i=0; i<10; i++) {// test random skipTo() and next()- ia = a.iterator();- ib = b.iterator();+ populateDisis(disis, disiSuppliers);+ bitsDoc = -1;int doc = -1;for (;;) {- int da,db;+ final int target;if (rand.nextBoolean()) {- da = ia.nextDoc();- db = ib.nextDoc();+ target = doc + 1;+ populateDocs(disis, docs, (disi) -> disi.nextDoc());} else {- int target = doc + rand.nextInt(10) + 1; // keep in mind future edge cases like probing (increase if necessary)- da = ia.advance(target);- db = ib.advance(target);+ target = doc + rand.nextInt(10) + 1; // keep in mind future edge cases like probing (increase if necessary)+ populateDocs(disis, docs, (disi) -> disi.advance(target));}- assertEquals(da, db);- assertEquals(ia.docID(), ib.docID());- if (da==DocIdSetIterator.NO_MORE_DOCS) break;- doc = da;+ final int expected = docs[0]; // arbitrarily pick the first as "expected"+ assertAll(expected, docs);+ populateDocs(disis, docs, (disi) -> disi.docID());+ assertAll(expected, docs);+ for (int j = target; j < expected && j < bitsLength; j++) {+ assertFalse(bits.get(j));+ }+ if (expected==DocIdSetIterator.NO_MORE_DOCS) break;+ assertTrue(bits.get(expected));+ doc = expected;}}}+ /**+ * Tests equivalence among {@link DocIdSetIterator} instances retrieved from {@link BitDocSet} and {@link SortedIntDocSet}+ * implementations, via {@link DocSet#getTopFilter()}/{@link Filter#getDocIdSet(LeafReaderContext, Bits)} and directly+ * via {@link DocSet#iterator(LeafReaderContext)}.+ * Also tests corresponding random-access {@link Bits} instances retrieved via {@link DocSet#getTopFilter()}/+ * {@link Filter#getDocIdSet(LeafReaderContext, Bits)}/{@link DocIdSet#bits()}.+ */public void doFilterTest(IndexReader reader) throws IOException {IndexReaderContext topLevelContext = reader.getContext();FixedBitSet bs = getRandomSet(reader.maxDoc(), rand.nextInt(reader.maxDoc()+1));@@ -451,7 +557,12 @@ public class TestDocSet extends SolrTestCase {for (LeafReaderContext readerContext : leaves) {da = fa.getDocIdSet(readerContext, null);db = fb.getDocIdSet(readerContext, null);- doTestIteratorEqual(da, db);++ // there are various ways that disis can be retrieved for each leafReader; they should all be equivalent.+ doTestIteratorEqual(da.bits(), disiSupplier(da), disiSupplier(db), () -> a.iterator(readerContext), () -> b.iterator(readerContext));++ // set b is SortedIntDocSet, so derivatives should not support random-access via Bits+ assertNull(db.bits());}int nReaders = leaves.size();@@ -460,7 +571,8 @@ public class TestDocSet extends SolrTestCase {LeafReaderContext readerContext = leaves.get(rand.nextInt(nReaders));da = fa.getDocIdSet(readerContext, null);db = fb.getDocIdSet(readerContext, null);- doTestIteratorEqual(da, db);+ doTestIteratorEqual(da.bits(), disiSupplier(da), disiSupplier(db), () -> a.iterator(readerContext), () -> b.iterator(readerContext));+ assertNull(db.bits());}}