Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15185: Optimize Hash QParser (#1524)

David Smileyddbd3b88ec8a
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c7afbcec241..ef474f96c25 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -88,6 +88,9 @@ when told to. The admin UI now tells it to. (Nazerke Seidan, David Smiley)
* SOLR-15100: Make the ConfigSetService pluggable/configurable via <string name="configSetService" /> in solr.xml
(baisui)
+* SOLR-15185: Various optimizations to the {!hash} QParser, typically used by the parallel()
+ streaming expression. The hash algorithm changed. (David Smiley)
+
Other Changes
----------------------
* SOLR-14656: Autoscaling framework removed (Ishan Chattopadhyaya, noble, Ilan Ginzburg)
diff --git a/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java
index 2cdf9f3149d..8bf227d6d11 100644
--- a/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java
@@ -18,45 +18,33 @@ package org.apache.solr.search;
import java.io.IOException;
import java.util.Arrays;
-import java.util.List;
+import java.util.Objects;
+import java.util.function.DoublePredicate;
+import java.util.stream.Collectors;
-import com.google.common.primitives.Longs;
-import org.apache.lucene.index.IndexReaderContext;
-import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.queries.function.FunctionMatchQuery;
+import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.LongValues;
+import org.apache.lucene.search.LongValuesSource;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.QueryVisitor;
-import org.apache.lucene.search.Scorable;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.BitDocIdSet;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.CharsRefBuilder;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.schema.FieldType;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.StrField;
/**
-* syntax fq={!hash workers=11 worker=4 keys=field1,field2}
-* */
-
+ * syntax fq={!hash workers=11 worker=4 keys=field1,field2}
+ */
public class HashQParserPlugin extends QParserPlugin {
public static final String NAME = "hash";
-
public QParser createParser(String query, SolrParams localParams, SolrParams params, SolrQueryRequest request) {
return new HashQParser(query, localParams, params, request);
}
@@ -75,308 +63,158 @@ public class HashQParserPlugin extends QParserPlugin {
int worker = localParams.getInt("worker", 0);
String keyParam = params.get("partitionKeys");
String[] keys = keyParam.replace(" ", "").split(",");
- if (keys.length > 4) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "HashQuery supports upto 4 partitionKeys");
- }
- return new HashQuery(keys, workers, worker);
- }
- }
-
- private static class HashQuery extends ExtendedQueryBase implements PostFilter {
+ Arrays.stream(keys).forEach(field -> req.getSchema().getField(field)); // validate all fields exist
- private String[] keys;
- private int workers;
- private int worker;
-
- public boolean getCache() {
- if(getCost() > 99) {
- return false;
- } else {
- return super.getCache();
- }
- }
-
- public int hashCode() {
- return classHash() +
- 31 * keys.hashCode() +
- 31 * workers +
- 31 * worker;
+ // TODO wish to provide matchCost on FunctionMatchQuery's TwoPhaseIterator -- LUCENE-9373
+ return new FunctionMatchQuery(
+ new HashCodeValuesSource(keys).toDoubleValuesSource(),
+ new HashPartitionPredicate(workers, worker));
}
- public boolean equals(Object other) {
- return sameClassAs(other) &&
- equalsTo(getClass().cast(other));
- }
+ }
- private boolean equalsTo(HashQuery other) {
- return keys.equals(other.keys) &&
- workers == other.workers &&
- worker == other.worker;
- }
+ /** A {@link LongValuesSource} that is a hash of some fields' values. They are fetched via DocValues */
+ private static class HashCodeValuesSource extends LongValuesSource {
- @Override
- public void visit(QueryVisitor visitor) {
- visitor.visitLeaf(this);
- }
+ private final String[] fields;
- public HashQuery(String[] keys, int workers, int worker) {
- this.keys = keys;
- this.workers = workers;
- this.worker = worker;
+ private HashCodeValuesSource(String[] fields) {
+ this.fields = fields;
}
@Override
- public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
-
- SolrIndexSearcher solrIndexSearcher = (SolrIndexSearcher)searcher;
- IndexReaderContext context = solrIndexSearcher.getTopReaderContext();
-
- List<LeafReaderContext> leaves = context.leaves();
- FixedBitSet[] fixedBitSets = new FixedBitSet[leaves.size()];
-
- for(LeafReaderContext leaf : leaves) {
- try {
- SegmentPartitioner segmentPartitioner = new SegmentPartitioner(leaf,worker,workers, keys, solrIndexSearcher);
- segmentPartitioner.run();
- fixedBitSets[segmentPartitioner.context.ord] = segmentPartitioner.docs;
- } catch(Exception e) {
- throw new IOException(e);
+ public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ // produce an array of the LongValues of the hash of each field
+ final LongValues[] resultValues = new LongValues[fields.length];
+ for (int i = 0; i < fields.length; i++) {
+ final String field = fields[i];
+ final NumericDocValues numericDocValues = ctx.reader().getNumericDocValues(field);
+ if (numericDocValues != null) {
+ // Numeric
+ resultValues[i] = new LongValues() {
+ // Even if not a Long field; could be int, double, float and this still works because DocValues numerics
+ // are based on a Long.
+ final NumericDocValues values = numericDocValues;
+ boolean atDoc = false;
+ @Override
+ public boolean advanceExact(int doc) throws IOException { atDoc = values.advanceExact(doc); return true; }
+ @Override
+ public long longValue() throws IOException { return atDoc ? Long.hashCode(values.longValue()) : 0; }
+ };
+ continue;
}
- }
-
- ConstantScoreQuery constantScoreQuery = new ConstantScoreQuery(new BitsFilter(fixedBitSets));
- return searcher.rewrite(constantScoreQuery).createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
- }
-
- public static class BitsFilter extends Filter {
- private FixedBitSet[] bitSets;
- public BitsFilter(FixedBitSet[] bitSets) {
- this.bitSets = bitSets;
- }
-
- public String toString(String s) {
- return s;
- }
-
- public DocIdSet getDocIdSet(LeafReaderContext context, Bits bits) {
- return BitsFilteredDocIdSet.wrap(new BitDocIdSet(bitSets[context.ord]), bits);
- }
-
- @Override
- public boolean equals(Object other) {
- return sameClassAs(other) &&
- equalsTo(getClass().cast(other));
- }
-
- private boolean equalsTo(BitsFilter other) {
- return Arrays.equals(bitSets, other.bitSets);
- }
-
- @Override
- public int hashCode() {
- return classHash() + Arrays.asList(bitSets).hashCode();
- }
- }
-
-
- static class SegmentPartitioner implements Runnable {
-
- public LeafReaderContext context;
- private int worker;
- private int workers;
- private HashKey k;
- public FixedBitSet docs;
- public SegmentPartitioner(LeafReaderContext context,
- int worker,
- int workers,
- String[] keys,
- SolrIndexSearcher solrIndexSearcher) {
- this.context = context;
- this.worker = worker;
- this.workers = workers;
-
- HashKey[] hashKeys = new HashKey[keys.length];
- IndexSchema schema = solrIndexSearcher.getSchema();
- for(int i=0; i<keys.length; i++) {
- String key = keys[i];
- FieldType ft = schema.getField(key).getType();
- HashKey h = null;
- if(ft instanceof StrField) {
- h = new BytesHash(key, ft);
- } else {
- h = new NumericHash(key);
- }
- hashKeys[i] = h;
+ final SortedDocValues sortedDocValues = ctx.reader().getSortedDocValues(field);
+ if (sortedDocValues != null) {
+ // String
+ resultValues[i] = new LongValues() {
+ final SortedDocValues values = sortedDocValues;
+ boolean atDoc = false;
+ @Override
+ public boolean advanceExact(int doc) throws IOException { atDoc = values.advanceExact(doc); return true; }
+ @Override
+ public long longValue() throws IOException { return atDoc ? values.binaryValue().hashCode() : 0; }
+ };
+ continue;
}
-
- k = (hashKeys.length > 1) ? new CompositeHash(hashKeys) : hashKeys[0];
+ // fail if some other DocValuesType is present
+ final FieldInfo fieldInfo = ctx.reader().getFieldInfos().fieldInfo(field);
+ if (fieldInfo != null && fieldInfo.getDocValuesType() != DocValuesType.NONE) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can't compute hash on field " + field);
+ }
+ // fall back on zero for empty case
+ resultValues[i] = LongValuesSource.constant(0).getValues(ctx, scores);
}
- public void run() {
- LeafReader reader = context.reader();
-
- try {
- k.setNextReader(context);
- this.docs = new FixedBitSet(reader.maxDoc());
- int maxDoc = reader.maxDoc();
- for(int i=0; i<maxDoc; i++) {
- if((k.hashCode(i) & 0x7FFFFFFF) % workers == worker) {
- docs.set(i);
+ // produce a hash LongValues of all those in this array
+ if (resultValues.length == 1) {
+ return resultValues[0];
+ } else {
+ // Combine
+ return new LongValues() {
+ private long result;
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ // compute the hash here.
+ // algorithm borrowed from Arrays.hashCode(Object[]) but without needing to call hashCode redundantly
+ result = 1;
+ for (LongValues longValues : resultValues) {
+ boolean present = longValues.advanceExact(doc);
+ result = 31 * result + (present == false ? 0 : longValues.longValue());
}
+ return true; // we always have a hash value
}
- }catch(Exception e) {
- throw new RuntimeException(e);
- }
+ @Override
+ public long longValue() throws IOException { return result; }
+ };
}
}
- public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) {
- HashKey[] hashKeys = new HashKey[keys.length];
- SolrIndexSearcher searcher = (SolrIndexSearcher)indexSearcher;
- IndexSchema schema = searcher.getSchema();
- for(int i=0; i<keys.length; i++) {
- String key = keys[i];
- FieldType ft = schema.getField(key).getType();
- HashKey h = null;
- if(ft instanceof StrField) {
- h = new BytesHash(key, ft);
- } else {
- h = new NumericHash(key);
- }
- hashKeys[i] = h;
- }
- HashKey k = (hashKeys.length > 1) ? new CompositeHash(hashKeys) : hashKeys[0];
- return new HashCollector(k, workers, worker);
- }
- }
-
- private static class HashCollector extends DelegatingCollector {
- private int worker;
- private int workers;
- private HashKey hashKey;
- private LeafCollector leafCollector;
-
- public HashCollector(HashKey hashKey, int workers, int worker) {
- this.hashKey = hashKey;
- this.workers = workers;
- this.worker = worker;
- }
-
- public void setScorer(Scorable scorer) throws IOException{
- leafCollector.setScorer(scorer);
+ @Override
+ public boolean needsScores() {
+ return false;
}
- public void doSetNextReader(LeafReaderContext context) throws IOException {
- this.hashKey.setNextReader(context);
- this.leafCollector = delegate.getLeafCollector(context);
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ HashCodeValuesSource that = (HashCodeValuesSource) o;
+ return Arrays.equals(fields, that.fields);
}
- public void collect(int doc) throws IOException {
- if((hashKey.hashCode(doc) & 0x7FFFFFFF) % workers == worker) {
- leafCollector.collect(doc);
- }
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(fields);
}
- }
-
- private interface HashKey {
- public void setNextReader(LeafReaderContext reader) throws IOException;
- public long hashCode(int doc) throws IOException;
- }
-
- private static class BytesHash implements HashKey {
- private SortedDocValues values;
- private String field;
- private FieldType fieldType;
- private CharsRefBuilder charsRefBuilder = new CharsRefBuilder();
-
- public BytesHash(String field, FieldType fieldType) {
- this.field = field;
- this.fieldType = fieldType;
+ @Override
+ public String toString() {
+ return Arrays.stream(fields).collect(Collectors.joining(",", "hash(", ")"));
}
- public void setNextReader(LeafReaderContext context) throws IOException {
- values = context.reader().getSortedDocValues(field);
+ @Override
+ public LongValuesSource rewrite(IndexSearcher searcher) throws IOException {
+ return this;
}
- public long hashCode(int doc) throws IOException {
- if (doc > values.docID()) {
- values.advance(doc);
- }
- BytesRef ref;
- if (doc == values.docID()) {
- ref = values.binaryValue();
- } else {
- ref = new BytesRef(); // EMPTY_BYTES . worker=0 will always process empty values
- }
- this.fieldType.indexedToReadable(ref, charsRefBuilder);
- CharsRef charsRef = charsRefBuilder.get();
- return charsRef.hashCode();
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return DocValues.isCacheable(ctx, fields);
}
}
- private static class NumericHash implements HashKey {
-
- private NumericDocValues values;
- private String field;
-
- public NumericHash(String field) {
- this.field = field;
- }
+ /**
+ * Simple modulus check against the input to see if the result is a particular value
+ * (standard hash partition approach).
+ * Can't use a lambda because need equals/hashcode
+ */
+ private static class HashPartitionPredicate implements DoublePredicate {
+ final int workers;
+ final int worker;
- public void setNextReader(LeafReaderContext context) throws IOException {
- values = context.reader().getNumericDocValues(field);
+ private HashPartitionPredicate(int workers, int worker) {
+ this.workers = workers;
+ this.worker = worker;
}
- public long hashCode(int doc) throws IOException {
- int valuesDocID = values.docID();
- if (valuesDocID < doc) {
- valuesDocID = values.advance(doc);
- }
- long l;
- if (valuesDocID == doc) {
- l = values.longValue();
- } else {
- l = 0; //worker=0 will always process empty values
- }
- return Longs.hashCode(l);
+ @Override
+ public boolean test(double hashAsDouble) {
+ return Math.abs((long)hashAsDouble) % workers == worker;
}
- }
-
- private static class ZeroHash implements HashKey {
- public long hashCode(int doc) {
- return 0;
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ HashPartitionPredicate that = (HashPartitionPredicate) o;
+ return workers == that.workers &&
+ worker == that.worker;
}
- public void setNextReader(LeafReaderContext context) {
-
+ @Override
+ public int hashCode() {
+ return Objects.hash(workers, worker);
}
}
- private static class CompositeHash implements HashKey {
-
- private HashKey key1;
- private HashKey key2;
- private HashKey key3;
- private HashKey key4;
-
- public CompositeHash(HashKey[] hashKeys) {
- key1 = hashKeys[0];
- key2 = hashKeys[1];
- key3 = (hashKeys.length > 2) ? hashKeys[2] : new ZeroHash();
- key4 = (hashKeys.length > 3) ? hashKeys[3] : new ZeroHash();
- }
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- key1.setNextReader(context);
- key2.setNextReader(context);
- key3.setNextReader(context);
- key4.setNextReader(context);
- }
-
- public long hashCode(int doc) throws IOException {
- return key1.hashCode(doc)+key2.hashCode(doc)+key3.hashCode(doc)+key4.hashCode(doc);
- }
- }
}
diff --git a/solr/core/src/test/org/apache/solr/search/TestHashQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestHashQParserPlugin.java
index 97b995285f5..3c17e286d81 100644
--- a/solr/core/src/test/org/apache/solr/search/TestHashQParserPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestHashQParserPlugin.java
@@ -71,7 +71,7 @@ public class TestHashQParserPlugin extends SolrTestCaseJ4 {
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!hash worker=0 workers=2 cost="+getCost(random())+"}");
- params.add("partitionKeys", "a_i,a_s,a_i,a_s,a_i");
+ params.add("partitionKeys", "nonexistent");
params.add("wt", "xml");
ModifiableSolrParams finalParams = params;
expectThrows(SolrException.class, () -> h.query(req(finalParams)));