Apache Solr open-source search software
SOLR-17678, SOLR-17732: Add matchScore support for ReRank queries (#3222)
Siju Vargheseb226f4726f22
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 61a1d98e47a..1194d8c1db1 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -195,6 +195,8 @@ New Features* SOLR-17714: Added a FuzzyQParser to enable all FuzzyQuery customizations. (Houston Putman, Siju Varghese)+* SOLR-17678: ReRank queries can now return the matchScore (original score) in addition to the re-ranked score. (Siju Varghese, Houston Putman)+Improvements---------------------* SOLR-15751: The v2 API now has parity with the v1 "COLSTATUS" and "segments" APIs, which can be used to fetch detailed information about@@ -217,6 +219,8 @@ Improvements* SOLR-10998: v2 APIs now obey the "Accept" request header for content-negotiation if 'wt' is unspecified. JSON is still used as a default whenneither 'Accept' or 'wt' are specified. (Jason Gerlowski)+* SOLR-17732: Score-based return fields other than "score" can now be returned in distributed queries. (Houston Putman)+Optimizations---------------------* SOLR-17578: Remove ZkController internal core supplier, for slightly faster reconnection after Zookeeper session loss. (Pierre Salagnac)diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.javaindex bed6203d61c..f037507d397 100644--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java@@ -25,8 +25,8 @@ import java.io.StringWriter;import java.lang.invoke.MethodHandles;import java.lang.reflect.Array;import java.util.ArrayList;-import java.util.Arrays;import java.util.Collection;+import java.util.Collections;import java.util.HashMap;import java.util.Iterator;import java.util.LinkedHashMap;@@ -35,6 +35,9 @@ import java.util.Locale;import java.util.Map;import java.util.Map.Entry;import java.util.Objects;+import java.util.Set;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.stream.Collectors;import org.apache.lucene.index.ExitableDirectoryReader;import org.apache.lucene.index.IndexReaderContext;import org.apache.lucene.index.LeafReaderContext;@@ -780,13 +783,17 @@ public class QueryComponent extends SearchComponent {// distrib.singlePass=true forces a one-pass query regardless of requested fieldsboolean distribSinglePass = rb.req.getParams().getBool(ShardParams.DISTRIB_SINGLE_PASS, false);- if (distribSinglePass- || (fields != null- && fields.wantsField(keyFieldName)- && fields.getRequestedFieldNames() != null- && (!fields.hasPatternMatching()- && Arrays.asList(keyFieldName, "score")- .containsAll(fields.getRequestedFieldNames())))) {+ boolean requiresNonIdAndScoreFields = true;+ if (!distribSinglePass) {+ Set<String> nonScoreDependentFieldNames = fields.getNonScoreDependentReturnFieldNames();+ requiresNonIdAndScoreFields =+ fields.hasPatternMatching()+ || nonScoreDependentFieldNames == null+ || nonScoreDependentFieldNames.size() > 1+ || (nonScoreDependentFieldNames.size() == 1+ && !nonScoreDependentFieldNames.contains(keyFieldName));+ }+ if (distribSinglePass || !requiresNonIdAndScoreFields) {sreq.purpose |= ShardRequest.PURPOSE_GET_FIELDS;rb.onePassDistributedQuery = true;}@@ -825,7 +832,7 @@ public class QueryComponent extends SearchComponent {|| rb.getSortSpec().includesScore();StringBuilder additionalFL = new StringBuilder();boolean additionalAdded = false;- if (distribSinglePass) {+ if (rb.onePassDistributedQuery) {String[] fls = rb.req.getParams().getParams(CommonParams.FL);if (fls != null && fls.length > 0 && (fls.length != 1 || !fls[0].isEmpty())) {// If the outer request contains actual FL's use them...@@ -838,15 +845,27 @@ public class QueryComponent extends SearchComponent {// additional fields belowsreq.params.set(CommonParams.FL, "*");}- if (!fields.wantsScore() && shardQueryIncludeScore) {- additionalAdded = addFL(additionalFL, "score", additionalAdded);- }} else {// reset so that only unique key is requested in shard requestssreq.params.set(CommonParams.FL, rb.req.getSchema().getUniqueKeyField().getName());- if (shardQueryIncludeScore) {- additionalAdded = addFL(additionalFL, "score", additionalAdded);- }++ final AtomicBoolean hasAdditionalAdded = new AtomicBoolean(additionalAdded);+ fields+ .getScoreDependentReturnFields()+ .forEach(+ (name, value) -> {+ if (value.isEmpty()) {+ addFL(additionalFL, name, hasAdditionalAdded.getAndSet(true));+ } else {+ addFL(additionalFL, name + ":" + value, hasAdditionalAdded.getAndSet(true));+ }+ });+ additionalAdded = hasAdditionalAdded.get();+ }+ if ((fields.getExplicitlyRequestedFieldNames() == null+ || !fields.getExplicitlyRequestedFieldNames().contains(SolrReturnFields.SCORE))+ && shardQueryIncludeScore) {+ additionalAdded = addFL(additionalFL, SolrReturnFields.SCORE, additionalAdded);}// TODO: should this really sendGlobalDfs if just includeScore?@@ -894,6 +913,19 @@ public class QueryComponent extends SearchComponent {sortFields = new SortField[] {SortField.FIELD_SCORE};}+ // If the shard request was also used to get fields (along with the scores), there is no reason+ // to copy over the score dependent fields, since those will already exist in the document with+ // the return fields+ Set<String> scoreDependentFields;+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) == 0) {+ scoreDependentFields =+ rb.rsp.getReturnFields().getScoreDependentReturnFields().keySet().stream()+ .filter(field -> !field.equals(SolrReturnFields.SCORE))+ .collect(Collectors.toSet());+ } else {+ scoreDependentFields = Collections.emptySet();+ }+IndexSchema schema = rb.req.getSchema();SchemaField uniqueKeyField = schema.getUniqueKeyField();@@ -1071,14 +1103,17 @@ public class QueryComponent extends SearchComponent {shardDoc.id = id;shardDoc.shard = srsp.getShard();shardDoc.orderInShard = i;- Object scoreObj = doc.getFieldValue("score");+ Object scoreObj = doc.getFieldValue(SolrReturnFields.SCORE);if (scoreObj != null) {if (scoreObj instanceof String) {shardDoc.score = Float.parseFloat((String) scoreObj);} else {- shardDoc.score = (Float) scoreObj;+ shardDoc.score = ((Number) scoreObj).floatValue();}}+ if (!scoreDependentFields.isEmpty()) {+ shardDoc.scoreDependentFields = doc.getSubsetOfFields(scoreDependentFields);+ }shardDoc.sortFieldValues = unmarshalledSortFieldValues;@@ -1302,11 +1337,15 @@ public class QueryComponent extends SearchComponent {// TODO: merge fsv to if requestedif ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {- boolean returnScores = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0;-final String uniqueKey = rb.req.getSchema().getUniqueKeyField().getName();String keyFieldName = uniqueKey;boolean removeKeyField = !rb.rsp.getReturnFields().wantsField(keyFieldName);+ boolean returnRawScore =+ rb.rsp.getReturnFields().getExplicitlyRequestedFieldNames() != null+ && rb.rsp+ .getReturnFields()+ .getExplicitlyRequestedFieldNames()+ .contains(SolrReturnFields.SCORE);if (rb.rsp.getReturnFields().getFieldRenames().get(keyFieldName) != null) {// if id was renamed we need to use the new namekeyFieldName = rb.rsp.getReturnFields().getFieldRenames().get(keyFieldName);@@ -1361,13 +1400,16 @@ public class QueryComponent extends SearchComponent {final ShardDoc sdoc = rb.resultIds.get(lastKeyString);if (sdoc != null) {shardDocFoundInResults = Boolean.TRUE;- if (returnScores) {- doc.setField("score", sdoc.score);- } else {+ // There is no need to add scores to a document if the documents were retrieved in+ // one-pass (GET_FIELDS was done at the same time as GET_TOP_IDS), because the scores+ // will already exist in the document+ if (!rb.onePassDistributedQuery) {+ sdoc.consumeScoreDependentFields(returnRawScore, doc::setField);+ } else if (!returnRawScore) {// Score might have been added (in createMainQuery) to shard-requests (and therefore// in shard-response-docs) Remove score if the outer request did not ask for it- // returned- doc.remove("score");+ // returned.+ doc.remove(SolrReturnFields.SCORE);}if (removeKeyField) {doc.removeFields(keyFieldName);diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.javaindex 0b6414834f2..badf4e337bf 100644--- a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java+++ b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java@@ -16,10 +16,14 @@*/package org.apache.solr.handler.component;+import java.util.Collections;import java.util.List;+import java.util.Map;import java.util.Objects;+import java.util.function.BiConsumer;import org.apache.lucene.search.FieldDoc;import org.apache.solr.common.util.NamedList;+import org.apache.solr.search.SolrReturnFields;public class ShardDoc extends FieldDoc {public String shard;@@ -46,6 +50,8 @@ public class ShardDoc extends FieldDoc {public int positionInResponse;+ public Map<String, Object> scoreDependentFields = Collections.emptyMap();+// the ordinal position in the merged response arraylistpublic ShardDoc(float score, Object[] fields, Object uniqueId, String shard) {@@ -58,6 +64,16 @@ public class ShardDoc extends FieldDoc {super(-1, Float.NaN);}+ public void consumeScoreDependentFields(+ boolean returnRawScore, BiConsumer<String, Object> consumer) {+ if (returnRawScore) {+ consumer.accept(SolrReturnFields.SCORE, score);+ }+ if (!scoreDependentFields.isEmpty()) {+ scoreDependentFields.forEach(consumer);+ }+ }+@Overridepublic boolean equals(Object o) {if (this == o) return true;diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.javaindex b42a7140392..08761b4c2aa 100644--- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java+++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java@@ -60,6 +60,8 @@ public class DocsStreamer implements Iterator<SolrDocument> {private final org.apache.solr.response.ResultContext rctx;private final SolrDocumentFetcher docFetcher; // a collaborator of SolrIndexSearcherprivate final DocList docs;+ private boolean doScore;+ private boolean doMatchScore;private final DocTransformer transformer;private final DocIterator docIterator;@@ -76,7 +78,9 @@ public class DocsStreamer implements Iterator<SolrDocument> {docFetcher = rctx.getDocFetcher();solrReturnFields = (SolrReturnFields) rctx.getReturnFields();- if (transformer != null) transformer.setContext(rctx);+ if (transformer != null) {+ transformer.setContext(rctx);+ }}public int currentIndex() {@@ -95,10 +99,9 @@ public class DocsStreamer implements Iterator<SolrDocument> {SolrDocument sdoc = docFetcher.solrDoc(id, solrReturnFields);if (transformer != null) {- boolean doScore = rctx.wantsScores();try {- if (doScore) {- transformer.transform(sdoc, id, docIterator.score());+ if (docs.hasScores()) {+ transformer.transform(sdoc, id, docIterator);} else {transformer.transform(sdoc, id);}@@ -114,8 +117,8 @@ public class DocsStreamer implements Iterator<SolrDocument> {* This method is less efficient then the 3 arg version because it may convert some fields that* are not needed*+ * @see #convertLuceneDocToSolrDoc(Document, IndexSchema, ReturnFields)* @deprecated use the 3 arg version for better performance- * @see #convertLuceneDocToSolrDoc(Document,IndexSchema,ReturnFields)*/@Deprecatedpublic static SolrDocument convertLuceneDocToSolrDoc(Document doc, final IndexSchema schema) {@@ -185,7 +188,9 @@ public class DocsStreamer implements Iterator<SolrDocument> {public static Object getValue(SchemaField sf, IndexableField f) {FieldType ft = null;- if (sf != null) ft = sf.getType();+ if (sf != null) {+ ft = sf.getType();+ }if (ft == null) { // handle fields not in the schemaBytesRef bytesRef = f.binaryValue();@@ -197,7 +202,9 @@ public class DocsStreamer implements Iterator<SolrDocument> {System.arraycopy(bytesRef.bytes, bytesRef.offset, bytes, 0, bytesRef.length);return bytes;}- } else return f.stringValue();+ } else {+ return f.stringValue();+ }} else {if (KNOWN_TYPES.contains(ft.getClass())) {return ft.toObject(f);diff --git a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.javaindex c93d49ee619..5ce18a4b3bc 100644--- a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java+++ b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java@@ -22,6 +22,7 @@ import java.util.Collections;import org.apache.solr.common.SolrDocument;import org.apache.solr.response.QueryResponseWriter;import org.apache.solr.response.ResultContext;+import org.apache.solr.search.DocIterationInfo;import org.apache.solr.search.SolrIndexSearcher;/**@@ -101,11 +102,30 @@ public abstract class DocTransformer {* @param score the score for this document* @throws IOException If there is a low-level I/O error.* @see #needsSolrIndexSearcher+ * @deprecated use {@link #transform(SolrDocument, int, DocIterationInfo)} instead*/+ @Deprecated(forRemoval = true, since = "9.9.0")public void transform(SolrDocument doc, int docid, float score) throws IOException {transform(doc, docid);}+ /**+ * This is where implementations do the actual work. If implementations require a valid docId and+ * index access, the {@link #needsSolrIndexSearcher} method must return true+ *+ * <p>Default implementation calls {@link #transform(SolrDocument, int)}.+ *+ * @param doc The document to alter+ * @param docid The Lucene internal doc id, or -1 in cases where the <code>doc</code> did not come+ * from the index+ * @param docInfo the document information for this document, including the score+ * @throws IOException If there is a low-level I/O error.+ * @see #needsSolrIndexSearcher+ */+ public void transform(SolrDocument doc, int docid, DocIterationInfo docInfo) throws IOException {+ transform(doc, docid, docInfo.score());+ }+/*** This is where implementations do the actual work. If implementations require a valid docId and* index access, the {@link #needsSolrIndexSearcher} method must return true@@ -115,7 +135,9 @@ public abstract class DocTransformer {* from the index* @throws IOException If there is a low-level I/O error.* @see #needsSolrIndexSearcher+ * @deprecated use {@link #transform(SolrDocument, int, DocIterationInfo)} instead*/+ @Deprecated(since = "9.9.0")public abstract void transform(SolrDocument doc, int docid) throws IOException;/**diff --git a/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java b/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.javaindex 238d1357daa..62785c0a759 100644--- a/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java+++ b/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java@@ -24,6 +24,7 @@ import java.util.List;import java.util.stream.Collectors;import org.apache.solr.common.SolrDocument;import org.apache.solr.response.ResultContext;+import org.apache.solr.search.DocIterationInfo;/** Transform a document before it gets sent out */public class DocTransformers extends DocTransformer {@@ -72,9 +73,9 @@ public class DocTransformers extends DocTransformer {}@Override- public void transform(SolrDocument doc, int docid, float score) throws IOException {+ public void transform(SolrDocument doc, int docid, DocIterationInfo docInfo) throws IOException {for (DocTransformer a : children) {- a.transform(doc, docid, score);+ a.transform(doc, docid, docInfo);}}@@ -88,11 +89,6 @@ public class DocTransformers extends DocTransformer {/** Returns true if and only if at least 1 child transformer returns true */@Overridepublic boolean needsSolrIndexSearcher() {- for (DocTransformer kid : children) {- if (kid.needsSolrIndexSearcher()) {- return true;- }- }- return false;+ return children.stream().anyMatch(DocTransformer::needsSolrIndexSearcher);}}diff --git a/solr/core/src/java/org/apache/solr/response/transform/MatchScoreAugmenter.java b/solr/core/src/java/org/apache/solr/response/transform/MatchScoreAugmenter.javanew file mode 100644index 00000000000..441c18dd285--- /dev/null+++ b/solr/core/src/java/org/apache/solr/response/transform/MatchScoreAugmenter.java@@ -0,0 +1,48 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.response.transform;++import org.apache.solr.common.SolrDocument;+import org.apache.solr.search.DocIterationInfo;++/**+ * Simple Augmenter that adds the matchScore+ *+ * @since solr 4.0+ */+public class MatchScoreAugmenter extends DocTransformer {+ final String name;++ public MatchScoreAugmenter(String display) {+ this.name = display;+ }++ @Override+ public String getName() {+ return name;+ }++ @Override+ public void transform(SolrDocument doc, int docid, DocIterationInfo docInfo) {+ doc.setField(name, docInfo.matchScore());+ }++ @Override+ public void transform(SolrDocument doc, int docid) {+ // No-op+ }+}diff --git a/solr/core/src/java/org/apache/solr/response/transform/ScoreAugmenter.java b/solr/core/src/java/org/apache/solr/response/transform/ScoreAugmenter.javaindex 22af639e80a..049ed2a9bec 100644--- a/solr/core/src/java/org/apache/solr/response/transform/ScoreAugmenter.java+++ b/solr/core/src/java/org/apache/solr/response/transform/ScoreAugmenter.java@@ -17,6 +17,7 @@package org.apache.solr.response.transform;import org.apache.solr.common.SolrDocument;+import org.apache.solr.search.DocIterationInfo;/*** Simple Augmenter that adds the score@@ -36,14 +37,16 @@ public class ScoreAugmenter extends DocTransformer {}@Override- public void transform(SolrDocument doc, int docid, float score) {+ public void transform(SolrDocument doc, int docid, DocIterationInfo docInfo) {if (context != null && context.wantsScores()) {- doc.setField(name, score);+ doc.setField(name, docInfo.score());}}@Overridepublic void transform(SolrDocument doc, int docid) {- transform(doc, docid, 0.0f);+ if (context != null && context.wantsScores()) {+ doc.setField(name, 0.0f);+ }}}diff --git a/solr/core/src/java/org/apache/solr/search/DocIterationInfo.java b/solr/core/src/java/org/apache/solr/search/DocIterationInfo.javanew file mode 100644index 00000000000..d67165738c3--- /dev/null+++ b/solr/core/src/java/org/apache/solr/search/DocIterationInfo.java@@ -0,0 +1,38 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.search;++/** Information for the current document in the <code>DocIterator</code>. */+public interface DocIterationInfo {++ /**+ * Returns the score for the document just returned by <code>nextDoc()</code>+ *+ * <p>The value returned may be meaningless depending on the context in which the DocIterator+ * instance was retrieved.+ */+ public float score();++ /**+ * Returns the query match score in case of rerank queries+ *+ * @return the query match score in case of a rerank query, null otherwise.+ */+ public default Float matchScore() {+ return null;+ }+}diff --git a/solr/core/src/java/org/apache/solr/search/DocIterator.java b/solr/core/src/java/org/apache/solr/search/DocIterator.javaindex f6e5833d352..390c4150b43 100644--- a/solr/core/src/java/org/apache/solr/search/DocIterator.java+++ b/solr/core/src/java/org/apache/solr/search/DocIterator.java@@ -24,7 +24,7 @@ import java.util.Iterator;* <p>The order of the documents is determined by the context in which the DocIterator instance was* retrieved.*/-public interface DocIterator extends Iterator<Integer> {+public interface DocIterator extends Iterator<Integer>, DocIterationInfo {// already declared in superclass, redeclaring prevents javadoc inheritance// public boolean hasNext();@@ -35,12 +35,4 @@ public interface DocIterator extends Iterator<Integer> {* @see #next()*/public int nextDoc();-- /**- * Returns the score for the document just returned by <code>nextDoc()</code>- *- * <p>The value returned may be meaningless depending on the context in which the DocIterator- * instance was retrieved.- */- public float score();}diff --git a/solr/core/src/java/org/apache/solr/search/DocSlice.java b/solr/core/src/java/org/apache/solr/search/DocSlice.javaindex 837d47cdd37..7c01493aac6 100644--- a/solr/core/src/java/org/apache/solr/search/DocSlice.java+++ b/solr/core/src/java/org/apache/solr/search/DocSlice.java@@ -32,15 +32,16 @@ public class DocSlice implements DocList, Accountable {RamUsageEstimator.shallowSizeOfInstance(DocSlice.class)+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;- final int offset; // starting position of the docs (zero based)- final int len; // number of positions used in arrays- final int[] docs; // a slice of documents (docs 0-100 of the query)+ protected final int offset; // starting position of the docs (zero based)+ protected final int len; // number of positions used in arrays+ protected final long matches;+ protected final TotalHits.Relation matchesRelation;+ protected final float maxScore;+ protected int docLength; // number of documents in the result- final float[] scores; // optional score list- final long matches;- final TotalHits.Relation matchesRelation;- final float maxScore;- final long ramBytesUsed; // cached value+ private final int[] docs; // a slice of documents (docs 0-100 of the query)+ private final float[] scores; // optional score list+ private final long ramBytesUsed; // cached value/*** Primary constructor for a DocSlice instance.@@ -73,6 +74,7 @@ public class DocSlice implements DocList, Accountable {? 0: ((long) scores.length << 2) + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER);this.matchesRelation = matchesRelation;+ this.docLength = docs == null ? 0 : docs.length;}@Overridediff --git a/solr/core/src/java/org/apache/solr/search/ReRankCollector.java b/solr/core/src/java/org/apache/solr/search/ReRankCollector.javaindex d5258ff7152..10abc972b3d 100644--- a/solr/core/src/java/org/apache/solr/search/ReRankCollector.java+++ b/solr/core/src/java/org/apache/solr/search/ReRankCollector.java@@ -17,6 +17,7 @@package org.apache.solr.search;import com.carrotsearch.hppc.IntFloatHashMap;+import com.carrotsearch.hppc.IntFloatMap;import com.carrotsearch.hppc.IntIntHashMap;import java.io.IOException;import java.util.Arrays;@@ -129,7 +130,8 @@ public class ReRankCollector extends TopDocsCollector<ScoreDoc> {ScoreDoc[] mainScoreDocs = mainDocs.scoreDocs;boolean zeroOutScores = reRankScaler != null && reRankScaler.scaleScores();- ScoreDoc[] mainScoreDocsClone = deepClone(mainScoreDocs, zeroOutScores);+ IntFloatMap docToOriginalScore = new IntFloatHashMap();+ ScoreDoc[] mainScoreDocsClone = deepClone(mainScoreDocs, docToOriginalScore, zeroOutScores);ScoreDoc[] reRankScoreDocs = new ScoreDoc[Math.min(mainScoreDocs.length, reRankDocs)];System.arraycopy(mainScoreDocs, 0, reRankScoreDocs, 0, reRankScoreDocs.length);@@ -175,7 +177,6 @@ public class ReRankCollector extends TopDocsCollector<ScoreDoc> {reRankScaler.scaleScores(mainScoreDocsClone, rescoredDocs.scoreDocs, reRankScoreDocs.length);}- return rescoredDocs; // Just return the rescoredDocs} else if (howMany > rescoredDocs.scoreDocs.length) {// We need to return more then we've reRanked, so create the combined page.ScoreDoc[] scoreDocs = new ScoreDoc[howMany];@@ -193,7 +194,6 @@ public class ReRankCollector extends TopDocsCollector<ScoreDoc> {reRankScaler.scaleScores(mainScoreDocsClone, rescoredDocs.scoreDocs, reRankScoreDocs.length);}- return rescoredDocs;} else {// We've rescored more then we need to return.@@ -205,18 +205,29 @@ public class ReRankCollector extends TopDocsCollector<ScoreDoc> {ScoreDoc[] scoreDocs = new ScoreDoc[howMany];System.arraycopy(rescoredDocs.scoreDocs, 0, scoreDocs, 0, howMany);rescoredDocs.scoreDocs = scoreDocs;- return rescoredDocs;}+ return toRescoredDocs(rescoredDocs, docToOriginalScore);} catch (Exception e) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);}}- private ScoreDoc[] deepClone(ScoreDoc[] scoreDocs, boolean zeroOut) {+ private TopDocs toRescoredDocs(TopDocs topDocs, IntFloatMap originalScores) {+ ScoreDoc[] scoreDocs = topDocs.scoreDocs;+ final RescoreDoc[] rescoredDocs = new RescoreDoc[scoreDocs.length];+ for (int i = 0; i < scoreDocs.length; i++) {+ rescoredDocs[i] = new RescoreDoc(scoreDocs[i], originalScores.get(scoreDocs[i].doc));+ }+ return new TopDocs(topDocs.totalHits, rescoredDocs);+ }++ private ScoreDoc[] deepClone(+ ScoreDoc[] scoreDocs, IntFloatMap originalScoreMap, boolean zeroOut) {ScoreDoc[] scoreDocs1 = new ScoreDoc[scoreDocs.length];for (int i = 0; i < scoreDocs.length; i++) {ScoreDoc scoreDoc = scoreDocs[i];if (scoreDoc != null) {+ originalScoreMap.put(scoreDoc.doc, scoreDoc.score);scoreDocs1[i] = new ScoreDoc(scoreDoc.doc, scoreDoc.score);if (zeroOut) {scoreDoc.score = 0f;@@ -258,4 +269,13 @@ public class ReRankCollector extends TopDocsCollector<ScoreDoc> {return -Float.compare(score1, score2);}}++ static class RescoreDoc extends ScoreDoc {+ public float matchScore;++ public RescoreDoc(ScoreDoc scoreDoc, float matchScore) {+ super(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex);+ this.matchScore = matchScore;+ }+ }}diff --git a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.javaindex 5bf5ce4067d..53373c03de5 100644--- a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java+++ b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java@@ -179,11 +179,10 @@ public class ReRankQParserPlugin extends QParserPlugin {@Overrideprotected float combine(float firstPassScore, boolean secondPassMatches, float secondPassScore) {- float score = firstPassScore;if (secondPassMatches) {- return scoreCombiner.func(score, secondPassScore);+ return scoreCombiner.func(firstPassScore, secondPassScore);}- return score;+ return firstPassScore;}}diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.javaindex 44dcb12491e..e01db9f1f5c 100644--- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java+++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java@@ -18,6 +18,7 @@ package org.apache.solr.search;import java.util.Map;import java.util.Set;+import java.util.stream.Collectors;import org.apache.solr.response.transform.DocTransformer;/**@@ -71,6 +72,31 @@ public abstract class ReturnFields {*/public abstract Map<String, String> getFieldRenames();+ /**+ * A mapping of return fields that depend on score and the names they are associated with.+ *+ * @return a mapping from return field name to the string representation of its definition+ */+ public abstract Map<String, String> getScoreDependentReturnFields();++ /**+ * The requested field names (includes pseudo fields) that do not depend on a score+ *+ * @return Set of field names or <code>null</code> (all fields).+ */+ public Set<String> getNonScoreDependentReturnFieldNames() {+ Set<String> allFieldNames = getRequestedFieldNames();+ Map<String, String> scoreDependentFields = getScoreDependentReturnFields();+ if (allFieldNames == null || scoreDependentFields == null) {+ return allFieldNames;+ } else {+ Set<String> scoreDependentFieldNames = scoreDependentFields.keySet();+ return allFieldNames.stream()+ .filter(fieldName -> !scoreDependentFieldNames.contains(fieldName))+ .collect(Collectors.toSet());+ }+ }+/*** Returns <code>true</code> if the specified field should be returned <em>to the external* client</em> -- either using its own name, or via an alias. This method returns <code>falsediff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.javaindex 4ecda884f67..1a2ce4adbf8 100644--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java@@ -32,7 +32,6 @@ import java.util.List;import java.util.Map;import java.util.Objects;import java.util.Set;-import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.TimeUnit;@@ -183,7 +182,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI@SuppressWarnings({"rawtypes"})private final SolrCache[] cacheList;- private DirectoryFactory directoryFactory;+ private final DirectoryFactory directoryFactory;private final LeafReader leafReader;// only for addIndexes etc (no fieldcache)@@ -194,7 +193,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrIprivate final StatsCache statsCache;- private Set<String> metricNames = ConcurrentHashMap.newKeySet();private SolrMetricsContext solrMetricsContext;private static DirectoryReader getReader(@@ -1875,11 +1873,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrIint last = len;if (last < 0 || last > maxDoc()) last = maxDoc();final int lastDocRequested = last;- int nDocsReturned = 0;int totalHits;- float maxScore;- int[] ids;- float[] scores;+ final float maxScore;+ final DocList docList;final boolean needScores = (cmd.getFlags() & GET_SCORES) != 0;@@ -1934,13 +1930,13 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrIbuildAndRunCollectorChain(qr, query, collector, cmd, pf.postFilter);- ids = new int[nDocsReturned];- scores = new float[nDocsReturned];totalHits = numHits[0];maxScore = totalHits > 0 ? topscore[0] : 0.0f;+ docList =+ new DocSlice(+ 0, 0, new int[0], new float[0], totalHits, maxScore, TotalHits.Relation.EQUAL_TO);// no docs on this page, so cursor doesn't changeqr.setNextCursorMark(cmd.getCursorMark());- hitsRelation = Relation.EQUAL_TO;} else {if (log.isDebugEnabled()) {log.debug("calling from 2, query: {}", query.getClass());@@ -1983,19 +1979,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrIhitsRelation = populateScoresIfNeeded(cmd, needScores, topDocs, query, scoreModeUsed);populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);- nDocsReturned = topDocs.scoreDocs.length;- ids = new int[nDocsReturned];- scores = needScores ? new float[nDocsReturned] : null;- for (int i = 0; i < nDocsReturned; i++) {- ScoreDoc scoreDoc = topDocs.scoreDocs[i];- ids[i] = scoreDoc.doc;- if (scores != null) scores[i] = scoreDoc.score;- }+ int nDocsReturned = topDocs.scoreDocs.length;+ int sliceLen = Math.min(lastDocRequested, nDocsReturned);+ docList =+ new TopDocsSlice(0, sliceLen, topDocs, totalHits, needScores, maxScore, hitsRelation);}-- int sliceLen = Math.min(lastDocRequested, nDocsReturned);- if (sliceLen < 0) sliceLen = 0;- qr.setDocList(new DocSlice(0, sliceLen, ids, scores, totalHits, maxScore, hitsRelation));+ qr.setDocList(docList);}// any DocSet returned is for the query only, without any filtering... that way it may@@ -2008,9 +1997,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrIfinal int nDocsReturned;final int totalHits;final float maxScore;- final int[] ids;- final float[] scores;final DocSet set;+ final DocList docList;final boolean needScores = (cmd.getFlags() & GET_SCORES) != 0;final int maxDoc = maxDoc();@@ -2059,11 +2047,10 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrIset = DocSetUtil.getDocSet(setCollector, this);- nDocsReturned = 0;- ids = new int[nDocsReturned];- scores = new float[nDocsReturned];- totalHits = set.size();- maxScore = totalHits > 0 ? topscore[0] : 0.0f;+ maxScore = set.size() > 0 ? topscore[0] : 0.0f;+ docList =+ new DocSlice(+ 0, 0, new int[0], new float[0], set.size(), maxScore, TotalHits.Relation.EQUAL_TO);// no docs on this page, so cursor doesn't changeqr.setNextCursorMark(cmd.getCursorMark());} else {@@ -2099,47 +2086,25 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI} else {log.trace("MULTI-THREADED search, using CollectorManager in getDocListAndSetNC");- boolean needMaxScore = needScores;MultiThreadedSearcher.SearchResult searchResult =new MultiThreadedSearcher(this)- .searchCollectorManagers(len, cmd, query, true, needMaxScore, true);+ .searchCollectorManagers(len, cmd, query, true, needScores, true);MultiThreadedSearcher.TopDocsResult topDocsResult = searchResult.getTopDocsResult();totalHits = topDocsResult.totalHits;topDocs = topDocsResult.topDocs;maxScore = searchResult.getMaxScore(totalHits);set = new BitDocSet(searchResult.getFixedBitSet());-- // TODO: Is this correct?- // hitsRelation = populateScoresIfNeeded(cmd, needScores, topDocs, query,- // searchResult.scoreMode);-- // nDocsReturned = topDocs.scoreDocs.length;- // TODO: Is this correct?- // hitsRelation = topDocs.totalHits.relation;- // } else {- // hitsRelation = Relation.EQUAL_TO;- // }-+ // TODO: Think about using ScoreMode from searchResult down below}-- populateScoresIfNeeded(cmd, needScores, topDocs, query, ScoreMode.COMPLETE);+ final Relation relation =+ populateScoresIfNeeded(cmd, needScores, topDocs, query, ScoreMode.COMPLETE);populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);nDocsReturned = topDocs.scoreDocs.length;-- ids = new int[nDocsReturned];- scores = needScores ? new float[nDocsReturned] : null;- for (int i = 0; i < nDocsReturned; i++) {- ScoreDoc scoreDoc = topDocs.scoreDocs[i];- ids[i] = scoreDoc.doc;- if (scores != null) scores[i] = scoreDoc.score;- }+ int sliceLen = Math.min(lastDocRequested, nDocsReturned);+ docList = new TopDocsSlice(0, sliceLen, topDocs, totalHits, needScores, maxScore, relation);}- int sliceLen = Math.min(lastDocRequested, nDocsReturned);- if (sliceLen < 0) sliceLen = 0;-- qr.setDocList(- new DocSlice(0, sliceLen, ids, scores, totalHits, maxScore, TotalHits.Relation.EQUAL_TO));+ qr.setDocList(docList);// TODO: if we collect results before the filter, we just need to intersect with// that filter to generate the DocSet for qr.setDocSet()qr.setDocSet(set);diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.javaindex baaaeb270ef..36b0a7b885b 100644--- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java+++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java@@ -40,6 +40,7 @@ import org.apache.solr.common.util.GlobPatternUtil;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.transform.DocTransformer;import org.apache.solr.response.transform.DocTransformers;+import org.apache.solr.response.transform.MatchScoreAugmenter;import org.apache.solr.response.transform.RenameFieldTransformer;import org.apache.solr.response.transform.ScoreAugmenter;import org.apache.solr.response.transform.TransformerFactory;@@ -50,6 +51,7 @@ import org.apache.solr.search.SolrDocumentFetcher.RetrieveFieldsOptimizer;public class SolrReturnFields extends ReturnFields {// Special Field Keyspublic static final String SCORE = "score";+ public static final String MATCH_SCORE = "matchScore";private final List<String> globs = new ArrayList<>(1);@@ -70,6 +72,8 @@ public class SolrReturnFields extends ReturnFields {protected boolean _wantsAllFields = false;protected Map<String, String> renameFields = Collections.emptyMap();+ private final Map<String, String> scoreDependentFields = new HashMap<>();+// Only set currently with the SolrDocumentFetcher.solrDoc method. Primarily used// at this time for testing to ensure we get fields from the expected places.public enum FIELD_SOURCES {@@ -107,13 +111,14 @@ public class SolrReturnFields extends ReturnFields {if (fl == null) {parseFieldList((String[]) null, req);} else {- if (fl.trim().length() == 0) {+ if (fl.trim().isEmpty()) {// legacy thing to support fl=' ' => fl=*,score!// maybe time to drop support for this?// See ConvertedLegacyTest_wantsScore = true;_wantsAllFields = true;transformer = new ScoreAugmenter(SCORE);+ scoreDependentFields.put(SCORE, "");} else {parseFieldList(new String[] {fl}, req);}@@ -535,6 +540,13 @@ public class SolrReturnFields extends ReturnFields {String disp = (key == null) ? field : key;augmenters.addTransformer(new ScoreAugmenter(disp));+ scoreDependentFields.put(disp, disp.equals(SCORE) ? "" : SCORE);+ } else if (MATCH_SCORE.equals(field)) {+ _wantsScore = true;++ String disp = (key == null) ? field : key;+ augmenters.addTransformer(new MatchScoreAugmenter(disp));+ scoreDependentFields.put(disp, disp.equals(MATCH_SCORE) ? "" : MATCH_SCORE);}}@@ -594,6 +606,11 @@ public class SolrReturnFields extends ReturnFields {return _wantsScore;}+ @Override+ public Map<String, String> getScoreDependentReturnFields() {+ return scoreDependentFields;+ }+@Overridepublic DocTransformer getTransformer() {return transformer;diff --git a/solr/core/src/java/org/apache/solr/search/TopDocsSlice.java b/solr/core/src/java/org/apache/solr/search/TopDocsSlice.javanew file mode 100644index 00000000000..d6141e156cd--- /dev/null+++ b/solr/core/src/java/org/apache/solr/search/TopDocsSlice.java@@ -0,0 +1,134 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.search;++import org.apache.lucene.search.TopDocs;+import org.apache.lucene.search.TotalHits;++/**+ * <code>TopDocsSlice</code> implements DocList based off provided <code>TopDocs</code>.+ *+ * @since solr 9.9+ */+public class TopDocsSlice extends DocSlice {++ private final TopDocs topDocs;++ private final boolean hasScores;++ /**+ * Construct a slice off topDocs+ *+ * @param offset starting offset for this range of docs+ * @param len length of results+ * @param matches total number of matches for the query+ */+ public TopDocsSlice(+ int offset,+ int len,+ TopDocs topDocs,+ long matches,+ boolean hasScores,+ float maxScore,+ TotalHits.Relation matchesRelation) {+ super(offset, len, null, null, matches, maxScore, matchesRelation);+ this.topDocs = topDocs;+ this.hasScores = hasScores;+ super.docLength = topDocs.scoreDocs.length;+ }++ @Override+ public TopDocsSlice subset(int offset, int len) {+ if (this.offset == offset && this.len == len) {+ return this;+ }++ // if we didn't store enough (and there was more to store)+ // then we can't take a subset.+ int requestedEnd = offset + len;+ if (requestedEnd > docLength && this.matches > docLength) {+ return null;+ }+ int realEndDoc = Math.min(requestedEnd, docLength);+ int realLen = Math.max(realEndDoc - offset, 0);+ if (this.offset == offset && this.len == realLen) {+ return this;+ }+ return new TopDocsSlice(+ offset, realLen, topDocs, matches, hasScores, maxScore, matchesRelation);+ }++ @Override+ public boolean hasScores() {+ return topDocs != null && hasScores;+ }++ @Override+ public DocIterator iterator() {+ boolean hasMatchScore =+ topDocs.scoreDocs.length > 0 && topDocs.scoreDocs[0] instanceof ReRankCollector.RescoreDoc;+ if (hasMatchScore) {+ return new ReRankedTopDocsIterator();+ } else {+ return new TopDocsIterator();+ }+ }++ class TopDocsIterator implements DocIterator {+ int pos = offset;+ final int end = offset + len;++ @Override+ public boolean hasNext() {+ return pos < end;+ }++ @Override+ public Integer next() {+ return nextDoc();+ }++ /** The remove operation is not supported by this Iterator. */+ @Override+ public void remove() {+ throw new UnsupportedOperationException(+ "The remove operation is not supported by this Iterator.");+ }++ @Override+ public int nextDoc() {+ return topDocs.scoreDocs[pos++].doc;+ }++ @Override+ public float score() {+ return topDocs.scoreDocs[pos - 1].score;+ }+ }++ class ReRankedTopDocsIterator extends TopDocsIterator {++ @Override+ public Float matchScore() {+ try {+ return ((ReRankCollector.RescoreDoc) topDocs.scoreDocs[pos - 1]).matchScore;+ } catch (ClassCastException e) {+ return null;+ }+ }+ }+}diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.javaindex c40a986c024..c9c2e03a62c 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java@@ -912,7 +912,8 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {}public void testAugmentersAndScore() throws Exception {- SolrParams params = params("q", "*:*", "fl", "[docid],x_alias:[value v=10 t=int],score");+ SolrParams params =+ params("q", "*:*", "fl", "[docid],x_alias:[value v=10 t=int],s_alias:score");SolrDocumentList docs = assertSearch(params);assertEquals(params + " => " + docs, 5, docs.getNumFound());// shouldn't matter what doc we pick...@@ -922,7 +923,8 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer);assertTrue(msg, doc.getFieldValue("x_alias") instanceof Integer);assertEquals(msg, 10, doc.getFieldValue("x_alias"));- assertTrue(msg, doc.getFieldValue("score") instanceof Float);+ assertTrue(msg, doc.getFieldValue("s_alias") instanceof Float);+ assertTrue(msg, (Float) doc.getFieldValue("s_alias") > 0);}for (SolrParams p :Arrays.asList(@@ -961,6 +963,22 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {assertTrue(msg, doc.getFieldValue("score") instanceof Float);}}+ params = params("q", "*:*", "fl", "[docid],x_alias:[value v=10 t=int],s_alias:score,score");+ docs = assertSearch(params);+ assertEquals(params + " => " + docs, 5, docs.getNumFound());+ // shouldn't matter what doc we pick...+ for (SolrDocument doc : docs) {+ String msg = params + " => " + doc;+ assertEquals(msg, 4, doc.size());+ assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer);+ assertTrue(msg, doc.getFieldValue("x_alias") instanceof Integer);+ assertEquals(msg, 10, doc.getFieldValue("x_alias"));+ assertTrue(msg, doc.getFieldValue("s_alias") instanceof Float);+ assertTrue(msg, (Float) doc.getFieldValue("s_alias") > 0);+ assertTrue(msg, doc.getFieldValue("score") instanceof Float);+ assertTrue(msg, (Float) doc.getFieldValue("score") > 0);+ assertEquals(msg, doc.getFieldValue("score"), doc.getFieldValue("s_alias"));+ }}public void testAugmentersAndScoreRTG() throws Exception {diff --git a/solr/core/src/test/org/apache/solr/handler/component/MockResponseBuilder.java b/solr/core/src/test/org/apache/solr/handler/component/MockResponseBuilder.javaindex 6ef06bcb6d3..241ff703fb5 100644--- a/solr/core/src/test/org/apache/solr/handler/component/MockResponseBuilder.java+++ b/solr/core/src/test/org/apache/solr/handler/component/MockResponseBuilder.java@@ -56,6 +56,7 @@ public class MockResponseBuilder extends ResponseBuilder {Mockito.when(params.getBool(ShardParams.SHARDS_INFO)).thenReturn(false);Mockito.when(request.getParams()).thenReturn(params);Mockito.when(response.getResponseHeader()).thenReturn(responseHeader);+ Mockito.when(response.getReturnFields()).thenCallRealMethod();List<SearchComponent> components = new ArrayList<>();return new MockResponseBuilder(request, response, components);diff --git a/solr/core/src/test/org/apache/solr/search/DistributedReRankExplainTest.java b/solr/core/src/test/org/apache/solr/search/DistributedReRankExplainTest.javaindex a0c99cb7f4f..f8af2c5fa43 100644--- a/solr/core/src/test/org/apache/solr/search/DistributedReRankExplainTest.java+++ b/solr/core/src/test/org/apache/solr/search/DistributedReRankExplainTest.java@@ -29,6 +29,7 @@ import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.client.solrj.request.UpdateRequest;import org.apache.solr.client.solrj.response.QueryResponse;import org.apache.solr.cloud.SolrCloudTestCase;+import org.apache.solr.common.SolrDocument;import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ShardParams;@@ -154,10 +155,24 @@ public class DistributedReRankExplainTest extends SolrCloudTestCase {final QueryRequest queryRequest =new QueryRequest(SolrParams.wrapDefaults(- params, params(CommonParams.Q, "test_s:hello", "fl", "id,test_s,score")));+ params,+ params(+ CommonParams.Q,+ "test_s:hello",+ "fl",+ "id,test_s,score,originalScore:matchScore,matchScore")));final QueryResponse queryResponse = queryRequest.process(client, COLLECTIONORALIAS);- assertNotNull(queryResponse.getResults().get(0).getFieldValue("test_s"));+ for (SolrDocument doc : queryResponse.getResults()) {+ assertNotNull("test_s", doc.getFieldValue("test_s"));+ assertNotNull("matchScore", doc.getFieldValue("matchScore"));+ assertTrue(queryResponse.toString(), doc.getFieldValue("matchScore") instanceof Float);+ assertNotNull("originalScore", doc.getFieldValue("originalScore"));+ assertTrue(+ doc.getFieldValue("originalScore").toString(),+ doc.getFieldValue("originalScore") instanceof Float);+ assertEquals(doc.getFieldValue("matchScore"), doc.getFieldValue("originalScore"));+ }return queryResponse;}}diff --git a/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java b/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.javaindex cd0836da9bc..1d3d3ca45bd 100644--- a/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java+++ b/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java@@ -680,12 +680,19 @@ public class TestPseudoReturnFields extends SolrTestCaseJ4 {public void testAugmentersAndScore() {assertQ(- req("q", "*:*", "rows", "1", "fl", "[docid],x_alias:[value v=10 t=int],score"),+ req(+ "q",+ "*:*",+ "rows",+ "1",+ "fl",+ "[docid],x_alias:[value v=10 t=int],score,s_alias:score"),"//result[@numFound='5']","//result/doc/int[@name='[docid]']","//result/doc/int[@name='x_alias'][.=10]","//result/doc/float[@name='score']",- "//result/doc[count(*)=3]");+ "//result/doc/float[@name='s_alias']",+ "//result/doc[count(*)=4]");for (SolrParams p :Arrays.asList(params("fl", "[docid],x_alias:[value v=10 t=int],[explain],score"),diff --git a/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.javaindex 0c53d2fcc70..e7d457b06be 100644--- a/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java+++ b/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java@@ -64,6 +64,143 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 {assertEquals(ReRankQParserPlugin.RERANK_OPERATOR, "reRankOperator");}+ @Test+ public void testRerankReturnMatchScore() throws Exception {++ assertU(delQ("*:*"));+ assertU(commit());++ String[] doc = {+ "id", "1", "term_s", "YYYY", "group_s", "group1", "test_ti", "5", "test_tl", "10", "test_tf",+ "2000"+ };+ assertU(adoc(doc));+ assertU(commit());+ String[] doc1 = {+ "id", "2", "term_s", "YYYY", "group_s", "group1", "test_ti", "50", "test_tl", "100",+ "test_tf", "200"+ };+ assertU(adoc(doc1));++ String[] doc2 = {+ "id", "3", "term_s", "YYYY", "test_ti", "5000", "test_tl", "100", "test_tf", "200"+ };+ assertU(adoc(doc2));+ assertU(commit());+ String[] doc3 = {+ "id", "4", "term_s", "YYYY", "test_ti", "500", "test_tl", "1000", "test_tf", "2000"+ };+ assertU(adoc(doc3));++ String[] doc4 = {+ "id", "5", "term_s", "YYYY", "group_s", "group2", "test_ti", "4", "test_tl", "10", "test_tf",+ "2000"+ };+ assertU(adoc(doc4));+ assertU(commit());+ String[] doc5 = {+ "id", "6", "term_s", "YYYY", "group_s", "group2", "test_ti", "10", "test_tl", "100",+ "test_tf", "200"+ };+ assertU(adoc(doc5));+ assertU(commit());++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.add(+ "rq",+ "{!"+ + ReRankQParserPlugin.NAME+ + " "+ + ReRankQParserPlugin.RERANK_QUERY+ + "=$rqq "+ + ReRankQParserPlugin.RERANK_DOCS+ + "=200}");+ params.add("q", "term_s:YYYY");+ params.add("rqq", "{!edismax bf=$bff}*:*");+ params.add("bff", "field(test_ti)");+ params.add("start", "0");+ params.add("rows", "6");+ params.add("df", "text");+ params.add("fl", "id,test_ti,score,matchScore");++ assertQ(+ req(params),+ "*[count(//doc)=6]",+ "//result/doc[1]/str[@name='id'][.='3']",+ "//result/doc[1]/float[@name='score'][.>'10000.03']",+ "//result/doc[1]/float[@name='matchScore'][.>'0.03']",+ "//result/doc[2]/str[@name='id'][.='4']",+ "//result/doc[2]/float[@name='score'][.>'1000.03']",+ "//result/doc[2]/float[@name='matchScore'][.>'0.03']",+ "//result/doc[3]/str[@name='id'][.='2']",+ "//result/doc[4]/str[@name='id'][.='6']",+ "//result/doc[5]/str[@name='id'][.='1']",+ "//result/doc[6]/str[@name='id'][.='5']");+ }++ @Test+ public void testRerankReturnMatchScoreNotRequested() throws Exception {++ assertU(delQ("*:*"));+ assertU(commit());++ String[] doc = {+ "id", "1", "term_s", "YYYY", "group_s", "group1", "test_ti", "5", "test_tl", "10", "test_tf",+ "2000"+ };+ assertU(adoc(doc));+ assertU(commit());+ String[] doc1 = {+ "id", "2", "term_s", "YYYY", "group_s", "group1", "test_ti", "50", "test_tl", "100",+ "test_tf", "200"+ };+ assertU(adoc(doc1));++ String[] doc2 = {+ "id", "3", "term_s", "YYYY", "test_ti", "5000", "test_tl", "100", "test_tf", "200"+ };+ assertU(adoc(doc2));+ assertU(commit());+ String[] doc3 = {+ "id", "4", "term_s", "YYYY", "test_ti", "500", "test_tl", "1000", "test_tf", "2000"+ };+ assertU(adoc(doc3));++ String[] doc4 = {+ "id", "5", "term_s", "YYYY", "group_s", "group2", "test_ti", "4", "test_tl", "10", "test_tf",+ "2000"+ };+ assertU(adoc(doc4));+ assertU(commit());+ String[] doc5 = {+ "id", "6", "term_s", "YYYY", "group_s", "group2", "test_ti", "10", "test_tl", "100",+ "test_tf", "200"+ };+ assertU(adoc(doc5));+ assertU(commit());++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.add(+ "rq",+ "{!"+ + ReRankQParserPlugin.NAME+ + " "+ + ReRankQParserPlugin.RERANK_QUERY+ + "=$rqq "+ + ReRankQParserPlugin.RERANK_DOCS+ + "=200}");+ params.add("q", "term_s:YYYY");+ params.add("rqq", "{!edismax bf=$bff}*:*");+ params.add("bff", "field(test_ti)");+ params.add("start", "0");+ params.add("rows", "6");+ params.add("df", "text");+ params.add("fl", "id,test_ti,score");++ String response = JQ(req(params));+ assertFalse(response.contains("matchScore"));+ }+@Testpublic void testReRankQueries() {diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.javaindex ebf8661bd70..8c2a92c9484 100644--- a/solr/modules/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java+++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java@@ -46,6 +46,7 @@ import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.ResultContext;import org.apache.solr.response.transform.DocTransformer;import org.apache.solr.response.transform.TransformerFactory;+import org.apache.solr.search.DocIterationInfo;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.util.SolrPluginUtils;@@ -402,8 +403,9 @@ public class LTRFeatureLoggerTransformerFactory extends TransformerFactory {}@Override- public void transform(SolrDocument doc, int docid, float score) throws IOException {- implTransform(doc, docid, score);+ public void transform(SolrDocument doc, int docid, DocIterationInfo docInfo)+ throws IOException {+ implTransform(doc, docid, docInfo);}@Override@@ -411,7 +413,8 @@ public class LTRFeatureLoggerTransformerFactory extends TransformerFactory {implTransform(doc, docid, null);}- private void implTransform(SolrDocument doc, int docid, Float score) throws IOException {+ private void implTransform(SolrDocument doc, int docid, DocIterationInfo docInfo)+ throws IOException {LTRScoringQuery rerankingQuery = rerankingQueries[0];LTRScoringQuery.ModelWeight rerankingModelWeight = modelWeights[0];for (int i = 1; i < rerankingQueries.length; i++) {@@ -430,7 +433,7 @@ public class LTRFeatureLoggerTransformerFactory extends TransformerFactory {LTRRescorer.extractFeaturesInfo(rerankingModelWeight,docid,- (!docsWereReranked ? score : null),+ (!docsWereReranked && docInfo != null) ? docInfo.score() : null,leafContexts));}doc.addField(name, featureVector);diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/response/transform/LTRInterleavingTransformerFactory.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/response/transform/LTRInterleavingTransformerFactory.javaindex e08b564fb22..8105777b857 100644--- a/solr/modules/ltr/src/java/org/apache/solr/ltr/response/transform/LTRInterleavingTransformerFactory.java+++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/response/transform/LTRInterleavingTransformerFactory.java@@ -87,11 +87,6 @@ public class LTRInterleavingTransformerFactory extends TransformerFactory {}}- @Override- public void transform(SolrDocument doc, int docid, float score) throws IOException {- implTransform(doc, docid);- }-@Overridepublic void transform(SolrDocument doc, int docid) throws IOException {implTransform(doc, docid);diff --git a/solr/solr-ref-guide/modules/query-guide/pages/query-re-ranking.adoc b/solr/solr-ref-guide/modules/query-guide/pages/query-re-ranking.adocindex 0cf15be3ff6..e4533412f5e 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/query-re-ranking.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/query-re-ranking.adoc@@ -107,6 +107,14 @@ q=greetings&rq={!rerank reRankQuery=$rqq reRankDocs=1000 reRankWeight=3}&rqq=(hi----If a document matches the original query, but does not match the re-ranking query, the document's original score will remain.+For reranked documents, an additional `matchScore` field in the response will indicate the original score for a reranked doc. This+is the score for the document prior to rerank being applied. For documents that were not reranked, the matchScore and score fields+will have the same value. For the example above, you would use the following to return the match score:++[source,text]+----+q=greetings&rq={!rerank reRankQuery=$rqq reRankDocs=1000 reRankWeight=3}&rqq=(hi+hello+hey+hiya)&fl=id,matchScore+----Setting `reRankOperator` to `multiply` will multiply the three numbers instead. This means that other multiplying operations such as xref:edismax-query-parser.adoc#extended-dismax-parameters[eDisMax `boost` functions] can be converted to Re-Rank operations.diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.javaindex a0176d31dc0..16ba4ee00db 100644--- a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java+++ b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java@@ -20,12 +20,14 @@ import java.io.IOException;import java.util.ArrayList;import java.util.Arrays;import java.util.Collection;+import java.util.HashMap;import java.util.Iterator;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Set;import java.util.function.BiConsumer;+import org.apache.solr.common.util.CollectionUtil;import org.apache.solr.common.util.NamedList;/**@@ -195,6 +197,19 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument>return null;}+ /** Get the value or collection of values for a given field. */+ public Map<String, Object> getSubsetOfFields(Set<String> fieldNames) {+ final HashMap<String, Object> subset = CollectionUtil.newHashMap(fieldNames.size());+ fieldNames.forEach(+ f -> {+ Object v = getFieldValue(f);+ if (v != null) {+ subset.put(f, getFieldValue(f));+ }+ });+ return subset;+ }+@Overridepublic String toString() {return "SolrDocument" + _fields;