Apache Solr open-source search software
SOLR-16643: reRankOperator=multiply/replace options (#1431)
Andy Webb3ac9714557d4
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex f5be3131cf0..bc9448b48e1 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -63,6 +63,8 @@ New Features* SOLR-16646: New function query operator isnan to verify if value is NaN (Gabriel Magno via Kevin Risden)+* SOLR-16643: Add reRankOperator=multiply/replace options to rerank query parser (Andy Webb, Christine Poerschke, Mikhail Khludnev)+Improvements---------------------diff --git a/solr/core/src/java/org/apache/solr/search/ReRankOperator.java b/solr/core/src/java/org/apache/solr/search/ReRankOperator.javanew file mode 100644index 00000000000..0b6583e095e--- /dev/null+++ b/solr/core/src/java/org/apache/solr/search/ReRankOperator.java@@ -0,0 +1,42 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.search;++import java.util.Locale;+import org.apache.solr.common.SolrException;++public enum ReRankOperator {+ ADD,+ MULTIPLY,+ REPLACE;++ public static ReRankOperator get(String p) {+ if (p != null) {+ try {+ return ReRankOperator.valueOf(p.toUpperCase(Locale.ROOT));+ } catch (Exception ex) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST, "Invalid reRankOperator: " + p);+ }+ }+ return null;+ }++ public String toLower() {+ return toString().toLowerCase(Locale.ROOT);+ }+}diff --git a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.javaindex 5e3f25e9890..6804942acd6 100644--- a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java+++ b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java@@ -44,6 +44,9 @@ public class ReRankQParserPlugin extends QParserPlugin {public static final String RERANK_WEIGHT = "reRankWeight";public static final double RERANK_WEIGHT_DEFAULT = 2.0d;+ public static final String RERANK_OPERATOR = "reRankOperator";+ public static final String RERANK_OPERATOR_DEFAULT = "add";+@Overridepublic QParser createParser(String query, SolrParams localParams, SolrParams params, SolrQueryRequest req) {@@ -68,21 +71,43 @@ public class ReRankQParserPlugin extends QParserPlugin {Query reRankQuery = reRankParser.parse();int reRankDocs = localParams.getInt(RERANK_DOCS, RERANK_DOCS_DEFAULT);- reRankDocs = Math.max(1, reRankDocs); //+ reRankDocs = Math.max(1, reRankDocs);double reRankWeight = localParams.getDouble(RERANK_WEIGHT, RERANK_WEIGHT_DEFAULT);- return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight);+ ReRankOperator reRankOperator =+ ReRankOperator.get(localParams.get(RERANK_OPERATOR, RERANK_OPERATOR_DEFAULT));++ return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight, reRankOperator);}}private static final class ReRankQueryRescorer extends QueryRescorer {- final double reRankWeight;+ final BiFloatFunction scoreCombiner;++ @FunctionalInterface+ interface BiFloatFunction {+ float func(float a, float b);+ }- public ReRankQueryRescorer(Query reRankQuery, double reRankWeight) {+ public ReRankQueryRescorer(+ Query reRankQuery, double reRankWeight, ReRankOperator reRankOperator) {super(reRankQuery);- this.reRankWeight = reRankWeight;+ switch (reRankOperator) {+ case ADD:+ scoreCombiner = (score, second) -> (float) (score + reRankWeight * second);+ break;+ case MULTIPLY:+ scoreCombiner = (score, second) -> (float) (score * reRankWeight * second);+ break;+ case REPLACE:+ scoreCombiner = (score, second) -> (float) (reRankWeight * second);+ break;+ default:+ scoreCombiner = null;+ throw new IllegalArgumentException("Unexpected: reRankOperator=" + reRankOperator);+ }}@Override@@ -90,7 +115,7 @@ public class ReRankQParserPlugin extends QParserPlugin {float firstPassScore, boolean secondPassMatches, float secondPassScore) {float score = firstPassScore;if (secondPassMatches) {- score = (float) (score + reRankWeight * secondPassScore);+ return scoreCombiner.func(score, secondPassScore);}return score;}@@ -99,6 +124,7 @@ public class ReRankQParserPlugin extends QParserPlugin {private static final class ReRankQuery extends AbstractReRankQuery {private final Query reRankQuery;private final double reRankWeight;+ private final ReRankOperator reRankOperator;@Overridepublic int hashCode() {@@ -106,7 +132,8 @@ public class ReRankQParserPlugin extends QParserPlugin {+ mainQuery.hashCode()+ reRankQuery.hashCode()+ (int) reRankWeight- + reRankDocs;+ + reRankDocs+ + reRankOperator.hashCode();}@Override@@ -118,13 +145,19 @@ public class ReRankQParserPlugin extends QParserPlugin {return mainQuery.equals(rrq.mainQuery)&& reRankQuery.equals(rrq.reRankQuery)&& reRankWeight == rrq.reRankWeight- && reRankDocs == rrq.reRankDocs;+ && reRankDocs == rrq.reRankDocs+ && reRankOperator.equals(rrq.reRankOperator);}- public ReRankQuery(Query reRankQuery, int reRankDocs, double reRankWeight) {- super(defaultQuery, reRankDocs, new ReRankQueryRescorer(reRankQuery, reRankWeight));+ public ReRankQuery(+ Query reRankQuery, int reRankDocs, double reRankWeight, ReRankOperator reRankOperator) {+ super(+ defaultQuery,+ reRankDocs,+ new ReRankQueryRescorer(reRankQuery, reRankWeight, reRankOperator));this.reRankQuery = reRankQuery;this.reRankWeight = reRankWeight;+ this.reRankOperator = reRankOperator;}@Override@@ -135,13 +168,15 @@ public class ReRankQParserPlugin extends QParserPlugin {sb.append(" mainQuery='").append(mainQuery.toString()).append("' ");sb.append(RERANK_QUERY).append("='").append(reRankQuery.toString()).append("' ");sb.append(RERANK_DOCS).append('=').append(reRankDocs).append(' ');- sb.append(RERANK_WEIGHT).append('=').append(reRankWeight).append('}');+ sb.append(RERANK_WEIGHT).append('=').append(reRankWeight).append(' ');+ sb.append(RERANK_OPERATOR).append('=').append(reRankOperator.toLower()).append('}');return sb.toString();}@Overrideprotected Query rewrite(Query rewrittenMainQuery) throws IOException {- return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight).wrap(rewrittenMainQuery);+ return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight, reRankOperator)+ .wrap(rewrittenMainQuery);}}}diff --git a/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.javaindex 8b978ff96e5..b25d725c0a7 100644--- a/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java+++ b/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java@@ -16,7 +16,9 @@*/package org.apache.solr.search;+import java.util.Locale;import java.util.Map;+import java.util.function.Function;import java.util.stream.Collectors;import java.util.stream.IntStream;import org.apache.solr.SolrTestCaseJ4;@@ -57,6 +59,8 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 {assertEquals(ReRankQParserPlugin.RERANK_WEIGHT, "reRankWeight");assertEquals(ReRankQParserPlugin.RERANK_WEIGHT_DEFAULT, 2.0d, 0.0d);++ assertEquals(ReRankQParserPlugin.RERANK_OPERATOR, "reRankOperator");}@Test@@ -126,6 +130,48 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 {"//result/doc[5]/str[@name='id'][.='1']","//result/doc[6]/str[@name='id'][.='5']");+ // check each of the reRankOperators return the expected score for item 3+ for (Map.Entry<String, String> scoreByOp :+ Map.of("add", "10002.1", "multiply", "1000.2", "replace", "10002.0").entrySet()) {+ params = new ModifiableSolrParams();+ String operation = scoreByOp.getKey();+ if (random().nextBoolean()) {+ operation = operation.toUpperCase(Locale.ROOT);+ }+ final Function<String, String> rerankQueryByOp =+ op ->+ "{!"+ + ReRankQParserPlugin.NAME+ + " "+ + ReRankQParserPlugin.RERANK_QUERY+ + "=$rqq "+ + ReRankQParserPlugin.RERANK_OPERATOR+ + "="+ + op+ + " "+ + ReRankQParserPlugin.RERANK_DOCS+ + "=200}";+ params.add("rq", rerankQueryByOp.apply(operation));+ params.add("q", "term_s:YYYY^=0.1"); // force score=0.1+ params.add("rqq", "{!edismax bf=$bff}*:*"); // returns 1 + $bff+ params.add("bff", "field(test_ti)"); // test_ti=5000 for item 3+ params.add("start", "0");+ params.add("rows", "6");+ params.add("df", "text");+ params.add("fl", "id,score");+ assertQ(+ req(params),+ "*[count(//doc)=6]",+ "//result/doc[1]/str[@name='id'][.='3']",+ "//result/doc[1]/float[@name='score'][.='" + scoreByOp.getValue() + "']");+ final String badOp =+ random().nextBoolean()+ ? operation + operation+ : operation.substring(0, operation.length() - 1);+ params.set("rq", rerankQueryByOp.apply(badOp));+ assertQEx("Wrong reRankOperation:" + badOp, req(params), SolrException.ErrorCode.BAD_REQUEST);+ }+params = new ModifiableSolrParams();params.add("rq",@@ -960,7 +1006,7 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 {"id_p_i", String.valueOf(i),"field_t",IntStream.range(0, numDocs)- .mapToObj(val -> Integer.toString(val))+ .mapToObj(Integer::toString).collect(Collectors.joining(" "))));}assertU(commit());diff --git a/solr/solr-ref-guide/modules/query-guide/pages/query-re-ranking.adoc b/solr/solr-ref-guide/modules/query-guide/pages/query-re-ranking.adocindex 82d971fa30a..fe32836c458 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/query-re-ranking.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/query-re-ranking.adoc@@ -67,9 +67,18 @@ This number will be treated as a minimum, and may be increased internally automa|Optional |Default: `2.0`|===+-A multiplicative factor that will be applied to the score from the reRankQuery for each of the top matching documents, before that score is added to the original score.+A multiplicative factor that will be applied to the score from the reRankQuery for each of the top matching documents, before that score is combined with the original score.-In the example below, the top 1000 documents matching the query "greetings" will be re-ranked using the query "(hi hello hey hiya)".+`reRankOperator`::+++[%autowidth,frame=none]+|===+|Optional |Default: `add`+|===+++By default the score from the reRankQuery multiplied by the `reRankWeight` is added to the original score.++In the example below using the default `add` behaviour, the top 1000 documents matching the query "greetings" will be re-ranked using the query "(hi hello hey hiya)".The resulting scores for each of those 1000 documents will be 3 times their score from the "(hi hello hey hiya)", plus the score from the original "greetings" query:[source,text]@@ -79,6 +88,25 @@ q=greetings&rq={!rerank reRankQuery=$rqq reRankDocs=1000 reRankWeight=3}&rqq=(hiIf a document matches the original query, but does not match the re-ranking query, the document's original score will remain.+Setting `reRankOperator` to `multiply` will multiply the three numbers instead. This means that other multiplying operations such as xref:edismax-query-parser.adoc#extended-dismax-parameters[eDisMax `boost` functions] can be converted to Re-Rank operations.++In the example below, the scores for the top 1000 documents matching the query "phone" will be multiplied by a function of the `price` field.++[source,text]+----+q=phone&rq={!rerank reRankQuery=$rqq reRankDocs=1000 reRankWeight=1 reRankOperator=multiply}&rqq={!func v=div(1,sum(1,price))}+----++Setting `reRankOperator` to `replace` will replace the score, so the final scores can be independent of documents' original scores.++In the example below, the scores for the top 1000 documents matching the query "phone" will be replaced with a function of the `price` field.++[source,text]+----+q=phone&rq={!rerank reRankQuery=$rqq reRankDocs=1000 reRankWeight=1 reRankOperator=replace}&rqq={!func v=div(1,sum(1,price))}+----++=== LTR Query ParserThe `ltr` stands for Learning To Rank, please see xref:learning-to-rank.adoc[] for more detailed information.