Apache Solr open-source search software
SOLR-15038: Add elevateOnlyDocsMatchingQuery and collectElevatedDocsWhenCollapsing parameters to query elevation.
Tobias Kaessmannf142bf9c5437
diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policyindex 12c44a8e5b4..0058bd6089a 100644--- a/gradle/testing/randomization/policies/solr-tests.policy+++ b/gradle/testing/randomization/policies/solr-tests.policy@@ -22,7 +22,7 @@grant {// 3rd party jar resources (where symlinks are not supported), test-files/ resourcespermission java.io.FilePermission "${common.dir}${/}-", "read";- permission java.io.FilePermission "${common.dir}${/}..${/}solr${/}-", "read";+ permission java.io.FilePermission "${common.dir}${/}..${/}solr${/}-", "read,write";// system jar resourcespermission java.io.FilePermission "${java.home}${/}-", "read";diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex bfb00b66432..99dcef50a28 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -239,6 +239,9 @@ Improvements* SOLR-15101: Add "list" and "delete" APIs for managing incremental backups (Jason Gerlowski, shalin, Cao Manh Dat)+* SOLR-15038: Add elevateOnlyDocsMatchingQuery and collectElevatedDocsWhenCollapsing parameters to query elevation.+ (Dennis Berger, Tobias Kässmann via Bruno Roustant)+Optimizations---------------------* SOLR-15079: Block Collapse - Faster collapse code when groups are co-located via Block Join style nested doc indexing.diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.javaindex 8318cc3ff18..bbaa9a1d02a 100644--- a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java@@ -509,7 +509,10 @@ public class QueryElevationComponent extends SearchComponent implements SolrCorerb.setQuery(new BoostQuery(elevation.includeQuery, 0f));} else {BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();- queryBuilder.add(rb.getQuery(), BooleanClause.Occur.SHOULD);+ BooleanClause.Occur queryOccurrence =+ params.getBool(QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, false) ?+ BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;+ queryBuilder.add(rb.getQuery(), queryOccurrence);queryBuilder.add(new BoostQuery(elevation.includeQuery, 0f), BooleanClause.Occur.SHOULD);if (elevation.excludeQueries != null) {if (params.getBool(QueryElevationParams.MARK_EXCLUDES, false)) {diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.javaindex 9e52f7173c7..a95f5659b4a 100644--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java@@ -146,6 +146,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {*/public static final String HINT_BLOCK = "block";+ /**+ * If elevation is used in combination with the collapse query parser, we can define that we only want to return the+ * representative and not all elevated docs by setting this parameter to false (true by default).+ */+ public static String COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING = "collectElevatedDocsWhenCollapsing";+/*** @deprecated use {@link NullPolicy} instead.*/@@ -585,6 +591,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {private int nullPolicy;private float nullScore = -Float.MAX_VALUE;private int nullDoc = -1;+ private boolean collectElevatedDocsWhenCollapsing;private FloatArrayList nullScores;private final BoostedDocsCollector boostedDocsCollector;@@ -594,9 +601,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {DocValuesProducer collapseValuesProducer,int nullPolicy,IntIntHashMap boostDocsMap,- IndexSearcher searcher) throws IOException {+ IndexSearcher searcher,+ boolean collectElevatedDocsWhenCollapsing) throws IOException {this.maxDoc = maxDoc;this.contexts = new LeafReaderContext[segments];+ this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();for(int i=0; i<con.size(); i++) {contexts[i] = con.get(i);@@ -653,12 +662,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {ord = -1;}}-- // Check to see if we have documents boosted by the QueryElevationComponent- if (0 <= ord) {- if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;- } else {- if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;++ if (collectElevatedDocsWhenCollapsing) {+ // Check to see if we have documents boosted by the QueryElevationComponent+ if (0 <= ord) {+ if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;+ } else {+ if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;+ }}if(ord > -1) {@@ -785,6 +796,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {private int nullDoc = -1;private FloatArrayList nullScores;private String field;+ private boolean collectElevatedDocsWhenCollapsing;private final BoostedDocsCollector boostedDocsCollector;@@ -794,9 +806,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {int size,String field,IntIntHashMap boostDocsMap,- IndexSearcher searcher) {+ IndexSearcher searcher,+ boolean collectElevatedDocsWhenCollapsing) {this.maxDoc = maxDoc;this.contexts = new LeafReaderContext[segments];+ this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();for(int i=0; i<con.size(); i++) {contexts[i] = con.get(i);@@ -827,8 +841,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {final int globalDoc = docBase+contextDoc;if (collapseValues.advanceExact(contextDoc)) {final int collapseValue = (int) collapseValues.longValue();- // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)- if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return;++ if (collectElevatedDocsWhenCollapsing) {+ // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)+ if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return;+ }float score = scorer.score();final int idx;@@ -847,9 +864,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {}} else { // Null Group...-- // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)- if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;++ if (collectElevatedDocsWhenCollapsing){+ // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)+ if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;+ }if(nullPolicy == NullPolicy.COLLAPSE.getCode()) {float score = scorer.score();@@ -958,7 +977,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {private OrdFieldValueStrategy collapseStrategy;private boolean needsScores4Collapsing;private boolean needsScores;-++ private boolean collectElevatedDocsWhenCollapsing;+private final BoostedDocsCollector boostedDocsCollector;public OrdFieldValueCollector(int maxDoc,@@ -971,10 +992,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {boolean needsScores,FieldType fieldType,IntIntHashMap boostDocsMap,- FunctionQuery funcQuery, IndexSearcher searcher) throws IOException{+ FunctionQuery funcQuery, IndexSearcher searcher,+ boolean collectElevatedDocsWhenCollapsing) throws IOException{assert ! GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type);+ this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;this.maxDoc = maxDoc;this.contexts = new LeafReaderContext[segments];List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();@@ -1053,12 +1076,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {ord = segmentValues.ordValue();}}-- // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)- if (-1 == ord) {- if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;- } else {- if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;++ if (collectElevatedDocsWhenCollapsing){+ // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)+ if (-1 == ord) {+ if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;+ } else {+ if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;+ }}collapseStrategy.collapse(ord, contextDoc, globalDoc);@@ -1166,6 +1191,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {private String collapseField;private final BoostedDocsCollector boostedDocsCollector;+ private boolean collectElevatedDocsWhenCollapsing;public IntFieldValueCollector(int maxDoc,int size,@@ -1179,7 +1205,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {FieldType fieldType,IntIntHashMap boostDocsMap,FunctionQuery funcQuery,- IndexSearcher searcher) throws IOException{+ IndexSearcher searcher,+ boolean collectElevatedDocsWhenCollapsing) throws IOException{+ this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;assert ! GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type);@@ -1243,10 +1271,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {collapseStrategy.collapse(collapseKey, contextDoc, globalDoc);} else { // Null Group...-- // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)- if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;+ if (collectElevatedDocsWhenCollapsing){+ // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)+ if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;+ }if (NullPolicy.IGNORE.getCode() != nullPolicy) {collapseStrategy.collapseNullGroup(contextDoc, globalDoc);}@@ -1894,20 +1923,23 @@ public class CollapsingQParserPlugin extends QParserPlugin {int maxDoc = searcher.maxDoc();int leafCount = searcher.getTopReaderContext().leaves().size();+ SolrRequestInfo req = SolrRequestInfo.getRequestInfo();+ boolean collectElevatedDocsWhenCollapsing = req != null && req.getReq().getParams().getBool(COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, true);+if (GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type)) {if (collapseFieldType instanceof StrField) {if (blockCollapse) {return new BlockOrdScoreCollector(collapseField, nullPolicy, boostDocs);}- return new OrdScoreCollector(maxDoc, leafCount, docValuesProducer, nullPolicy, boostDocs, searcher);+ return new OrdScoreCollector(maxDoc, leafCount, docValuesProducer, nullPolicy, boostDocs, searcher, collectElevatedDocsWhenCollapsing);} else if (isNumericCollapsible(collapseFieldType)) {if (blockCollapse) {return new BlockIntScoreCollector(collapseField, nullPolicy, boostDocs);}- return new IntScoreCollector(maxDoc, leafCount, nullPolicy, size, collapseField, boostDocs, searcher);+ return new IntScoreCollector(maxDoc, leafCount, nullPolicy, size, collapseField, boostDocs, searcher, collectElevatedDocsWhenCollapsing);} else {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,@@ -1937,7 +1969,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {minMaxFieldType,boostDocs,funcQuery,- searcher);+ searcher,+ collectElevatedDocsWhenCollapsing);} else if (isNumericCollapsible(collapseFieldType)) {@@ -1962,7 +1995,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {minMaxFieldType,boostDocs,funcQuery,- searcher);+ searcher,+ collectElevatedDocsWhenCollapsing);} else {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"Collapsing field should be of either String, Int or Float type");diff --git a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.javaindex 3f7a3407580..4db38252378 100644--- a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java+++ b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java@@ -37,6 +37,7 @@ import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.SolrCore;import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.search.CollapsingQParserPlugin;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.util.FileUtils;import org.junit.Before;@@ -932,6 +933,106 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {}}+ @Test+ public void testOnlyDocsInSearchResultsWillBeElevated() throws Exception {+ try {+ init("schema12.xml");+ assertU(adoc("id", "1", "title", "XXXX", "str_s1", "a"));+ assertU(adoc("id", "2", "title", "YYYY", "str_s1", "b"));+ assertU(adoc("id", "3", "title", "ZZZZ", "str_s1", "c"));++ assertU(adoc("id", "4", "title", "XXXX XXXX", "str_s1", "x"));+ assertU(adoc("id", "5", "title", "YYYY YYYY", "str_s1", "y"));+ assertU(adoc("id", "6", "title", "XXXX XXXX", "str_s1", "z"));+ assertU(adoc("id", "7", "title", "AAAA", "str_s1", "a"));++ assertU(commit());++ // default behaviour+ assertQ("", req(+ CommonParams.Q, "YYYY",+ CommonParams.QT, "/elevate",+ QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, "false",+ CommonParams.FL, "id, score, [elevated]"),+ "//*[@numFound='3']",+ "//result/doc[1]/str[@name='id'][.='1']",+ "//result/doc[2]/str[@name='id'][.='2']",+ "//result/doc[3]/str[@name='id'][.='5']",+ "//result/doc[1]/bool[@name='[elevated]'][.='true']",+ "//result/doc[2]/bool[@name='[elevated]'][.='true']",+ "//result/doc[3]/bool[@name='[elevated]'][.='false']"+ );++ // only docs that matches q+ assertQ("", req(+ CommonParams.Q, "YYYY",+ CommonParams.QT, "/elevate",+ QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, "true",+ CommonParams.FL, "id, score, [elevated]"),+ "//*[@numFound='2']",+ "//result/doc[1]/str[@name='id'][.='2']",+ "//result/doc[2]/str[@name='id'][.='5']",+ "//result/doc[1]/bool[@name='[elevated]'][.='true']",+ "//result/doc[2]/bool[@name='[elevated]'][.='false']"+ );++++ } finally {+ delete();+ }+ }++ @Test+ public void testOnlyRepresentativeIsVisibleWhenCollapsing() throws Exception {+ try {+ init("schema12.xml");+ assertU(adoc("id", "1", "title", "ZZZZ", "str_s1", "a"));+ assertU(adoc("id", "2", "title", "ZZZZ", "str_s1", "b"));+ assertU(adoc("id", "3", "title", "ZZZZ ZZZZ", "str_s1", "a"));+ assertU(adoc("id", "4", "title", "ZZZZ ZZZZ", "str_s1", "c"));++ assertU(commit());++ // default behaviour - all elevated docs are visible+ assertQ("", req(+ CommonParams.Q, "ZZZZ",+ CommonParams.QT, "/elevate",+ CollapsingQParserPlugin.COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, "true",+ CommonParams.FQ, "{!collapse field=str_s1 sort='score desc'}",+ CommonParams.FL, "id, score, [elevated]"),+ "//*[@numFound='4']",+ "//result/doc[1]/str[@name='id'][.='1']",+ "//result/doc[2]/str[@name='id'][.='2']",+ "//result/doc[3]/str[@name='id'][.='3']",+ "//result/doc[4]/str[@name='id'][.='4']",+ "//result/doc[1]/bool[@name='[elevated]'][.='true']",+ "//result/doc[2]/bool[@name='[elevated]'][.='true']",+ "//result/doc[3]/bool[@name='[elevated]'][.='true']",+ "//result/doc[4]/bool[@name='[elevated]'][.='false']"+ );++ // only representative elevated doc visible+ assertQ("", req(+ CommonParams.Q, "ZZZZ",+ CommonParams.QT, "/elevate",+ CollapsingQParserPlugin.COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, "false",+ CommonParams.FQ, "{!collapse field=str_s1 sort='score desc'}",+ CommonParams.FL, "id, score, [elevated]"),+ "//*[@numFound='3']",+ "//result/doc[1]/str[@name='id'][.='2']",+ "//result/doc[2]/str[@name='id'][.='3']",+ "//result/doc[3]/str[@name='id'][.='4']",+ "//result/doc[1]/bool[@name='[elevated]'][.='true']",+ "//result/doc[2]/bool[@name='[elevated]'][.='true']",+ "//result/doc[3]/bool[@name='[elevated]'][.='false']"+ );++ } finally {+ delete();+ }+ }+private static Set<BytesRef> toIdSet(String... ids) {return Arrays.stream(ids).map(BytesRef::new).collect(Collectors.toSet());}diff --git a/solr/solr-ref-guide/src/collapse-and-expand-results.adoc b/solr/solr-ref-guide/src/collapse-and-expand-results.adocindex f3846c1074c..379136a8e3b 100644--- a/solr/solr-ref-guide/src/collapse-and-expand-results.adoc+++ b/solr/solr-ref-guide/src/collapse-and-expand-results.adoc@@ -81,6 +81,10 @@ The data structures used for collapsing grow dynamically when collapsing on nume+The default is 100,000.+`collectElevatedDocsWhenCollapsing`::+In combination with the <<collapse-and-expand-results.adoc#collapsing-query-parser,Collapse Query Parser>> all elevated docs are visible at the beginning of the result set.+If this parameter is `false`, only the representative is visible if the elevated docs has the same collapse key (default is `true`).+=== Sample Usage Syntaxdiff --git a/solr/solr-ref-guide/src/the-query-elevation-component.adoc b/solr/solr-ref-guide/src/the-query-elevation-component.adocindex f43a73a6ae4..75b187b3b4f 100644--- a/solr/solr-ref-guide/src/the-query-elevation-component.adoc+++ b/solr/solr-ref-guide/src/the-query-elevation-component.adoc@@ -93,6 +93,10 @@ they be subject to whatever the sort criteria is? True by default.This is also a request parameter, which will override the config.The effect is most apparent when forceElevation is true and there is sorting on fields.+`elevateOnlyDocsMatchingQuery`::+By default, the component will also elevate docs that aren't part of the search result (matching the query).+If you only want to elevate the docs that are part of the search result, set this to `true` (default is `false`).+=== The elevate.xml FileElevated query results can be configured in an external XML file specified in the `config-file` argument. An `elevate.xml` file might look like this:diff --git a/solr/solrj/src/java/org/apache/solr/common/params/QueryElevationParams.java b/solr/solrj/src/java/org/apache/solr/common/params/QueryElevationParams.javaindex 9a31f691889..e77408e47bc 100644--- a/solr/solrj/src/java/org/apache/solr/common/params/QueryElevationParams.java+++ b/solr/solrj/src/java/org/apache/solr/common/params/QueryElevationParams.java@@ -55,4 +55,11 @@ public interface QueryElevationParams {* they be subject to whatever the sort criteria is? True by default.*/String USE_CONFIGURED_ELEVATED_ORDER = "useConfiguredElevatedOrder";++ /**+ * By default, the component will also elevate docs that aren't part of the search result (matching the query).+ * If you only want to elevate the docs that are part of the search result, set this to true. False by default.+ */+ String ELEVATE_ONLY_DOCS_MATCHING_QUERY = "elevateOnlyDocsMatchingQuery";+}\ No newline at end of file