Apache Solr open-source search software
SOLR-18290: combined-query / RRF : support shards.rows (#4546)
Sonu Sharma4a3bfa598e39
diff --git a/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml b/changelog/unreleased/SOLR-17319-added_combined_query_RRF.ymlindex 17d64bf0233..a3f4f6177f8 100644--- a/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml+++ b/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml@@ -6,3 +6,6 @@ authors:links:- name: SOLR-17319url: https://issues.apache.org/jira/browse/SOLR-17319+ - name: SOLR-18290+ url: https://issues.apache.org/jira/browse/SOLR-18290+diff --git a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.javaindex 1e5c0b11aa1..28b4a80bb6e 100644--- a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java+++ b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java@@ -323,13 +323,14 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw// Used to filter per-query docs so that RRF doesn't reintroduce docs// excluded by collapse at the shard level.Map<String, Set<Object>> combinedDocIdsPerShard = HashMap.newHashMap(sreq.responses.size());+ int perQueryQueueSize = Math.max(ss.getOffset() + ss.getCount(), rb.shards_rows);// TODO: to be parallelized outer loopfor (int queryIndex = 0; queryIndex < queriesToCombineKeys.length; queryIndex++) {int failedShardCount = 0;long queryNumFound = 0;long queryApproximateTotalHits = 0;final ShardDocQueue queuePerQuery =- newShardDocQueue(rb.req.getSearcher(), sortFields, ss.getOffset() + ss.getCount());+ newShardDocQueue(rb.req.getSearcher(), sortFields, perQueryQueueSize);for (ShardResponse srsp : sreq.responses) {SolrDocumentList docs = null;NamedList<?> responseHeader;diff --git a/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java b/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.javaindex 39a7e69e23f..b4b72e0f2e5 100644--- a/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java+++ b/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java@@ -126,16 +126,17 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {@Testpublic void testSingleLexicalQuery() throws Exception {prepareIndexDocs();- QueryResponse rsp =- query(- CommonParams.JSON,- "{\"queries\":"- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:2^=10\"}}},"- + "\"limit\":5,"- + "\"fields\":[\"id\",\"score\",\"title\"],"- + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\"]}}",- CommonParams.QT,- "/search");+ String jsonQuery =+ """+ {+ "queries": {+ "lexical1": {"lucene": {"query": "id:2^=10"}}+ },+ "limit": 5,+ "fields": ["id", "score", "title"],+ "params": {"combiner": true, "combiner.query": ["lexical1"]}+ }""";+ QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search");assertEquals(1, rsp.getResults().size());assertFieldValues(rsp.getResults(), id, "2");}@@ -149,12 +150,16 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {public void testMultipleLexicalQuery() throws Exception {prepareIndexDocs();String jsonQuery =- "{\"queries\":"- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}},"- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}},"- + "\"limit\":5,"- + "\"fields\":[\"id\",\"score\",\"title\"],"- + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}";+ """+ {+ "queries": {+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)"}},+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)"}}+ },+ "limit": 5,+ "fields": ["id", "score", "title"],+ "params": {"combiner": true, "combiner.query": ["lexical1", "lexical2"]}+ }""";QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search");assertEquals(5, rsp.getResults().size());assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6", "3");@@ -169,12 +174,17 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {public void testMultipleQueryWithSort() throws Exception {prepareIndexDocs();String jsonQuery =- "{\"queries\":"- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}},"- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}},"- + "\"limit\":5,\"sort\":\"mod3_idv desc, score desc\""- + "\"fields\":[\"id\",\"score\",\"title\"],"- + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}";+ """+ {+ "queries": {+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)"}},+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)"}}+ },+ "limit": 5,+ "sort": "mod3_idv desc, score desc",+ "fields": ["id", "score", "title"],+ "params": {"combiner": true, "combiner.query": ["lexical1", "lexical2"]}+ }""";QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search");assertEquals(5, rsp.getResults().size());assertFieldValues(rsp.getResults(), id, "5", "2", "8", "7", "10");@@ -188,40 +198,60 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {@Testpublic void testHybridQueryWithPagination() throws Exception {prepareIndexDocs();- QueryResponse rsp =- query(- CommonParams.JSON,- "{\"queries\":"- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}},"- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}},"- + "\"fields\":[\"id\",\"score\",\"title\"],"- + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}",- CommonParams.QT,- "/search");+ String jsonQueryAll =+ """+ {+ "queries": {+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)"}},+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)"}}+ },+ "fields": ["id", "score", "title"],+ "params": {"combiner": true, "combiner.query": ["lexical1", "lexical2"]}+ }""";+ QueryResponse rsp = query(CommonParams.JSON, jsonQueryAll, CommonParams.QT, "/search");+ // ideal orderingassertFieldValues(rsp.getResults(), id, "5", "7", "2", "6", "3", "10", "8");- rsp =- query(- CommonParams.JSON,- "{\"queries\":"- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}},"- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}},"- + "\"limit\":4,"- + "\"fields\":[\"id\",\"score\",\"title\"],"- + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}",- CommonParams.QT,- "/search");- assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6");- rsp =- query(- CommonParams.JSON,- "{\"queries\":"- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}},"- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}},"- + "\"limit\":4,\"offset\":3,"- + "\"fields\":[\"id\",\"score\",\"title\"],"- + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}",- CommonParams.QT,- "/search");+ String jsonQueryLimit2 =+ """+ {+ "queries": {+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)"}},+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)"}}+ },+ "limit": 2,+ "fields": ["id", "score", "title"],+ "params": {"combiner": true, "combiner.query": ["lexical1", "lexical2"]}+ }""";+ rsp = query(CommonParams.JSON, jsonQueryLimit2, CommonParams.QT, "/search");+ // assert proper ordering due to presence of shards.rows+ assertFieldValues(rsp.getResults(), id, "7", "2");+ String jsonQueryWithShardRows =+ """+ {+ "queries": {+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)"}},+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)"}}+ },+ "limit": 2,+ "fields": ["id", "score", "title"],+ "params": {"combiner": true, "combiner.query": ["lexical1", "lexical2"], "shards.rows": 10}+ }""";+ rsp = query(CommonParams.JSON, jsonQueryWithShardRows, CommonParams.QT, "/search");+ assertFieldValues(rsp.getResults(), id, "5", "7");+ // assert improper ordering due to lack of shards.rows+ String jsonQueryPage =+ """+ {+ "queries": {+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)"}},+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)"}}+ },+ "limit": 4,+ "offset": 3,+ "fields": ["id", "score", "title"],+ "params": {"combiner": true, "combiner.query": ["lexical1", "lexical2"]}+ }""";+ rsp = query(CommonParams.JSON, jsonQueryPage, CommonParams.QT, "/search");assertEquals(4, rsp.getResults().size());assertFieldValues(rsp.getResults(), id, "6", "3", "10", "8");}@@ -235,12 +265,22 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {public void testQueryWithFaceting() throws Exception {prepareIndexDocs();String jsonQuery =- "{\"queries\":"- + "{\"lexical\":{\"lucene\":{\"query\":\"id:(2^=2 OR 3^=1 OR 6^=2 OR 5^=1)\"}}},"- + "\"limit\":3,\"offset\":1"- + "\"fields\":[\"id\",\"score\",\"title\"],"- + "\"params\":{\"combiner\":true,\"facet\":true,\"facet.field\":\"mod3_idv\",\"facet.mincount\":1,"- + "\"combiner.query\":[\"lexical\"]}}";+ """+ {+ "queries": {+ "lexical": {"lucene": {"query": "id:(2^=2 OR 3^=1 OR 6^=2 OR 5^=1)"}}+ },+ "limit": 3,+ "offset": 1,+ "fields": ["id", "score", "title"],+ "params": {+ "combiner": true,+ "facet": true,+ "facet.field": "mod3_idv",+ "facet.mincount": 1,+ "combiner.query": ["lexical"]+ }+ }""";QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search");assertEquals(3, rsp.getResults().size());assertEquals(4, rsp.getResults().getNumFound());@@ -256,14 +296,24 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {public void testQueriesWithFacetAndHighlights() throws Exception {prepareIndexDocs();String jsonQuery =- "{\"queries\":"- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}},"- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}},"- + "\"limit\":4,"- + "\"fields\":[\"id\",\"score\",\"title\"],"- + "\"params\":{\"combiner\":true,\"facet\":true,\"facet.field\":\"mod3_idv\","- + "\"combiner.query\":[\"lexical1\",\"lexical2\"], \"hl\": true,"- + "\"hl.fl\": \"title\",\"hl.q\":\"test doc\"}}";+ """+ {+ "queries": {+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)"}},+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)"}}+ },+ "limit": 4,+ "fields": ["id", "score", "title"],+ "params": {+ "combiner": true,+ "facet": true,+ "facet.field": "mod3_idv",+ "combiner.query": ["lexical1", "lexical2"],+ "hl": true,+ "hl.fl": "title",+ "hl.q": "test doc"+ }+ }""";QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search");assertEquals(4, rsp.getResults().size());assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6");diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.javaindex 17423bba672..50196cae1bc 100644--- a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java+++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java@@ -211,20 +211,36 @@ public class DistributedCombinedQueryComponentTest extends BaseDistributedSearch"params": {"combiner": true, "combiner.query": ["lexical1", "lexical2"]}}""";QueryResponse rsp = query(CommonParams.JSON, jsonQueryAll, CommonParams.QT, "/search");+ // ideal orderingassertFieldValues(rsp.getResults(), id, "5", "7", "2", "6", "3", "10", "4");- String jsonQueryLimit4 =+ String jsonQueryLimit1 ="""{"queries": {"lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)"}},"lexical2": {"lucene": {"query": "id:(4^=1 OR 5^=3 OR 7^=4 OR 10^=2)"}}},- "limit": 4,+ "limit": 1,"fields": ["id", "score", "title"],"params": {"combiner": true, "combiner.query": ["lexical1", "lexical2"]}}""";- rsp = query(CommonParams.JSON, jsonQueryLimit4, CommonParams.QT, "/search");- assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6");+ rsp = query(CommonParams.JSON, jsonQueryLimit1, CommonParams.QT, "/search");+ // assert proper ordering due to presence of shards.rows+ assertFieldValues(rsp.getResults(), id, "7");+ String jsonQueryWithShardRows =+ """+ {+ "queries": {+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)"}},+ "lexical2": {"lucene": {"query": "id:(4^=1 OR 5^=3 OR 7^=4 OR 10^=2)"}}+ },+ "limit": 1,+ "fields": ["id", "score", "title"],+ "params": {"combiner": true, "combiner.query": ["lexical1", "lexical2"], "shards.rows": 10}+ }""";+ rsp = query(CommonParams.JSON, jsonQueryWithShardRows, CommonParams.QT, "/search");+ assertFieldValues(rsp.getResults(), id, "5");+ // assert improper ordering due to lack of shards.rowsString jsonQueryPage ="""{@@ -322,6 +338,7 @@ public class DistributedCombinedQueryComponentTest extends BaseDistributedSearch* @throws Exception if any unexpected error occurs during the test execution.*/@Test+ @ShardsFixed(num = 2)public void testElevatedQueriesWithFacetAndHighlights() throws Exception {prepareIndexDocs();String jsonQuery =diff --git a/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc b/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adocindex de62c7a6d1b..61b7aed0b82 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc@@ -76,6 +76,11 @@ The query structure is similar to JSON Query DSL except for how multiple queries`combiner.rrf.k` | Default: `60`::The k parameter in the RRF algorithm.+[NOTE]+====+Consider setting `shards.rows` to at least your deepest expected `start + rows` to enlarge the per-shard candidate pool used for fusion, giving more consistent ordering across pages.+====+=== ExampleBelow is an example with sample JSON query payload: