Apache Solr open-source search software
SOLR-18195: CombinedQueryComponent supports Collapse now (#4277)
Sonu Sharmaef75ab82a5a8
diff --git a/changelog/unreleased/SOLR-18195-collapse-results-combined-query.yml b/changelog/unreleased/SOLR-18195-collapse-results-combined-query.ymlnew file mode 100644index 00000000000..5ddc729b3dc--- /dev/null+++ b/changelog/unreleased/SOLR-18195-collapse-results-combined-query.yml@@ -0,0 +1,9 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: Support for using {!collapse} with CombinedQueryComponent (RRF)+type: added+authors:+ - name: Sonu Sharma+ nick: ercsonusharma+links:+ - name: SOLR-18195+ url: https://issues.apache.org/jira/browse/SOLR-18195diff --git a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.javaindex 9831281d2b2..8b097bc304d 100644--- a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java+++ b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java@@ -31,6 +31,7 @@ import java.util.Objects;import java.util.Set;import java.util.stream.Collectors;import org.apache.lucene.search.Explanation;+import org.apache.lucene.search.Query;import org.apache.lucene.search.Sort;import org.apache.lucene.search.SortField;import org.apache.solr.client.solrj.SolrServerException;@@ -42,6 +43,7 @@ import org.apache.solr.common.params.CursorMarkParams;import org.apache.solr.common.params.GroupParams;import org.apache.solr.common.params.ShardParams;import org.apache.solr.common.params.SolrParams;+import org.apache.solr.common.util.CollectionUtil;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.common.util.StrUtils;@@ -53,8 +55,10 @@ import org.apache.solr.response.ResultContext;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.schema.IndexSchema;import org.apache.solr.schema.SchemaField;+import org.apache.solr.search.CollapsingQParserPlugin;import org.apache.solr.search.DocListAndSet;import org.apache.solr.search.QueryResult;+import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.search.SolrReturnFields;import org.apache.solr.search.SortSpec;import org.apache.solr.util.SolrResponseUtil;@@ -236,7 +240,10 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAwboolean partialResults,boolean segmentTerminatedEarly,Boolean setMaxHitsTerminatedEarly) {- QueryResult combinedQueryResult = QueryAndResponseCombiner.simpleCombine(queryResults);+ SolrIndexSearcher searcher = crb.req.getSearcher();+ List<Query> collapseFilters = getCollapseFilters(crb.getFilters());+ QueryResult combinedQueryResult =+ QueryAndResponseCombiner.simpleCombine(queryResults, collapseFilters, searcher);combinedQueryResult.setPartialResults(partialResults);combinedQueryResult.setSegmentTerminatedEarly(segmentTerminatedEarly);combinedQueryResult.setMaxHitsTerminatedEarly(setMaxHitsTerminatedEarly);@@ -258,6 +265,21 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw}}+ /** Extracts the list of CollapsingPostFilter query from the filter list, if present. */+ private static List<Query> getCollapseFilters(List<Query> filters) {+ if (CollectionUtil.isNotEmpty(filters)) {+ return filters.stream()+ .filter(q -> q instanceof CollapsingQParserPlugin.CollapsingPostFilter)+ .toList();+ }+ return List.of();+ }++ /**+ * Each shard response contains both "response" and "response_per_query". Only "response" is+ * deduplicated across sub-queries, so any processing of "response_per_query" must exclude docs+ * not present in "response" to avoid reintroducing docs that were eliminated while deduplication.+ */@Overrideprotected void mergeIds(ResponseBuilder rb, ShardRequest sreq) {SortSpec ss = rb.getSortSpec();@@ -296,6 +318,10 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAwlong approximateTotalHits = 0;Map<String, List<ShardDoc>> shardDocMap = new HashMap<>();String[] queriesToCombineKeys = rb.req.getParams().getParams(CombinerParams.COMBINER_QUERY);+ // Build per-shard set of doc IDs from the shard's combined (deduplicated) response.+ // Used to filter per-query docs so that RRF doesn't reintroduce docs+ // excluded by collapse at the shard level.+ Map<String, Set<Object>> combinedDocIdsPerShard = HashMap.newHashMap(sreq.responses.size());// TODO: to be parallelized outer loopfor (int queryIndex = 0; queryIndex < queriesToCombineKeys.length; queryIndex++) {int failedShardCount = 0;@@ -377,9 +403,15 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw: new NamedList<>();// go through every doc in this response, construct a ShardDoc, and// put it in the uniqueDoc to dedup+ Set<Object> thisShardCombinedIds =+ combinedDocIdsPerShard.computeIfAbsent(+ srsp.getShard(), shard -> extractIdsFromCombinedResponse(rb, srsp, uniqueKeyField));for (int i = 0; i < docs.size(); i++) {SolrDocument doc = docs.get(i);Object id = doc.getFieldValue(uniqueKeyField.getName());+ if (!thisShardCombinedIds.contains(id)) {+ continue;+ }ShardDoc shardDoc = new ShardDoc();shardDoc.id = id;shardDoc.orderInShard = i;@@ -610,4 +642,19 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAwfor (int i = 0; i < resultSize; i++) responseDocs.add(null);return resultIds;}++ /**+ * Extracts the set of doc IDs from the shard's combined response (produced by simpleCombine).+ * Returns an empty set if the combined response is not available.+ */+ private static Set<Object> extractIdsFromCombinedResponse(+ ResponseBuilder rb, ShardResponse srsp, SchemaField uniqueKeyField) {+ Object response = SolrResponseUtil.getSubsectionFromShardResponse(rb, srsp, "response", false);+ if (response instanceof SolrDocumentList docList) {+ return docList.stream()+ .map(doc -> doc.getFieldValue(uniqueKeyField.getName()))+ .collect(Collectors.toSet());+ }+ return Set.of();+ }}diff --git a/solr/core/src/java/org/apache/solr/handler/component/combine/QueryAndResponseCombiner.java b/solr/core/src/java/org/apache/solr/handler/component/combine/QueryAndResponseCombiner.javaindex c7d357fb12b..0c023231939 100644--- a/solr/core/src/java/org/apache/solr/handler/component/combine/QueryAndResponseCombiner.java+++ b/solr/core/src/java/org/apache/solr/handler/component/combine/QueryAndResponseCombiner.java@@ -16,19 +16,33 @@*/package org.apache.solr.handler.component.combine;+import java.io.IOException;import java.util.HashMap;+import java.util.HashSet;import java.util.List;import java.util.Map;+import java.util.Set;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.internal.hppc.IntDoubleHashMap;+import org.apache.lucene.queries.function.FunctionScoreQuery;+import org.apache.lucene.search.DoubleValues;+import org.apache.lucene.search.DoubleValuesSource;import org.apache.lucene.search.Explanation;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.Query;import org.apache.lucene.search.TotalHits;import org.apache.solr.common.SolrException;import org.apache.solr.common.params.SolrParams;+import org.apache.solr.common.util.CollectionUtil;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.handler.component.ShardDoc;import org.apache.solr.search.DocIterator;import org.apache.solr.search.DocSet;import org.apache.solr.search.DocSlice;+import org.apache.solr.search.QueryCommand;import org.apache.solr.search.QueryResult;+import org.apache.solr.search.SolrIndexSearcher;+import org.apache.solr.search.SortedIntDocSet;import org.apache.solr.util.plugin.NamedListInitializedPlugin;/**@@ -49,12 +63,19 @@ public abstract class QueryAndResponseCombiner implements NamedListInitializedPlMap<String, List<ShardDoc>> queriesDocMap, SolrParams solrParams);/**- * Simple combine query result list as a union.+ * Combine query result list as a union, optionally deduplicating by a collapse field. When a+ * collapse filter is provided, only one document per unique field value is kept (based on the+ * collapse sort/score selection). This ensures that collapse semantics are preserved across+ * combined queries.** @param queryResults the query results to be combined+ * @param collapseFilters the collapse post filters, or empty if no collapse dedup is needed+ * @param searcher the searcher to read field values from, required when collapseFilters is+ * non-empty* @return the combined query result*/- public static QueryResult simpleCombine(List<QueryResult> queryResults) {+ public static QueryResult simpleCombine(+ List<QueryResult> queryResults, List<Query> collapseFilters, SolrIndexSearcher searcher) {QueryResult combinedQueryResults = new QueryResult();DocSet combinedDocSet = null;Map<Integer, Float> uniqueDocIds = new HashMap<>();@@ -71,6 +92,19 @@ public abstract class QueryAndResponseCombiner implements NamedListInitializedPlcombinedDocSet = combinedDocSet.union(queryResult.getDocSet());}}++ // If collapse fields are specified, deduplicate by field value across combined queries.+ // Each sub-query already collapsed individually, but different sub-queries may have+ // selected different group heads for the same field value.+ int removedByCollapse = 0;+ if (CollectionUtil.isNotEmpty(collapseFilters) && searcher != null && queryResults.size() > 1) {+ int preCollapseSize = uniqueDocIds.size();+ combinedDocSet =+ removeCollapsedDuplicatesViaSearcher(+ collapseFilters, searcher, uniqueDocIds, combinedDocSet);+ removedByCollapse = preCollapseSize - uniqueDocIds.size();+ }+int combinedResultsLength = uniqueDocIds.size();int[] combinedResultsDocIds = new int[combinedResultsLength];float[] combinedResultScores = new float[combinedResultsLength];@@ -87,7 +121,7 @@ public abstract class QueryAndResponseCombiner implements NamedListInitializedPlcombinedResultsLength,combinedResultsDocIds,combinedResultScores,- Math.max(combinedResultsLength, totalMatches),+ Math.max(combinedResultsLength, totalMatches - removedByCollapse),combinedResultScores.length > 0 ? combinedResultScores[0] : 0,TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO);combinedQueryResults.setDocList(combinedResultSlice);@@ -95,6 +129,56 @@ public abstract class QueryAndResponseCombiner implements NamedListInitializedPlreturn combinedQueryResults;}+ /**+ * Removes collapsed duplicates across combined sub-queries. Ensures that only one document per+ * collapse field value retained across the merged results. Entries removed by collapsing are also+ * removed from {@code uniqueDocIds} (mutated in place).+ *+ * @return the collapsed combined DocSet, or null if combinedDocSet was null+ */+ private static DocSet removeCollapsedDuplicatesViaSearcher(+ List<Query> collapseFilters,+ SolrIndexSearcher searcher,+ Map<Integer, Float> uniqueDocIds,+ DocSet combinedDocSet) {+ IntDoubleHashMap scoreMap = new IntDoubleHashMap(uniqueDocIds.size());+ uniqueDocIds.forEach((doc, score) -> scoreMap.put(doc, score.doubleValue()));+ Query baseQuery;+ boolean needDocSet;+ if (combinedDocSet != null) {+ baseQuery = combinedDocSet.makeQuery();+ needDocSet = true;+ } else {+ int[] queryDocIds =+ uniqueDocIds.keySet().stream().mapToInt(Integer::intValue).sorted().toArray();+ baseQuery = new SortedIntDocSet(queryDocIds).makeQuery();+ needDocSet = false;+ }+ Query scoredQuery =+ FunctionScoreQuery.boostByValue(baseQuery, new PrecomputedScoreValuesSource(scoreMap));++ try {+ QueryCommand cmd =+ new QueryCommand()+ .setQuery(scoredQuery)+ .setFilterList(collapseFilters)+ .setLen(uniqueDocIds.size())+ .setNeedDocSet(needDocSet);+ QueryResult result = searcher.search(cmd);++ Set<Integer> retainedDocIds = HashSet.newHashSet(result.getDocList().size());+ DocIterator iter = result.getDocList().iterator();+ while (iter.hasNext()) {+ retainedDocIds.add(iter.nextDoc());+ }++ uniqueDocIds.keySet().retainAll(retainedDocIds);+ return needDocSet ? result.getDocSet() : null;+ } catch (IOException e) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ }+ }+/*** Retrieves a list of explanations for the given queries and results.*@@ -127,4 +211,67 @@ public abstract class QueryAndResponseCombiner implements NamedListInitializedPlthrow new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown Combining algorithm: " + algorithm);}++ /**+ * A {@link DoubleValuesSource} backed by a global doc ID to score map. Returns pre-computed+ * scores for specific document IDs.+ */+ private static class PrecomputedScoreValuesSource extends DoubleValuesSource {++ private final IntDoubleHashMap scoreByDoc;++ PrecomputedScoreValuesSource(IntDoubleHashMap scoreByDoc) {+ this.scoreByDoc = scoreByDoc;+ }++ @Override+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues existing) {+ int base = ctx.docBase;+ return new DoubleValues() {+ private double currentScore;++ @Override+ public double doubleValue() {+ return currentScore;+ }++ @Override+ public boolean advanceExact(int doc) {+ int globalDoc = base + doc;+ currentScore = scoreByDoc.get(globalDoc);+ return true;+ }+ };+ }++ @Override+ public boolean needsScores() {+ return false;+ }++ @Override+ public DoubleValuesSource rewrite(IndexSearcher searcher) {+ return this;+ }++ @Override+ public boolean isCacheable(LeafReaderContext ctx) {+ return false;+ }++ @Override+ public boolean equals(Object o) {+ return o instanceof PrecomputedScoreValuesSource other && scoreByDoc.equals(other.scoreByDoc);+ }++ @Override+ public int hashCode() {+ return scoreByDoc.hashCode();+ }++ @Override+ public String toString() {+ return "PrecomputedScoreValuesSource(docs=" + scoreByDoc.size() + ")";+ }+ }}diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml b/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xmlindex 37bb2b98557..7716a8ee0f5 100644--- a/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml+++ b/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml@@ -504,7 +504,7 @@</analyzer></fieldType><copyField source="*_commongrams" dest="*_commongrams_stop" />-+<fieldType name="binary" class="solr.BinaryField" /><fieldType name="collation" class="solr.CollationField" language="en" /><fieldType name="icuCollation" class="solr.ICUCollationField" locale="en" />@@ -691,6 +691,7 @@<dynamicField name="*_s" type="string" indexed="true" stored="true"/>+ <dynamicField name="*_sdv" type="string" indexed="true" stored="true" docValues="true" multiValued="false"/><dynamicField name="*_s1" type="string" indexed="true" stored="true" multiValued="false"/><dynamicField name="*_s1_ns" type="string" indexed="true" stored="false" multiValued="false"/><dynamicField name="*_l" type="long" indexed="true" stored="true"/>@@ -857,14 +858,9 @@<copyField source="lowerfilt1" dest="lowerfilt1and2"/><copyField source="lowerfilt" dest="lowerfilt1and2"/>- <copyField source="*" dest="text"/>- <copyField source="id" dest="range_facet_l"/>- <copyField source="id" dest="id_i1"/><copyField source="range_facet_f" dest="range_facet_d"/><copyField source="range_facet_f1" dest="range_facet_f1_dv"/>- <copyField source="id" dest="range_facet_l_dv"/>- <copyField source="id" dest="range_facet_i_dv"/><copyField source="range_facet_f" dest="range_facet_f_dv"/><copyField source="range_facet_f" dest="range_facet_d_dv"/><copyField source="bday" dest="range_facet_dt_dv"/>diff --git a/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java b/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.javaindex 4fbbb7f17e5..448ced51254 100644--- a/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java+++ b/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java@@ -45,6 +45,8 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {super();sliceCount = 2;fixShardCount(2);+ schemaString = "schema-vector-catchall.xml";+ configString = "solrconfig-combined-query.xml";}@Override@@ -58,6 +60,28 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {}private synchronized void prepareIndexDocs() throws Exception {+ List<SolrInputDocument> docs = getSolrDocuments();+ del("*:*");+ for (SolrInputDocument doc : docs) {+ indexDoc(doc);+ }+ commit();+ }++ /** Index all docs to the same shard for co-location (limitation for collapse). */+ private synchronized void prepareIndexDocsColocated() throws Exception {+ del("*:*");+ List<SolrInputDocument> docs = getSolrDocuments();+ for (SolrInputDocument doc : docs) {+ doc.setField("id", doc.getFieldValue("mod3_idv") + "!" + doc.getField("id").getValue());+ }+ for (SolrInputDocument doc : docs) {+ indexDoc(doc);+ }+ commit();+ }++ private static synchronized List<SolrInputDocument> getSolrDocuments() {List<SolrInputDocument> docs = new ArrayList<>();for (int i = 1; i <= NUM_DOCS; i++) {SolrInputDocument doc = new SolrInputDocument();@@ -87,11 +111,7 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {docs.get(8).addField(vectorField, Arrays.asList(200f, 50f, 100f, 25f));// cosine distance vector1= 0.997docs.get(9).addField(vectorField, Arrays.asList(1.8f, 2.5f, 3.7f, 4.9f));- del("*:*");- for (SolrInputDocument doc : docs) {- indexDoc(doc);- }- commit();+ return docs;}@Test@@ -249,6 +269,69 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {rsp.getHighlighting().get("5").get("title").getFirst());}+ /**+ * Tests the combined query feature with faceting, highlighting and collapse.+ *+ * @throws Exception if any unexpected error occurs during the test execution.+ */+ @Test+ public void testQueriesWithFacetAndHighlightsCollapse() throws Exception {+ // Re-index all docs to the first shard for co-location (required for collapse).+ prepareIndexDocsColocated();+ String jsonQuery =+ """+ {+ "queries": {+ "lexical1": {+ "lucene": {+ "query": "id:(2!2^2 OR 0!3^1 OR 0!6^2 OR 2!5^1)"+ }+ },+ "lexical2": {+ "lucene": {+ "query": "id:(2!8^1 OR 2!5^2 OR 1!7^3 OR 1!10^2)"+ }+ }+ },+ "limit": 1,+ "fields": [+ "id",+ "score",+ "title"+ ],+ "params": {+ "combiner": true,+ "facet": true,+ "facet.field": "id",+ "fq": [+ "{!collapse field=mod3_idv sort='id asc'}"+ ],+ "expand": true,+ "expand.q": "*:*",+ "combiner.query": [+ "lexical1",+ "lexical2"+ ],+ "hl": true,+ "hl.fl": "title",+ "hl.q": "test doc"+ }+ }""";+ handle.put("expanded", UNORDERED);+ QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search");+ assertEquals(1, rsp.getResults().size());+ assertFieldValues(rsp.getResults(), id, "2!2");+ assertEquals("id", rsp.getFacetFields().getFirst().getName());+ assertEquals(+ "[0!3 (1), 1!10 (1), 2!2 (1), 0!6 (0), 0!9 (0), 1!1 (0), 1!4 (0), 1!7 (0), 2!5 (0), 2!8 (0)]",+ rsp.getFacetFields().getFirst().getValues().toString());+ assertEquals(1, rsp.getHighlighting().size());+ assertEquals(+ "title <em>test</em> for <em>doc</em> 2",+ rsp.getHighlighting().get("2!2").get("title").getFirst());+ assertEquals(1, rsp.getExpandedResults().size());+ }+/** To test that we can force distrib */public static class ForcedDistribSearchHandler extends SearchHandler {@Overridediff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.javaindex 46441382c30..bb9105cad30 100644--- a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java+++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java@@ -290,4 +290,143 @@ public class DistributedCombinedQueryComponentTest extends BaseDistributedSearchQueryResponse rsp = query("qt", "/forcedDistribTest", "q", "*:*", "rows", "0");// ForcedDistribSearchHandler would trigger a failure if this didn't work}++ /**+ * Reproduces duplicate documents in combined query results when using collapse. Two different+ * sub-queries may select different group heads for the same collapse field value, and+ * simpleCombine() merges by Lucene doc ID — not by collapse field value — so both survive.+ *+ * <p>Data setup (mod3_idv is the collapse field, values cycle 1,2,0,1,2,0,...):+ *+ * <pre>+ * id=1 mod3_idv=1 text="alpha bravo" title="alpha bravo"+ * id=2 mod3_idv=2 text="alpha charlie" title="alpha charlie"+ * id=3 mod3_idv=0 text="bravo delta" title="bravo delta"+ * id=4 mod3_idv=1 text="charlie delta" title="charlie delta"+ * id=5 mod3_idv=2 text="alpha delta" title="alpha delta"+ * id=6 mod3_idv=0 text="bravo charlie" title="bravo charlie"+ * </pre>+ *+ * <p>Query 1: "alpha bravo" → matches docs 1,2,3,5,6 with varying scores. After collapse on+ * mod3_idv, picks one group head per value (0,1,2).+ *+ * <p>Query 2: "charlie delta" → matches docs 2,3,4,5,6 with varying scores. After collapse on+ * mod3_idv, picks one group head per value (0,1,2).+ *+ * <p>Because the queries score differently, they may pick DIFFERENT group heads for the same+ * mod3_idv value. After simpleCombine() merges by doc ID, both heads appear → duplicates on the+ * collapse field.+ */+ @Test+ public void testCollapseWithCombinedQueryProducesDuplicates() throws Exception {+ del("*:*");++ // Index 6 docs where mod3_idv groups docs: {3,6}→0, {1,4}→1, {2,5}→2+ // All docs indexed to the same shard (first client) to ensure co-location,+ // which is a requirement for collapse in SolrCloud.+ List<SolrInputDocument> docs = new ArrayList<>();+ String[][] data = {+ {"1", "alpha bravo", "alpha bravo"},+ {"2", "alpha charlie", "alpha charlie"},+ {"3", "bravo delta", "bravo delta"},+ {"4", "charlie delta", "charlie delta"},+ {"5", "alpha delta", "alpha delta"},+ {"6", "bravo charlie", "bravo charlie"},+ };+ for (String[] row : data) {+ SolrInputDocument doc = new SolrInputDocument();+ doc.addField("id", row[0]);+ doc.addField("text", row[1]);+ doc.addField("title", row[2]);+ int idVal = Integer.parseInt(row[0]);+ doc.addField("mod3_idv", idVal % 3);+ doc.addField("mod3_sdv", String.valueOf(idVal % 3));+ docs.add(doc);+ }+ // Index all docs to the first shard (co-location for collapse).+ // indexDoc(client, doc) also indexes to controlClient automatically.+ for (SolrInputDocument doc : docs) {+ indexDoc(clients.getFirst(), doc);+ }+ commit();++ // Two queries that score docs differently, both collapsed on mod3_idv.+ // Query 1 "alpha bravo" favours doc 1 (mod3_idv=1) and doc 3 (mod3_idv=0).+ // Query 2 "charlie delta" favours doc 4 (mod3_idv=1) and doc 3 or 6 (mod3_idv=0).+ // After collapse, each query returns 3 docs (one per mod3_idv value 0,1,2).+ // But the group heads for mod3_idv=1 differ: query1 picks id=1, query2 picks id=4.+ // simpleCombine() merges by doc ID, so both id=1 and id=4 survive → duplicate on mod3_idv=1.+ // q1 -> 1,2,5,3,6 q2 -> 2,4,6,3,5+ // q1 -> 1,2,2,0,0 q2 -> 2,1,0,0,2+ // q1 -> 1 for sure, 2,3 => 4 for sure, 3 and 2+ String jsonQuery =+ """+ {+ "queries": {+ "q1": {+ "lucene": {+ "query": "text:alpha OR text:bravo"+ }+ },+ "q2": {+ "lucene": {+ "query": "text:charlie OR text:delta"+ }+ }+ },+ "limit": 10,+ "fields": [+ "id",+ "score",+ "mod3_idv"+ ],+ "params": {+ "combiner": true,+ "combiner.query": [+ "q1",+ "q2"+ ],+ "fq": [+ "{!collapse field=mod3_idv sort='mod3_sdv asc, id desc'}"+ ],+ "facet": true,+ "facet.field": "id",+ "hl": true,+ "hl.fl": "title",+ "hl.q": "alpha delta",+ "expand": true,+ "expand.q": "text:alpha OR text:bravo OR text:charlie OR text:delta"+ }+ }""";++ handle.put("expanded", UNORDERED);+ QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search");++ // Collect mod3_idv values from the result to check for duplicates+ List<Integer> collapseValues = new ArrayList<>();+ for (int i = 0; i < rsp.getResults().size(); i++) {+ Object val = rsp.getResults().get(i).getFieldValue("mod3_idv");+ collapseValues.add(((Number) val).intValue());+ }++ // Without the fix: more than 3 results because different group heads survive for the same+ // mod3_idv value. With the fix: exactly 3 results (one per mod3_idv value 0, 1, 2).+ long uniqueCollapseValues = collapseValues.stream().distinct().count();+ assertEquals(+ "Expected no duplicate collapse field values in combined results, "+ + "but got collapseValues="+ + collapseValues,+ uniqueCollapseValues,+ collapseValues.size());+ assertEquals("Expected exactly 3 groups (mod3_idv values 0, 1, 2)", 3, collapseValues.size());+ assertEquals("id", rsp.getFacetFields().getFirst().getName());+ assertEquals(+ "[4 (1), 5 (1), 6 (1), 1 (0), 2 (0), 3 (0)]",+ rsp.getFacetFields().getFirst().getValues().toString());+ assertEquals(3, rsp.getHighlighting().size());+ assertEquals("charlie <em>delta</em>", rsp.getHighlighting().get("4").get("title").getFirst());+ assertEquals(+ "<em>alpha</em> <em>delta</em>", rsp.getHighlighting().get("5").get("title").getFirst());+ assertEquals(3, rsp.getExpandedResults().size());+ }}diff --git a/solr/core/src/test/org/apache/solr/handler/component/combine/QueryAndResponseCombinerTest.java b/solr/core/src/test/org/apache/solr/handler/component/combine/QueryAndResponseCombinerTest.javaindex aece94c5ea3..a03aa7219e5 100644--- a/solr/core/src/test/org/apache/solr/handler/component/combine/QueryAndResponseCombinerTest.java+++ b/solr/core/src/test/org/apache/solr/handler/component/combine/QueryAndResponseCombinerTest.java@@ -17,15 +17,29 @@package org.apache.solr.handler.component.combine;import java.util.List;+import org.apache.lucene.search.Query;import org.apache.lucene.search.TotalHits;import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.request.SolrRequestInfo;+import org.apache.solr.response.SolrQueryResponse;+import org.apache.solr.search.CollapsingQParserPlugin.CollapsingPostFilter;import org.apache.solr.search.DocSlice;+import org.apache.solr.search.QParser;import org.apache.solr.search.QueryResult;+import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.search.SortedIntDocSet;+import org.apache.solr.util.RefCounted;+import org.junit.BeforeClass;import org.junit.Test;public class QueryAndResponseCombinerTest extends SolrTestCaseJ4 {+ @BeforeClass+ public static void beforeClass() throws Exception {+ initCore("solrconfig.xml", "schema11.xml");+ }+public static List<QueryResult> getQueryResults() {QueryResult r1 = new QueryResult();r1.setDocList(@@ -52,10 +66,100 @@ public class QueryAndResponseCombinerTest extends SolrTestCaseJ4 {return List.of(r1, r2);}+ /** Parses a collapse fq string and returns the CollapsingPostFilter. */+ private static CollapsingPostFilter parseCollapseFilter(String collapseFq) throws Exception {+ SolrQueryRequest solrReq = req();+ SolrQueryResponse rsp = new SolrQueryResponse();+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, rsp));+ try {+ Query q = QParser.getParser(collapseFq, solrReq).getQuery();+ assertTrue(+ "Expected CollapsingPostFilter but got " + q.getClass(),+ q instanceof CollapsingPostFilter);+ return (CollapsingPostFilter) q;+ } finally {+ SolrRequestInfo.clearRequestInfo();+ solrReq.close();+ }+ }++ /** Tests that simpleCombine without using collapse. */@Testpublic void simpleCombine() {- QueryResult queryResult = QueryAndResponseCombiner.simpleCombine(getQueryResults());+ QueryResult queryResult = QueryAndResponseCombiner.simpleCombine(getQueryResults(), null, null);assertEquals(3, queryResult.getDocList().size());assertEquals(4, queryResult.getDocSet().size());}++ /**+ * Tests that simpleCombine deduplicates by collapse field value when a collapse field is+ * provided. Uses the same query results from getQueryResults() but indexes docs so that Lucene+ * doc IDs 0 and 1 share the same collapse field value — only the higher-scoring doc should+ * survive.+ */+ @Test+ public void simpleCombineWithCollapseField() throws Exception {+ // getQueryResults() returns:+ // r1: docs [1, 2] scores [0.67, 0.62]+ // r2: docs [0] scores [0.87]+ assertU(delQ("*:*"));+ assertU(adoc("id", "0", "group_ti_dv", "1"));+ assertU(adoc("id", "1", "group_ti_dv", "1"));+ assertU(adoc("id", "2", "group_ti_dv", "2"));+ assertU(commit());++ CollapsingPostFilter collapseFilter = parseCollapseFilter("{!collapse field=group_ti_dv}");++ RefCounted<SolrIndexSearcher> searcherRef = h.getCore().getSearcher();+ try {+ SolrIndexSearcher searcher = searcherRef.get();++ QueryResult combined =+ QueryAndResponseCombiner.simpleCombine(+ getQueryResults(), List.of(collapseFilter), searcher);++ // group=1: doc 0 (0.87) beats doc 1 (0.67) → keep doc 0+ // group=2: doc 2 (0.62) → kept+ assertEquals(+ "Expected 2 docs after collapse dedup (one per group value)",+ 2,+ combined.getDocList().size());+ assertEquals(0, combined.getDocList().iterator().nextDoc());+ } finally {+ searcherRef.decref();+ }+ }++ /**+ * Tests that docs with missing collapse field values are kept (null policy pass-through) and not+ * considered duplicates.+ */+ @Test+ public void simpleCombineWithCollapseFieldNullPolicy() throws Exception {+ assertU(delQ("*:*"));+ assertU(adoc("id", "0", "group_ti_dv", "1"));+ assertU(adoc("id", "1", "group_ti_dv", "1"));+ assertU(adoc("id", "2")); // no collapse field+ assertU(commit());++ CollapsingPostFilter collapseFilter =+ parseCollapseFilter("{!collapse field=group_ti_dv nullPolicy=collapse sort='score asc'}");++ RefCounted<SolrIndexSearcher> searcherRef = h.getCore().getSearcher();+ try {+ SolrIndexSearcher searcher = searcherRef.get();++ QueryResult combined =+ QueryAndResponseCombiner.simpleCombine(+ getQueryResults(), List.of(collapseFilter), searcher);++ // group=1: doc 0 (0.87) > doc 1 (0.67) → keep doc 1+ // doc 2 has no group value → kept (null policy)+ assertEquals(+ "Expected 2 docs: one group head + one null-field doc", 2, combined.getDocList().size());+ assertEquals(1, combined.getDocList().iterator().nextDoc());+ } finally {+ searcherRef.decref();+ }+ }}diff --git a/solr/core/src/test/org/apache/solr/handler/component/combine/ReciprocalRankFusionTest.java b/solr/core/src/test/org/apache/solr/handler/component/combine/ReciprocalRankFusionTest.javaindex 27ab8c5a312..ac416c49aab 100644--- a/solr/core/src/test/org/apache/solr/handler/component/combine/ReciprocalRankFusionTest.java+++ b/solr/core/src/test/org/apache/solr/handler/component/combine/ReciprocalRankFusionTest.java@@ -55,7 +55,7 @@ public class ReciprocalRankFusionTest extends SolrTestCaseJ4 {@Testpublic void testSimpleCombine() {List<QueryResult> rankedList = QueryAndResponseCombinerTest.getQueryResults();- QueryResult result = QueryAndResponseCombiner.simpleCombine(rankedList);+ QueryResult result = QueryAndResponseCombiner.simpleCombine(rankedList, null, null);assertEquals(3, result.getDocList().size());assertEquals(4, result.getDocSet().size());}