Apache Solr open-source search software
SOLR-8291, SOLR-13217: Skip null per-leaf export bitsets (#4953)
Nick56ec140e3636
diff --git a/changelog/unreleased/SOLR-8291.yml b/changelog/unreleased/SOLR-8291.ymlnew file mode 100644index 00000000000..58cd1447f0b--- /dev/null+++ b/changelog/unreleased/SOLR-8291.yml@@ -0,0 +1,9 @@+title: Skip null per-leaf bitsets in /export instead of throwing NPE.+type: fixed+authors:+ - name: Nick Shanin+links:+ - name: SOLR-8291+ url: https://issues.apache.org/jira/browse/SOLR-8291+ - name: SOLR-13217+ url: https://issues.apache.org/jira/browse/SOLR-13217diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.javaindex 2b9731152fe..7b82ad28b14 100644--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java@@ -857,6 +857,11 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {private void topDocs() throws IOException {try {+ // null bitset = leaf was never collected+ if (bits == null) {+ index = -1;+ return;+ }queue.reset();SortDoc top = queue.top();this.sortDoc.setNextReader(context);diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-export-usefilter.xml b/solr/core/src/test-files/solr/collection1/conf/schema-export-usefilter.xmlnew file mode 100644index 00000000000..e49148e503e--- /dev/null+++ b/solr/core/src/test-files/solr/collection1/conf/schema-export-usefilter.xml@@ -0,0 +1,22 @@+<?xml version="1.0" encoding="UTF-8" ?>+<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->+<schema name="export-usefilter" version="1.7">+ <fieldType name="string" class="solr.StrField" />+ <field name="id" type="string" indexed="true" stored="true" docValues="true" required="true" />+ <uniqueKey>id</uniqueKey>+</schema>diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-export-usefilter.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-export-usefilter.xmlnew file mode 100644index 00000000000..e29ec1b0640--- /dev/null+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-export-usefilter.xml@@ -0,0 +1,43 @@+<?xml version="1.0" ?>++<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<!-- Minimal export config that forces the useFilterForSortedQuery / sortDocSet+ path. solrconfig-sortingresponse.xml has neither the flag nor a filterCache,+ so it cannot reproduce SOLR-8291. -->+<config>+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>+ <dataDir>${solr.data.dir:}</dataDir>+ <indexConfig>+ <mergePolicyFactory class="${solr.tests.mergePolicyFactory:org.apache.solr.index.LogDocMergePolicyFactory}" />+ </indexConfig>+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>+ <schemaFactory class="ClassicIndexSchemaFactory"/>+ <requestHandler name="/select" class="solr.SearchHandler" />++ <query>+ <enableLazyFieldLoading>true</enableLazyFieldLoading>+ <queryResultWindowSize>20</queryResultWindowSize>+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>+ <filterCache size="50" initialSize="50" autowarmCount="0"/>+ <useFilterForSortedQuery>true</useFilterForSortedQuery>+ <documentCache size="512"+ initialSize="512"+ autowarmCount="0"/>+ </query>+</config>diff --git a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.javaindex 5954a72a690..61384d50553 100644--- a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java+++ b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java@@ -1754,4 +1754,47 @@ public class TestExportWriter extends SolrTestCaseJ4 {"Error message should mention DocValues requirement",ex.getMessage().contains("DocValues"));}++ /**+ * SOLR-13217: collapse only replays group heads, so trailing leaves with no heads leave a null+ * export bitset. {@code /export} must skip those leaves instead of NPEing in {@code+ * BitSetIterator}.+ */+ @Test+ public void testCollapseWithTrailingEmptyLeaf() throws Exception {+ assertU(adoc("id", "1", "group_s_dv", "A", "intdv", "10"));+ assertU(commit());+ assertU(adoc("id", "2", "group_s_dv", "A", "intdv", "1"));+ assertU(commit());++ assertJQ(+ req(+ "q",+ "*:*",+ "fq",+ "{!collapse field=group_s_dv sort='intdv desc'}",+ "fl",+ "id",+ "sort",+ "id asc"),+ "/response/numFound==1",+ "/response/docs/[0]/id=='1'");++ String resp =+ h.query(+ req(+ "q",+ "*:*",+ "qt",+ "/export",+ "fq",+ "{!collapse field=group_s_dv sort='intdv desc'}",+ "fl",+ "id,group_s_dv,intdv",+ "sort",+ "id asc"));+ assertJsonEquals(+ resp,+ "{\"responseHeader\":{\"status\":0},\"response\":{\"numFound\":1,\"docs\":[{\"id\":\"1\",\"group_s_dv\":\"A\",\"intdv\":10}]}}");+ }}diff --git a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriterUseFilterForSortedQuery.java b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriterUseFilterForSortedQuery.javanew file mode 100644index 00000000000..e1dc726d956--- /dev/null+++ b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriterUseFilterForSortedQuery.java@@ -0,0 +1,113 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.handler.export;++import org.apache.solr.SolrTestCase;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.client.solrj.response.json.CanonicalJsonResponseParser;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.index.LogDocMergePolicyFactory;+import org.apache.solr.util.EmbeddedSolrServerTestRule;+import org.apache.solr.util.ExternalPaths;+import org.junit.Before;+import org.junit.BeforeClass;+import org.junit.ClassRule;+import org.junit.Test;++/**+ * SOLR-8291: {@code useFilterForSortedQuery=true} sends {@code /export} through {@code sortDocSet},+ * which never calls {@code ExportCollector.getLeafCollector} for trailing unmatched leaves. The+ * writer must treat those null bitsets as empty leaves instead of constructing a {@code+ * BitSetIterator}.+ */+public class TestExportWriterUseFilterForSortedQuery extends SolrTestCase {++ @ClassRule+ public static final EmbeddedSolrServerTestRule solrTestRule = new EmbeddedSolrServerTestRule();++ @BeforeClass+ public static void beforeClass() throws Exception {+ System.setProperty("solr.tests.mergePolicyFactory", LogDocMergePolicyFactory.class.getName());+ solrTestRule.startSolr();+ solrTestRule+ .newCollection()+ .withConfigSet(+ ExternalPaths.SOURCE_HOME.resolve("core/src/test-files/solr/collection1/conf"))+ .withConfigFile("solrconfig-export-usefilter.xml")+ .withSchemaFile("schema-export-usefilter.xml")+ .create();+ }++ @Before+ public void clearIndex() throws Exception {+ SolrClient client = solrTestRule.getSolrClient();+ client.deleteByQuery("*:*");+ client.commit();+ }++ @Test+ public void testZeroHitsDoesNotNpe() throws Exception {+ SolrClient client = solrTestRule.getSolrClient();+ client.add(doc("1"));+ client.commit();++ QueryResponse response = query(client, "id:does-not-exist");+ assertEquals(0, response.getResults().getNumFound());+ }++ @Test+ public void testHitsOnlyInEarlierSegment() throws Exception {+ SolrClient client = solrTestRule.getSolrClient();+ client.add(doc("1"));+ client.commit();+ client.add(doc("2"));+ client.commit();++ QueryResponse response = query(client, "id:1");+ assertEquals(1, response.getResults().getNumFound());+ assertEquals("1", response.getResults().get(0).getFieldValue("id"));++ // Null must mean "this leaf is empty", not "skip the rest of the index".+ response = query(client, "*:*");+ assertEquals(2, response.getResults().getNumFound());+ assertEquals("1", response.getResults().get(0).getFieldValue("id"));+ assertEquals("2", response.getResults().get(1).getFieldValue("id"));+ }++ private static SolrQuery exportQuery(String queryString) {+ SolrQuery query = new SolrQuery(queryString);+ query.setRequestHandler("/export");+ query.setFields("id");+ query.setSort("id", SolrQuery.ORDER.asc);+ return query;+ }++ private static QueryResponse query(SolrClient client, String queryString) throws Exception {+ QueryRequest request = new QueryRequest(exportQuery(queryString));+ request.setResponseParser(new CanonicalJsonResponseParser());+ return request.process(client);+ }++ private static SolrInputDocument doc(String id) {+ SolrInputDocument doc = new SolrInputDocument();+ doc.addField("id", id);+ return doc;+ }+}