Apache Solr open-source search software
SOLR-17677: Ensure DBQ is safe before running (#3203)
Jason Gerlowski76e9b331a35a
diff --git a/solr/core/src/java/org/apache/solr/search/JoinQuery.java b/solr/core/src/java/org/apache/solr/search/JoinQuery.javaindex 3ec6b85b94c..ede3edaddc7 100644--- a/solr/core/src/java/org/apache/solr/search/JoinQuery.java+++ b/solr/core/src/java/org/apache/solr/search/JoinQuery.java@@ -29,6 +29,7 @@ import org.apache.lucene.index.MultiPostingsEnum;import org.apache.lucene.index.PostingsEnum;import org.apache.lucene.index.Terms;import org.apache.lucene.index.TermsEnum;+import org.apache.lucene.search.BooleanClause;import org.apache.lucene.search.ConstantScoreScorer;import org.apache.lucene.search.ConstantScoreWeight;import org.apache.lucene.search.DocIdSetIterator;@@ -54,7 +55,7 @@ import org.apache.solr.search.join.GraphPointsCollector;import org.apache.solr.util.RTimer;import org.apache.solr.util.RefCounted;-class JoinQuery extends Query {+class JoinQuery extends Query implements SolrSearcherRequirer {String fromField;String toField;// TODO: name is missleading here compared to JoinQParserPlugin usage - here it must be a core@@ -86,7 +87,10 @@ class JoinQuery extends Query {}@Override- public void visit(QueryVisitor visitor) {}+ public void visit(QueryVisitor visitor) {+ QueryVisitor sub = visitor.getSubVisitor(BooleanClause.Occur.MUST, this);+ q.visit(sub);+ }@Overridepublic Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)diff --git a/solr/core/src/java/org/apache/solr/search/SolrSearcherRequirementDetector.java b/solr/core/src/java/org/apache/solr/search/SolrSearcherRequirementDetector.javanew file mode 100644index 00000000000..9abc601fa6d--- /dev/null+++ b/solr/core/src/java/org/apache/solr/search/SolrSearcherRequirementDetector.java@@ -0,0 +1,65 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.search;++import org.apache.lucene.search.BooleanClause;+import org.apache.lucene.search.Query;+import org.apache.lucene.search.QueryVisitor;++/**+ * Detects whether a query can be run using a standard Lucene {@link+ * org.apache.lucene.search.IndexSearcher}+ *+ * <p>Some Solr {@link Query} implementations are written to assume access to a {@link+ * SolrIndexSearcher}. But these objects aren't always available: some code-paths (e.g. when+ * executing a "delete-by-query") execute the query using the standard {@link+ * org.apache.lucene.search.IndexSearcher} available in Lucene. This {@link QueryVisitor} allows+ * code to detect whether a given Query requires SolrIndexSearcher or not.+ *+ * <p>Instances should not be reused for multiple query-tree inspections.+ *+ * @see SolrSearcherRequirer+ * @lucene.experimental+ */+public class SolrSearcherRequirementDetector extends QueryVisitor {++ private boolean requiresSolrSearcher = false;++ @Override+ public QueryVisitor getSubVisitor(BooleanClause.Occur occur, Query parent) {+ // This method is primarily intended to swap out visitors when descending through the Query+ // tree, but it's also the only place to put visiting logic for non-leaf nodes, since the+ // QueryVisitor interface largely assumes that only leaf-nodes are worth visiting. See+ // LUCENE-????? for more details - this can be restructured if that ticket is ever addressed.+ if (parent instanceof SolrSearcherRequirer) {+ requiresSolrSearcher = true;+ }++ return this;+ }++ @Override+ public void visitLeaf(Query query) {+ if (query instanceof SolrSearcherRequirer) {+ requiresSolrSearcher = true;+ }+ }++ public boolean getRequiresSolrSearcher() {+ return requiresSolrSearcher;+ }+}diff --git a/solr/core/src/java/org/apache/solr/search/SolrSearcherRequirer.java b/solr/core/src/java/org/apache/solr/search/SolrSearcherRequirer.javanew file mode 100644index 00000000000..888060a4231--- /dev/null+++ b/solr/core/src/java/org/apache/solr/search/SolrSearcherRequirer.java@@ -0,0 +1,27 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.search;++import org.apache.lucene.search.IndexSearcher;++/**+ * Marker interface indicating that the tagged class assumes access to {@link SolrIndexSearcher}+ * functionality and cannot be used by a vanilla Lucene {@link IndexSearcher}+ *+ * @lucene.experimental+ */+public interface SolrSearcherRequirer {}diff --git a/solr/core/src/java/org/apache/solr/search/TopLevelJoinQuery.java b/solr/core/src/java/org/apache/solr/search/TopLevelJoinQuery.javaindex ce05a02eb95..bbaae01f2ca 100644--- a/solr/core/src/java/org/apache/solr/search/TopLevelJoinQuery.java+++ b/solr/core/src/java/org/apache/solr/search/TopLevelJoinQuery.java@@ -46,7 +46,7 @@ import org.slf4j.LoggerFactory;* {@link JoinQuery} implementation using global (top-level) DocValues ordinals to efficiently* compare values in the "from" and "to" fields.*/-public class TopLevelJoinQuery extends JoinQuery {+public class TopLevelJoinQuery extends JoinQuery implements SolrSearcherRequirer {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());public TopLevelJoinQuery(String fromField, String toField, String coreName, Query subQuery) {@@ -60,6 +60,7 @@ public class TopLevelJoinQuery extends JoinQuery {log.debug("Falling back to JoinQueryWeight because searcher [{}] is not the required SolrIndexSearcher",searcher);+ // TODO This check no longer makes sense as super.createWeight *also* requires a SISreturn super.createWeight(searcher, scoreMode, boost);}diff --git a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.javaindex b0b396cf6cf..619025800ac 100644--- a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java+++ b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java@@ -60,8 +60,9 @@ import org.apache.solr.search.BitDocSet;import org.apache.solr.search.DocSet;import org.apache.solr.search.DocSetUtil;import org.apache.solr.search.SolrIndexSearcher;+import org.apache.solr.search.SolrSearcherRequirer;-public class CrossCollectionJoinQuery extends Query {+public class CrossCollectionJoinQuery extends Query implements SolrSearcherRequirer {protected final String query;protected final String zkHost;diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.javaindex 3dddedb755f..051c3f639f5 100644--- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java+++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java@@ -46,6 +46,7 @@ import org.apache.solr.schema.SchemaField;import org.apache.solr.search.BitDocSet;import org.apache.solr.search.DocSet;import org.apache.solr.search.SolrIndexSearcher;+import org.apache.solr.search.SolrSearcherRequirer;/*** GraphQuery - search for nodes and traverse edges in an index.@@ -58,7 +59,7 @@ import org.apache.solr.search.SolrIndexSearcher;** @lucene.experimental*/-public class GraphQuery extends Query {+public class GraphQuery extends Query implements SolrSearcherRequirer {/** The inital node matching query */private Query q;diff --git a/solr/core/src/java/org/apache/solr/search/join/HashRangeQuery.java b/solr/core/src/java/org/apache/solr/search/join/HashRangeQuery.javaindex d0bdc326747..df70e7c9bf0 100644--- a/solr/core/src/java/org/apache/solr/search/join/HashRangeQuery.java+++ b/solr/core/src/java/org/apache/solr/search/join/HashRangeQuery.java@@ -38,8 +38,9 @@ import org.apache.lucene.util.BytesRef;import org.apache.solr.common.util.Hash;import org.apache.solr.search.SolrCache;import org.apache.solr.search.SolrIndexSearcher;+import org.apache.solr.search.SolrSearcherRequirer;-public class HashRangeQuery extends Query {+public class HashRangeQuery extends Query implements SolrSearcherRequirer {protected final String field;protected final int lower;diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.javaindex 2704ef4e027..4c55eaab9b5 100644--- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java+++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java@@ -65,6 +65,7 @@ import org.apache.solr.search.FunctionRangeQuery;import org.apache.solr.search.QParser;import org.apache.solr.search.QueryUtils;import org.apache.solr.search.SolrIndexSearcher;+import org.apache.solr.search.SolrSearcherRequirementDetector;import org.apache.solr.search.SyntaxError;import org.apache.solr.search.function.ValueSourceRangeFilter;import org.apache.solr.util.RefCounted;@@ -79,6 +80,8 @@ import org.slf4j.LoggerFactory;public class DirectUpdateHandler2 extends UpdateHandlerimplements SolrCoreState.IndexWriterCloser, SolrMetricProducer {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+private static final int NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER = -1;protected final SolrCoreState solrCoreState;@@ -115,8 +118,6 @@ public class DirectUpdateHandler2 extends UpdateHandlerthis.commitWithinSoftCommit = value;}- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-public DirectUpdateHandler2(SolrCore core) {super(core, null, false);@@ -572,12 +573,23 @@ public class DirectUpdateHandler2 extends UpdateHandlerdeleteByQueryCommandsCumulative.mark();boolean madeIt = false;try {+ Query q = getQuery(cmd);++ // Parts of the DBQ codepath run the query using a standard Lucene IndexSearcher, so block any+ // queries that we know require a SolrIndexSearcher+ final var unsupportedQDetector = new SolrSearcherRequirementDetector();+ q.visit(unsupportedQDetector);+ if (unsupportedQDetector.getRequiresSolrSearcher()) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ "Query [" + cmd.getQuery() + "] is not supported in delete-by-query operations");+ }+if ((cmd.getFlags() & UpdateCommand.IGNORE_INDEXWRITER) != 0) {if (ulog != null) ulog.deleteByQuery(cmd);madeIt = true;return;}- Query q = getQuery(cmd);boolean delAll = MatchAllDocsQuery.class == q.getClass();diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.javaindex f2a6d7c1d76..1e6a5c04b5c 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java@@ -16,11 +16,15 @@*/package org.apache.solr.cloud;+import static org.hamcrest.Matchers.containsString;++import java.lang.invoke.MethodHandles;import java.net.URL;import java.nio.file.Path;import java.util.Arrays;import java.util.HashMap;import java.util.Map;+import java.util.UUID;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;@@ -40,9 +44,14 @@ import org.apache.solr.embedded.JettySolrRunner;import org.junit.AfterClass;import org.junit.Before;import org.junit.BeforeClass;+import org.junit.Test;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;public class TestCloudDeleteByQuery extends SolrCloudTestCase {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+private static final int NUM_SHARDS = 2;private static final int REPLICATION_FACTOR = 2;private static final int NUM_SERVERS = 5;@@ -255,6 +264,37 @@ public class TestCloudDeleteByQuery extends SolrCloudTestCase {testMalformedDBQ(NO_COLLECTION_CLIENT);}+ // See SOLR-17677 for context+ @Test+ public void testDBQWithUnsupportedQueryReturns400() throws Exception {+ final var unsupportedQueryExamples =+ new String[] {+ "{!join from=expected_shard_s to=expected_shard_s v=\"expected_shard_s:5\"}",+ "{!graph from=expected_shard_s to=expected_shard_s v=\"expected_shard_s:5\"}",+ "{!hash_range f=\"foo_i\" l=\"0\" u=\"12345\"}"+ };++ update(params()).add(doc(f("id", UUID.randomUUID().toString()))).process(COLLECTION_CLIENT);+ for (String queryStr : unsupportedQueryExamples) {+ log.info("Testing unsupported DBQ query: {}", queryStr);+ SolrException e =+ expectThrows(+ SolrException.class,+ () -> {+ update(params()).deleteByQuery(queryStr).process(COLLECTION_CLIENT);+ });+ assertEquals("Unexpected status code for DBQ with query " + queryStr, 400, e.code());+ final var expectedStr =+ "Query [" + queryStr + "] is not supported in delete-by-query operations";+ assertThat(e.getMessage(), containsString(expectedStr));+ }++ final var acceptableJoin =+ "{!join method=dvWithScore score=None from=expected_shard_s to=expected_shard_s v=\"expected_shard_s:5\"}";+ final var response = update(params()).deleteByQuery(acceptableJoin).process(COLLECTION_CLIENT);+ assertEquals(0, response.getStatus());+ }+public static UpdateRequest update(SolrParams params, SolrInputDocument... docs) {UpdateRequest r = new UpdateRequest();r.setParams(new ModifiableSolrParams(params));diff --git a/solr/core/src/test/org/apache/solr/search/SolrSearcherRequirementDetectorTest.java b/solr/core/src/test/org/apache/solr/search/SolrSearcherRequirementDetectorTest.javanew file mode 100644index 00000000000..526d9a23a26--- /dev/null+++ b/solr/core/src/test/org/apache/solr/search/SolrSearcherRequirementDetectorTest.java@@ -0,0 +1,93 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.search;++import static org.hamcrest.Matchers.instanceOf;+import static org.hamcrest.Matchers.not;++import org.apache.lucene.index.Term;+import org.apache.lucene.search.BooleanClause;+import org.apache.lucene.search.BooleanQuery;+import org.apache.lucene.search.TermQuery;+import org.apache.solr.SolrTestCase;+import org.apache.solr.search.join.GraphQuery;+import org.junit.Test;++/** Unit tests for {@link SolrSearcherRequirementDetector} */+public class SolrSearcherRequirementDetectorTest extends SolrTestCase {++ @Test+ public void testDetectsWhenQueriesDontRequireSolrSearcher() {+ final var termQuery = new TermQuery(new Term("someField", "someFieldValue"));+ final var termQuery2 = new TermQuery(new Term("someField", "someOtherFieldValue"));++ var needsSearcherDetector = new SolrSearcherRequirementDetector();+ termQuery.visit(needsSearcherDetector);+ assertFalse(needsSearcherDetector.getRequiresSolrSearcher());++ final var boolQuery =+ new BooleanQuery.Builder()+ .add(termQuery, BooleanClause.Occur.MUST)+ .add(termQuery2, BooleanClause.Occur.SHOULD)+ .build();+ needsSearcherDetector = new SolrSearcherRequirementDetector();+ boolQuery.visit(needsSearcherDetector);+ assertFalse(needsSearcherDetector.getRequiresSolrSearcher());+ }++ @Test+ public void testDetectsWhenQueriesDoRequireSolrSearcher_TopLevel() {+ final var termQuery = new TermQuery(new Term("someField", "someFieldValue"));++ var needsSearcherDetector = new SolrSearcherRequirementDetector();+ final var joinQuery = new JoinQuery("fromField", "toField", "someCoreName", termQuery);+ assertThat(+ joinQuery,+ instanceOf(SolrSearcherRequirer.class)); // Ensure that JoinQuery still requires SIS+ joinQuery.visit(needsSearcherDetector);+ assertTrue(needsSearcherDetector.getRequiresSolrSearcher());++ needsSearcherDetector = new SolrSearcherRequirementDetector();+ final var graphQuery = new GraphQuery(termQuery, "fromField", "toField");+ assertThat(+ graphQuery,+ instanceOf(SolrSearcherRequirer.class)); // Ensure that GraphQuery still requires SIS+ graphQuery.visit(needsSearcherDetector);+ assertTrue(needsSearcherDetector.getRequiresSolrSearcher());+ }++ @Test+ public void testDeteectsWhenQueriesDoRequireSolrSearcher_Nested() {+ final var termQuery = new TermQuery(new Term("someField", "someFieldValue"));+ final var termQuery2 = new TermQuery(new Term("someField", "someOtherFieldValue"));+ final var joinQuery = new JoinQuery("fromField", "toField", "someCoreName", termQuery);+ final var boolQuery =+ new BooleanQuery.Builder()+ .add(new BooleanClause(termQuery, BooleanClause.Occur.MUST))+ .add(new BooleanClause(termQuery2, BooleanClause.Occur.SHOULD))+ .add(new BooleanClause(joinQuery, BooleanClause.Occur.SHOULD))+ .build();++ final var needsSearcherDetector = new SolrSearcherRequirementDetector();+ // Top level query and some leaves don't require SIS, but JoinQuery does.+ assertThat(boolQuery, not(instanceOf(SolrSearcherRequirer.class)));+ assertThat(termQuery, not(instanceOf(SolrSearcherRequirer.class)));+ assertThat(joinQuery, instanceOf(SolrSearcherRequirer.class));+ boolQuery.visit(needsSearcherDetector);+ assertTrue(needsSearcherDetector.getRequiresSolrSearcher());+ }+}diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-update-handlers.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-update-handlers.adocindex 92b7bedcdec..44aa38a85a7 100644--- a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-update-handlers.adoc+++ b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-update-handlers.adoc@@ -181,8 +181,10 @@ A single delete message can contain multiple delete operations.[IMPORTANT]====-When using the Join query parser in a Delete By Query, you should use the `score` parameter with a value of "none" to avoid a `ClassCastException`.-See the section on the xref:query-guide:join-query-parser.adoc[] for more details on the `score` parameter.+Some queries, including many `{!join}` and `{!graph}` queries, are not supported in delete operations and will return a 400 error.++Users interested in using the Join query parser in a Delete By Query may do so by specifying a `score` parameter with the value "none" (and ensuring no `method` parameter is provided).+See the section on the xref:query-guide:join-query-parser.adoc[] for more details on the `score` parameter and its usage.====