Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-17335: New "vectorSimilarity" QParser for matching documents mased on a minimum vector similarity threshold

Chris Hostetterb49f3d076942
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ca642e9bbc3..44a7af702b1 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -113,6 +113,8 @@ New Features
* SOLR-17277: Circuit breakers may now be configured in a "soft" or "warnOnly" mode in order to more easily test out new thresholds. Soft breakers will log out a
message on each relevant request when tripped, but will not otherwise impact or short circuit the requests. (Jason Gerlowski)
+* SOLR-17335: New "vectorSimilarity" QParser for matching documents mased on a minimum vector similarity threshold. (hossman)
+
Improvements
---------------------
* SOLR-17137: Enable Prometheus exporter to communicate with SSL protected Solr. (Eivind Bergstøl via Eric Pugh)
diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
index 99cb9efb0c7..8146c585cf0 100644
--- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
@@ -31,6 +31,7 @@ import org.apache.solr.search.join.HashRangeQParserPlugin;
import org.apache.solr.search.mlt.MLTContentQParserPlugin;
import org.apache.solr.search.mlt.MLTQParserPlugin;
import org.apache.solr.search.neural.KnnQParserPlugin;
+import org.apache.solr.search.neural.VectorSimilarityQParserPlugin;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrInfoBean {
@@ -89,6 +90,7 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI
map.put(HashRangeQParserPlugin.NAME, new HashRangeQParserPlugin());
map.put(RankQParserPlugin.NAME, new RankQParserPlugin());
map.put(KnnQParserPlugin.NAME, new KnnQParserPlugin());
+ map.put(VectorSimilarityQParserPlugin.NAME, new VectorSimilarityQParserPlugin());
standardPlugins = Collections.unmodifiableMap(map);
}
diff --git a/solr/core/src/java/org/apache/solr/search/neural/AbstractVectorQParserBase.java b/solr/core/src/java/org/apache/solr/search/neural/AbstractVectorQParserBase.java
new file mode 100644
index 00000000000..4cafb45744e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/neural/AbstractVectorQParserBase.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.neural;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.lucene.search.Query;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.DenseVectorField;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QueryParsing;
+import org.apache.solr.search.QueryUtils;
+import org.apache.solr.search.SyntaxError;
+
+public abstract class AbstractVectorQParserBase extends QParser {
+
+ static final String PRE_FILTER = "preFilter";
+ static final String EXCLUDE_TAGS = "excludeTags";
+ static final String INCLUDE_TAGS = "includeTags";
+
+ private final String denseVectorFieldName;
+ private final String vectorToSearch;
+
+ public AbstractVectorQParserBase(
+ String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ super(qstr, localParams, params, req);
+ vectorToSearch = localParams.get(QueryParsing.V);
+ denseVectorFieldName = localParams.get(QueryParsing.F);
+ }
+
+ protected String getVectorToSearch() {
+ if (vectorToSearch == null || vectorToSearch.isEmpty()) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST, "the Dense Vector value 'v' to search is missing");
+ }
+ return vectorToSearch;
+ }
+
+ protected String getFieldName() {
+ if (denseVectorFieldName == null || denseVectorFieldName.isEmpty()) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST, "the Dense Vector field 'f' is missing");
+ }
+ return denseVectorFieldName;
+ }
+
+ protected static DenseVectorField getCheckedFieldType(SchemaField schemaField) {
+ FieldType fieldType = schemaField.getType();
+ if (!(fieldType instanceof DenseVectorField)) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "only DenseVectorField is compatible with Vector Query Parsers");
+ }
+
+ return (DenseVectorField) fieldType;
+ }
+
+ protected Query getFilterQuery() throws SolrException, SyntaxError {
+
+ // Default behavior of FQ wrapping, and suitability of some local params
+ // depends on wether we are a sub-query or not
+ final boolean isSubQuery = recurseCount != 0;
+
+ // include/exclude tags for global fqs to wrap;
+ // Check these up front for error handling if combined with `fq` local param.
+ final List<String> includedGlobalFQTags = getLocalParamTags(INCLUDE_TAGS);
+ final List<String> excludedGlobalFQTags = getLocalParamTags(EXCLUDE_TAGS);
+ final boolean haveGlobalFQTags =
+ !(includedGlobalFQTags.isEmpty() && excludedGlobalFQTags.isEmpty());
+
+ if (haveGlobalFQTags) {
+ // Some early error handling of incompatible options...
+
+ if (isFilter()) { // this knn query is itself a filter query
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Knn Query Parser used as a filter does not support "
+ + INCLUDE_TAGS
+ + " or "
+ + EXCLUDE_TAGS
+ + " localparams");
+ }
+
+ if (isSubQuery) { // this knn query is a sub-query of a broader query (possibly disjunction)
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Knn Query Parser used as a sub-query does not support "
+ + INCLUDE_TAGS
+ + " or "
+ + EXCLUDE_TAGS
+ + " localparams");
+ }
+ }
+
+ // Explicit local params specifying the filter(s) to wrap
+ final String[] preFilters = getLocalParams().getParams(PRE_FILTER);
+ if (null != preFilters) {
+
+ // We don't particularly care if preFilters is empty, the usage below will still work,
+ // but SolrParams API says it should be null not empty...
+ assert 0 != preFilters.length
+ : "SolrParams.getParams should return null, never zero len array";
+
+ if (haveGlobalFQTags) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Knn Query Parser does not support combining "
+ + PRE_FILTER
+ + " localparam with either "
+ + INCLUDE_TAGS
+ + " or "
+ + EXCLUDE_TAGS
+ + " localparams");
+ }
+
+ final List<Query> preFilterQueries = new ArrayList<>(preFilters.length);
+ for (String f : preFilters) {
+ final QParser parser = subQuery(f, null);
+ parser.setIsFilter(true);
+
+ // maybe null, ie: `preFilter=""`
+ final Query filter = parser.getQuery();
+ if (null != filter) {
+ preFilterQueries.add(filter);
+ }
+ }
+ try {
+ return req.getSearcher().getProcessedFilter(preFilterQueries).filter;
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ // No explicit `preFilter` localparams specifying what we should filter on.
+ //
+ // So now, if we're either a filter or a subquery, we have to default to
+ // not wrapping anything...
+ if (isFilter() || isSubQuery) {
+ return null;
+ }
+
+ // At this point we now are a (regular) query and can wrap global `fq` filters...
+ try {
+ // Start by assuming we wrap all global filters,
+ // then adjust our list based on include/exclude tag params
+ List<Query> globalFQs = QueryUtils.parseFilterQueries(req);
+
+ // Adjust our globalFQs based on any include/exclude we may have
+ if (!includedGlobalFQTags.isEmpty()) {
+ // NOTE: Even if no FQs match the specified tag(s) the fact that tags were specified
+ // means we should replace globalFQs (even with a possibly empty list)
+ globalFQs = new ArrayList<>(QueryUtils.getTaggedQueries(req, includedGlobalFQTags));
+ }
+ if (null != excludedGlobalFQTags) {
+ globalFQs.removeAll(QueryUtils.getTaggedQueries(req, excludedGlobalFQTags));
+ }
+
+ return req.getSearcher().getProcessedFilter(globalFQs).filter;
+
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ /**
+ * @return set (possibly empty) of tags specified in the given local param
+ * @see StrUtils#splitSmart
+ * @see QueryUtils#getTaggedQueries
+ * @see #localParams
+ */
+ private List<String> getLocalParamTags(final String param) {
+ final String[] strVals = localParams.getParams(param);
+ if (null == strVals) {
+ return Collections.emptyList();
+ }
+ final List<String> tags = new ArrayList<>(strVals.length * 2);
+ for (String val : strVals) {
+ // This ensures parity w/how QParser constructor builds tagMap,
+ // and that empty strings will make it into our List (for "include nothing")
+ if (0 < val.indexOf(',')) {
+ tags.addAll(StrUtils.splitSmart(val, ','));
+ } else {
+ tags.add(val);
+ }
+ }
+ return tags;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java
index 252a4fcabc7..166dada5b7f 100644
--- a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java
+++ b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java
@@ -16,204 +16,31 @@
*/
package org.apache.solr.search.neural;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import org.apache.lucene.search.Query;
-import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.DenseVectorField;
-import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.search.QParser;
-import org.apache.solr.search.QueryParsing;
-import org.apache.solr.search.QueryUtils;
import org.apache.solr.search.SyntaxError;
-public class KnnQParser extends QParser {
-
- static final String PRE_FILTER = "preFilter";
- static final String EXCLUDE_TAGS = "excludeTags";
- static final String INCLUDE_TAGS = "includeTags";
+public class KnnQParser extends AbstractVectorQParserBase {
// retrieve the top K results based on the distance similarity function
static final String TOP_K = "topK";
static final int DEFAULT_TOP_K = 10;
- /**
- * Constructor for the QParser
- *
- * @param qstr The part of the query string specific to this parser
- * @param localParams The set of parameters that are specific to this QParser. See
- * https://solr.apache.org/guide/solr/latest/query-guide/local-params.html
- * @param params The rest of the {@link SolrParams}
- * @param req The original {@link SolrQueryRequest}.
- */
public KnnQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req);
}
@Override
public Query parse() throws SyntaxError {
- String denseVectorField = localParams.get(QueryParsing.F);
- String vectorToSearch = localParams.get(QueryParsing.V);
- int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K);
-
- if (denseVectorField == null || denseVectorField.isEmpty()) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST, "the Dense Vector field 'f' is missing");
- }
-
- if (vectorToSearch == null || vectorToSearch.isEmpty()) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST, "the Dense Vector value 'v' to search is missing");
- }
-
- SchemaField schemaField = req.getCore().getLatestSchema().getField(denseVectorField);
- FieldType fieldType = schemaField.getType();
- if (!(fieldType instanceof DenseVectorField)) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "only DenseVectorField is compatible with Knn Query Parser");
- }
-
- DenseVectorField denseVectorType = (DenseVectorField) fieldType;
+ final SchemaField schemaField = req.getCore().getLatestSchema().getField(getFieldName());
+ final DenseVectorField denseVectorType = getCheckedFieldType(schemaField);
+ final String vectorToSearch = getVectorToSearch();
+ final int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K);
return denseVectorType.getKnnVectorQuery(
schemaField.getName(), vectorToSearch, topK, getFilterQuery());
}
-
- private Query getFilterQuery() throws SolrException, SyntaxError {
-
- // Default behavior of FQ wrapping, and suitability of some local params
- // depends on wether we are a sub-query or not
- final boolean isSubQuery = recurseCount != 0;
-
- // include/exclude tags for global fqs to wrap;
- // Check these up front for error handling if combined with `fq` local param.
- final List<String> includedGlobalFQTags = getLocalParamTags(INCLUDE_TAGS);
- final List<String> excludedGlobalFQTags = getLocalParamTags(EXCLUDE_TAGS);
- final boolean haveGlobalFQTags =
- !(includedGlobalFQTags.isEmpty() && excludedGlobalFQTags.isEmpty());
-
- if (haveGlobalFQTags) {
- // Some early error handling of incompatible options...
-
- if (isFilter()) { // this knn query is itself a filter query
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "Knn Query Parser used as a filter does not support "
- + INCLUDE_TAGS
- + " or "
- + EXCLUDE_TAGS
- + " localparams");
- }
-
- if (isSubQuery) { // this knn query is a sub-query of a broader query (possibly disjunction)
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "Knn Query Parser used as a sub-query does not support "
- + INCLUDE_TAGS
- + " or "
- + EXCLUDE_TAGS
- + " localparams");
- }
- }
-
- // Explicit local params specifying the filter(s) to wrap
- final String[] preFilters = getLocalParams().getParams(PRE_FILTER);
- if (null != preFilters) {
-
- // We don't particularly care if preFilters is empty, the usage below will still work,
- // but SolrParams API says it should be null not empty...
- assert 0 != preFilters.length
- : "SolrParams.getParams should return null, never zero len array";
-
- if (haveGlobalFQTags) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "Knn Query Parser does not support combining "
- + PRE_FILTER
- + " localparam with either "
- + INCLUDE_TAGS
- + " or "
- + EXCLUDE_TAGS
- + " localparams");
- }
-
- final List<Query> preFilterQueries = new ArrayList<>(preFilters.length);
- for (String f : preFilters) {
- final QParser parser = subQuery(f, null);
- parser.setIsFilter(true);
-
- // maybe null, ie: `preFilter=""`
- final Query filter = parser.getQuery();
- if (null != filter) {
- preFilterQueries.add(filter);
- }
- }
- try {
- return req.getSearcher().getProcessedFilter(preFilterQueries).filter;
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
- }
-
- // No explicit `preFilter` localparams specifying what we should filter on.
- //
- // So now, if we're either a filter or a subquery, we have to default to
- // not wrapping anything...
- if (isFilter() || isSubQuery) {
- return null;
- }
-
- // At this point we now are a (regular) query and can wrap global `fq` filters...
- try {
- // Start by assuming we wrap all global filters,
- // then adjust our list based on include/exclude tag params
- List<Query> globalFQs = QueryUtils.parseFilterQueries(req);
-
- // Adjust our globalFQs based on any include/exclude we may have
- if (!includedGlobalFQTags.isEmpty()) {
- // NOTE: Even if no FQs match the specified tag(s) the fact that tags were specified
- // means we should replace globalFQs (even with a possibly empty list)
- globalFQs = new ArrayList<>(QueryUtils.getTaggedQueries(req, includedGlobalFQTags));
- }
- if (null != excludedGlobalFQTags) {
- globalFQs.removeAll(QueryUtils.getTaggedQueries(req, excludedGlobalFQTags));
- }
-
- return req.getSearcher().getProcessedFilter(globalFQs).filter;
-
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
- }
-
- /**
- * @return set (possibly empty) of tags specified in the given local param
- * @see StrUtils#splitSmart
- * @see QueryUtils#getTaggedQueries
- * @see #localParams
- */
- private List<String> getLocalParamTags(final String param) {
- final String[] strVals = localParams.getParams(param);
- if (null == strVals) {
- return Collections.emptyList();
- }
- final List<String> tags = new ArrayList<>(strVals.length * 2);
- for (String val : strVals) {
- // This ensures parity w/how QParser constructor builds tagMap,
- // and that empty strings will make it into our List (for "include nothing")
- if (0 < val.indexOf(',')) {
- tags.addAll(StrUtils.splitSmart(val, ','));
- } else {
- tags.add(val);
- }
- }
- return tags;
- }
}
diff --git a/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParser.java b/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParser.java
new file mode 100644
index 00000000000..e3ec2f242f7
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParser.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.neural;
+
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.search.ByteVectorSimilarityQuery;
+import org.apache.lucene.search.FloatVectorSimilarityQuery;
+import org.apache.lucene.search.Query;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.DenseVectorField;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.SyntaxError;
+import org.apache.solr.util.vector.DenseVectorParser;
+
+public class VectorSimilarityQParser extends AbstractVectorQParserBase {
+
+ // retrieve the top results based on the distance similarity function thresholds
+ static final String MIN_RETURN = "minReturn";
+ static final String MIN_TRAVERSE = "minTraverse";
+
+ static final float DEFAULT_MIN_TRAVERSE = Float.NEGATIVE_INFINITY;
+
+ public VectorSimilarityQParser(
+ String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ super(qstr, localParams, params, req);
+ }
+
+ @Override
+ public Query parse() throws SyntaxError {
+ final String fieldName = getFieldName();
+ final SchemaField schemaField = req.getCore().getLatestSchema().getField(fieldName);
+ final DenseVectorField denseVectorType = getCheckedFieldType(schemaField);
+ final String vectorToSearch = getVectorToSearch();
+ final float minTraverse = localParams.getFloat(MIN_TRAVERSE, DEFAULT_MIN_TRAVERSE);
+ final Float minReturn = localParams.getFloat(MIN_RETURN);
+ if (null == minReturn) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ MIN_RETURN + " is required to use Vector Similarity QParser");
+ }
+
+ final DenseVectorParser vectorBuilder =
+ denseVectorType.getVectorBuilder(vectorToSearch, DenseVectorParser.BuilderPhase.QUERY);
+
+ final VectorEncoding vectorEncoding = denseVectorType.getVectorEncoding();
+ switch (vectorEncoding) {
+ case FLOAT32:
+ return new FloatVectorSimilarityQuery(
+ fieldName, vectorBuilder.getFloatVector(), minTraverse, minReturn, getFilterQuery());
+ case BYTE:
+ return new ByteVectorSimilarityQuery(
+ fieldName, vectorBuilder.getByteVector(), minTraverse, minReturn, getFilterQuery());
+ default:
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Unexpected state. Vector Encoding: " + vectorEncoding);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParserPlugin.java
new file mode 100644
index 00000000000..4109d0df388
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParserPlugin.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.neural;
+
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QParserPlugin;
+
+/** A neural query parser to run min-similarity search on Dense Vector fields. */
+public class VectorSimilarityQParserPlugin extends QParserPlugin {
+ public static final String NAME = "vectorSimilarity";
+
+ @Override
+ public QParser createParser(
+ String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ return new VectorSimilarityQParser(qstr, localParams, params, req);
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
index 653c0935879..6e675771560 100644
--- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
+++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
@@ -1469,6 +1469,80 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
}
}
+ public void testQueryVecSim() throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "0");
+ doc.addField("vector", Arrays.asList(1, 2, 3, 4));
+ assertU(adoc(doc));
+ assertU(commit());
+
+ final String common = "!vectorSimilarity minReturn=0.3 f=vector";
+ final String qvec = "[1.0,2.0,3.0,4.0]";
+
+ try (SolrQueryRequest req0 = req()) {
+
+ // no filters
+ final Query fqNull =
+ assertQueryEqualsAndReturn(
+ "vectorSimilarity",
+ req0,
+ "{" + common + "}" + qvec,
+ "{" + common + " minTraverse='-Infinity'}" + qvec,
+ "{" + common + " preFilter=''}" + qvec,
+ "{" + common + " v=" + qvec + "}");
+
+ try (SolrQueryRequest req1 = req("fq", "{!tag=t1}id:1", "xxx", "id:1")) {
+ // either global fq, or (same) preFilter as localparam
+ final Query fqOne =
+ assertQueryEqualsAndReturn(
+ "vectorSimilarity",
+ req1,
+ "{" + common + "}" + qvec,
+ "{" + common + " includeTags=t1}" + qvec,
+ "{" + common + " preFilter='id:1'}" + qvec,
+ "{" + common + " preFilter=$xxx}" + qvec,
+ "{" + common + " v=" + qvec + "}");
+ QueryUtils.checkUnequal(fqNull, fqOne);
+
+ try (SolrQueryRequest req2 = req("fq", "{!tag=t2}id:2", "xxx", "id:1", "yyy", "")) {
+ // override global fq with local param to use different preFilter
+ final Query fqOneOverride =
+ assertQueryEqualsAndReturn(
+ "vectorSimilarity",
+ req2,
+ "{" + common + " preFilter='id:1'}" + qvec,
+ "{" + common + " preFilter=$xxx}" + qvec);
+ QueryUtils.checkEqual(fqOne, fqOneOverride);
+
+ // override global fq with local param to use no preFilters
+ final Query fqNullOverride =
+ assertQueryEqualsAndReturn(
+ "vectorSimilarity",
+ req2,
+ "{" + common + " preFilter=''}" + qvec,
+ "{" + common + " excludeTags=t2}" + qvec,
+ "{" + common + " preFilter=$yyy}" + qvec);
+ QueryUtils.checkEqual(fqNull, fqNullOverride);
+ }
+ }
+
+ try (SolrQueryRequest reqPostFilter = req("fq", "{!tag=post frange cache=false l=0}9.9")) {
+ // global post-filter fq should always be ignored
+ final Query fqPostFilter =
+ assertQueryEqualsAndReturn(
+ "vectorSimilarity",
+ reqPostFilter,
+ "{" + common + "}" + qvec,
+ "{" + common + " includeTags=post}" + qvec);
+ QueryUtils.checkEqual(fqNull, fqPostFilter);
+ }
+
+ } finally {
+ delQ("id:0");
+ assertU(commit());
+ }
+ }
+
/**
* NOTE: defType is not only used to pick the parser, but also to record the parser being tested
* for coverage sanity checking
diff --git a/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java b/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java
index ccd0e3ecc44..f5d5668a7e5 100644
--- a/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java
+++ b/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java
@@ -178,7 +178,7 @@ public class KnnQParserTest extends SolrTestCaseJ4 {
assertQEx(
"Incorrect vector field type should throw Exception",
- "only DenseVectorField is compatible with Knn Query Parser",
+ "only DenseVectorField is compatible with Vector Query Parsers",
req(CommonParams.Q, "{!knn f=id topK=10}" + vectorToSearch, "fl", "id"),
SolrException.ErrorCode.BAD_REQUEST);
}
diff --git a/solr/core/src/test/org/apache/solr/search/neural/VectorSimilarityQParserTest.java b/solr/core/src/test/org/apache/solr/search/neural/VectorSimilarityQParserTest.java
new file mode 100644
index 00000000000..cf6aeb65973
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/neural/VectorSimilarityQParserTest.java
@@ -0,0 +1,902 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.neural;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class VectorSimilarityQParserTest extends SolrTestCaseJ4 {
+ String IDField = "id";
+ String vectorField = "vector";
+ String vectorField2 = "vector2";
+ String vectorFieldByteEncoding = "vector_byte_encoding";
+
+ @Before
+ public void prepareIndex() throws Exception {
+ /* vectorDimension="4" similarityFunction="cosine" */
+ initCore("solrconfig_codec.xml", "schema-densevector.xml");
+
+ List<SolrInputDocument> docsToIndex = this.prepareDocs();
+ for (SolrInputDocument doc : docsToIndex) {
+ assertU(adoc(doc));
+ }
+
+ assertU(commit());
+ }
+
+ private List<SolrInputDocument> prepareDocs() {
+ int docsCount = 13;
+ List<SolrInputDocument> docs = new ArrayList<>(docsCount);
+ for (int i = 1; i < docsCount + 1; i++) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField(IDField, i);
+ docs.add(doc);
+ }
+
+ docs.get(0)
+ .addField(vectorField, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector1= 1.0
+ docs.get(1)
+ .addField(
+ vectorField, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector1= 0.998
+ docs.get(2)
+ .addField(
+ vectorField,
+ Arrays.asList(7.5f, 15.5f, 17.5f, 22.5f)); // cosine distance vector1= 0.992
+ docs.get(3)
+ .addField(
+ vectorField, Arrays.asList(1.4f, 2.4f, 3.4f, 4.4f)); // cosine distance vector1= 0.999
+ docs.get(4)
+ .addField(vectorField, Arrays.asList(30f, 22f, 35f, 20f)); // cosine distance vector1= 0.862
+ docs.get(5)
+ .addField(vectorField, Arrays.asList(40f, 1f, 1f, 200f)); // cosine distance vector1= 0.756
+ docs.get(6)
+ .addField(vectorField, Arrays.asList(5f, 10f, 20f, 40f)); // cosine distance vector1= 0.970
+ docs.get(7)
+ .addField(
+ vectorField, Arrays.asList(120f, 60f, 30f, 15f)); // cosine distance vector1= 0.515
+ docs.get(8)
+ .addField(
+ vectorField, Arrays.asList(200f, 50f, 100f, 25f)); // cosine distance vector1= 0.554
+ docs.get(9)
+ .addField(
+ vectorField, Arrays.asList(1.8f, 2.5f, 3.7f, 4.9f)); // cosine distance vector1= 0.997
+ docs.get(10)
+ .addField(vectorField2, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector2= 1
+ docs.get(11)
+ .addField(
+ vectorField2,
+ Arrays.asList(7.5f, 15.5f, 17.5f, 22.5f)); // cosine distance vector2= 0.992
+ docs.get(12)
+ .addField(
+ vectorField2, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector2= 0.998
+
+ docs.get(0).addField(vectorFieldByteEncoding, Arrays.asList(1, 2, 3, 4));
+ docs.get(1).addField(vectorFieldByteEncoding, Arrays.asList(2, 2, 1, 4));
+ docs.get(2).addField(vectorFieldByteEncoding, Arrays.asList(1, 2, 1, 2));
+ docs.get(3).addField(vectorFieldByteEncoding, Arrays.asList(7, 2, 1, 3));
+ docs.get(4).addField(vectorFieldByteEncoding, Arrays.asList(19, 2, 4, 4));
+ docs.get(5).addField(vectorFieldByteEncoding, Arrays.asList(19, 2, 4, 4));
+ docs.get(6).addField(vectorFieldByteEncoding, Arrays.asList(18, 2, 4, 4));
+ docs.get(7).addField(vectorFieldByteEncoding, Arrays.asList(8, 3, 2, 4));
+
+ return docs;
+ }
+
+ @After
+ public void cleanUp() {
+ clearIndex();
+ deleteCore();
+ }
+
+ @Test
+ public void incorrectVectorFieldType_shouldThrowException() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQEx(
+ "Incorrect vector field type should throw Exception",
+ "only DenseVectorField is compatible with ",
+ req(CommonParams.Q, "{!vectorSimilarity f=id}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void undefinedVectorField_shouldThrowException() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQEx(
+ "Undefined vector field should throw Exception",
+ "undefined field: \"notExistent\"",
+ req(CommonParams.Q, "{!vectorSimilarity f=notExistent}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void missingVectorField_shouldThrowException() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQEx(
+ "missing vector field should throw Exception",
+ "the Dense Vector field 'f' is missing",
+ req(CommonParams.Q, "{!vectorSimilarity}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void correctVectorField_shouldSearchOnThatField() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQ(
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector2 minReturn=0.8}" + vectorToSearch,
+ "fl",
+ "id"),
+ "//result[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.='11']",
+ "//result/doc[2]/str[@name='id'][.='13']",
+ "//result/doc[3]/str[@name='id'][.='12']");
+ }
+
+ @Test
+ public void vectorByteEncodingField_shouldSearchOnThatField() {
+ String vectorToSearch = "[2, 2, 1, 3]";
+
+ float minR =
+ VectorSimilarityFunction.COSINE.compare(new byte[] {2, 2, 1, 3}, new byte[] {1, 2, 1, 2});
+ assertQ(
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector_byte_encoding minReturn=" + minR + "}" + vectorToSearch,
+ "fl",
+ "id"),
+ "//result[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='2']",
+ "//result/doc[2]/str[@name='id'][.='3']");
+
+ vectorToSearch = "[8, 3, 2, 4]";
+ minR =
+ VectorSimilarityFunction.COSINE.compare(new byte[] {8, 3, 2, 4}, new byte[] {7, 2, 1, 3});
+
+ assertQ(
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector_byte_encoding minReturn=" + minR + "}" + vectorToSearch,
+ "fl",
+ "id,score,vector_byte_encoding",
+ "indent",
+ "true"),
+ "//result[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='8']",
+ "//result/doc[2]/str[@name='id'][.='4']");
+ }
+
+ @Test
+ public void vectorByteEncodingField_shouldRaiseExceptionIfQueryUsesFloatVectors() {
+ String vectorToSearch = "[8.3, 4.3, 2.1, 4.1]";
+
+ assertQEx(
+ "incorrect vector element: '8.3'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ "incorrect vector element: '8.3'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector_byte_encoding minReturn=0.0}" + vectorToSearch,
+ "fl",
+ "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void
+ vectorByteEncodingField_shouldRaiseExceptionWhenQueryContainsValuesOutsideByteValueRange() {
+ String vectorToSearch = "[1, -129, 3, 5]";
+
+ assertQEx(
+ "incorrect vector element: ' -129'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ "incorrect vector element: ' -129'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector_byte_encoding minReturn=0.0}" + vectorToSearch,
+ "fl",
+ "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "[1, 3, 156, 5]";
+
+ assertQEx(
+ "incorrect vector element: ' 156'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ "incorrect vector element: ' 156'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector_byte_encoding minReturn=0.0}" + vectorToSearch,
+ "fl",
+ "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void missingVectorToSearch_shouldThrowException() {
+ assertQEx(
+ "missing vector to search should throw Exception",
+ "the Dense Vector value 'v' to search is missing",
+ req(CommonParams.Q, "{!vectorSimilarity f=vector minReturn=0.0}", "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void incorrectVectorToSearchDimension_shouldThrowException() {
+ String vectorToSearch = "[2.0, 4.4, 3.]";
+ assertQEx(
+ "missing vector to search should throw Exception",
+ "incorrect vector dimension. The vector value has size 3 while it is expected a vector with size 4",
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch,
+ "fl",
+ "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "[2.0, 4.4,,]";
+ assertQEx(
+ "incorrect vector to search should throw Exception",
+ "incorrect vector dimension. The vector value has size 2 while it is expected a vector with size 4",
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch,
+ "fl",
+ "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void incorrectVectorToSearch_shouldThrowException() {
+ String vectorToSearch = "2.0, 4.4, 3.5, 6.4";
+ assertQEx(
+ "incorrect vector to search should throw Exception",
+ "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float",
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch,
+ "fl",
+ "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "[2.0, 4.4, 3.5, 6.4";
+ assertQEx(
+ "incorrect vector to search should throw Exception",
+ "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float",
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch,
+ "fl",
+ "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "2.0, 4.4, 3.5, 6.4]";
+ assertQEx(
+ "incorrect vector to search should throw Exception",
+ "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float",
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch,
+ "fl",
+ "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "[2.0, 4.4, 3.5, stringElement]";
+ assertQEx(
+ "incorrect vector to search should throw Exception",
+ "incorrect vector element: ' stringElement'. The expected format is:'[f1,f2..f3]' where each element f is a float",
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch,
+ "fl",
+ "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "[2.0, 4.4, , ]";
+ assertQEx(
+ "incorrect vector to search should throw Exception",
+ "incorrect vector element: ' '. The expected format is:'[f1,f2..f3]' where each element f is a float",
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch,
+ "fl",
+ "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void correctQuery_shouldRankBySimilarityFunction() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQ(
+ req(
+ CommonParams.Q,
+ "{!vectorSimilarity f=vector minReturn=0.8}" + vectorToSearch,
+ "fl",
+ "id"),
+ "//result[@numFound='8']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='4']",
+ "//result/doc[3]/str[@name='id'][.='2']",
+ "//result/doc[4]/str[@name='id'][.='10']",
+ "//result/doc[5]/str[@name='id'][.='3']",
+ "//result/doc[6]/str[@name='id'][.='7']",
+ "//result/doc[7]/str[@name='id'][.='5']",
+ "//result/doc[8]/str[@name='id'][.='6']");
+ }
+
+ @Test
+ public void vecSimQueryUsedInFilters_shouldFilterResultsBeforeTheQueryExecution() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ // minReturn=0.8 -> 1,4,2,10,3,7,5,6
+ assertQ(
+ req(
+ CommonParams.Q,
+ "id:(3 4 9 2)",
+ "fq",
+ "{!vectorSimilarity f=vector minReturn=0.8}" + vectorToSearch,
+ "fq",
+ "id:(4 20 9)",
+ "fl",
+ "id"),
+ "//result[@numFound='1']",
+ "//result/doc[1]/str[@name='id'][.='4']");
+ }
+
+ @Test
+ public void vecSimQueryUsedInFiltersWithPreFilter_shouldFilterResultsBeforeTheQueryExecution() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ // minReturn=0.8 w/localparam preFilter -> 1,4,7
+ assertQ(
+ req(
+ CommonParams.Q,
+ "id:(3 4 7 2)",
+ "fq",
+ "{!vectorSimilarity f=vector minReturn=0.8 preFilter='id:(1 4 7 8 9)'}"
+ + vectorToSearch,
+ "fq",
+ "id:(4 20 7)",
+ "fl",
+ "id"),
+ "//result[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='4']",
+ "//result/doc[2]/str[@name='id'][.='7']");
+ }
+
+ @Test
+ public void vecSimQueryUsedInFilters_rejectIncludeExclude() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ for (String fq :
+ Arrays.asList(
+ "{!vectorSimilarity f=vector minReturn=0.8 includeTags=xxx}" + vectorToSearch,
+ "{!vectorSimilarity f=vector minReturn=0.8 excludeTags=xxx}" + vectorToSearch)) {
+ assertQEx(
+ "fq={!vectorSimilarity...} incompatible with include/exclude localparams",
+ "used as a filter does not support",
+ req("q", "*:*", "fq", fq),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+ }
+
+ @Test
+ public void vecSimQueryAsSubQuery() {
+ final SolrParams common = params("fl", "id", "vec", "[1.0, 2.0, 3.0, 4.0]");
+ final String filt = "id:(2 4 7 9 8 20)";
+
+ // When vecSim parser is a subquery, it should not pre-filter on any global fq params
+ // minReturn -> 1,4,2,10,3,7,5,6 -> fq -> 4,2,7
+ assertQ(
+ req(common, "fq", filt, "q", "*:* AND {!vectorSimilarity f=vector minReturn=0.8 v=$vec}"),
+ "//result[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.='4']",
+ "//result/doc[2]/str[@name='id'][.='2']",
+ "//result/doc[3]/str[@name='id'][.='7']");
+ // minReturn -> 1,4,2,10,3,7,5,6 + '8' -> fq -> 4,2,7,8
+ assertQ(
+ req(
+ common,
+ "fq",
+ filt,
+ "q",
+ "id:8^=0.01 OR {!vectorSimilarity f=vector minReturn=0.8 v=$vec}"),
+ "//result[@numFound='4']",
+ "//result/doc[1]/str[@name='id'][.='4']",
+ "//result/doc[2]/str[@name='id'][.='2']",
+ "//result/doc[3]/str[@name='id'][.='7']",
+ "//result/doc[4]/str[@name='id'][.='8']");
+ }
+
+ @Test
+ public void vecSimQueryAsSubQuery_withPreFilter() {
+ final SolrParams common = params("fl", "id", "vec", "[1.0, 2.0, 3.0, 4.0]");
+ final String filt = "id:(2 4 7 9 8 20 3)";
+
+ // vecSim subquery should still accept `preFilter` local param
+ // filt -> minReturn -> 4,2,3,7
+ assertQ(
+ req(
+ common,
+ "q",
+ "*:* AND {!vectorSimilarity f=vector minReturn=0.8 preFilter='" + filt + "' v=$vec}"),
+ "//result[@numFound='4']",
+ "//result/doc[1]/str[@name='id'][.='4']",
+ "//result/doc[2]/str[@name='id'][.='2']",
+ "//result/doc[3]/str[@name='id'][.='3']",
+ "//result/doc[4]/str[@name='id'][.='7']");
+
+ // it should not pre-filter on any global fq params
+ // filt -> minReturn -> 4,2,3,7 -> fq -> 3,7
+ assertQ(
+ req(
+ common,
+ "fq",
+ "id:(1 9 20 3 5 7 8)",
+ "q",
+ "*:* AND {!vectorSimilarity f=vector minReturn=0.8 preFilter='" + filt + "' v=$vec}"),
+ "//result[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='3']",
+ "//result/doc[2]/str[@name='id'][.='7']");
+ // filt -> minReturn -> 4,2,3,7 + '8' -> fq -> 8,3,7
+ assertQ(
+ req(
+ common,
+ "fq",
+ "id:(1 9 20 3 5 7 8)",
+ "q",
+ "id:8^=100 OR {!vectorSimilarity f=vector minReturn=0.8 preFilter='"
+ + filt
+ + "' v=$vec}"),
+ "//result[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.='8']",
+ "//result/doc[2]/str[@name='id'][.='3']",
+ "//result/doc[3]/str[@name='id'][.='7']");
+ }
+
+ @Test
+ public void vecSimQueryAsSubQuery_rejectIncludeExclude() {
+ final SolrParams common = params("fl", "id", "vec", "[1.0, 2.0, 3.0, 4.0]");
+
+ for (String subq :
+ Arrays.asList(
+ "{!vectorSimilarity f=vector minReturn=0.8 includeTags=xxx v=$vec}",
+ "{!vectorSimilarity f=vector minReturn=0.8 excludeTags=xxx v=$vec}")) {
+ assertQEx(
+ "vecSim as subquery incompatible with include/exclude localparams",
+ "used as a sub-query does not support",
+ req(common, "q", "*:* OR " + subq),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+ }
+
+ @Test
+ public void vecSimQueryWithFilterQuery_singlePreFilterEquivilence() {
+ final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+ final SolrParams common = params("fl", "id");
+
+ // these requests should be equivalent
+ final String filt = "id:(1 2 7 20)";
+ for (SolrQueryRequest req :
+ Arrays.asList(
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector minReturn=0.8}" + vectorToSearch,
+ "fq",
+ filt),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector preFilter=\""
+ + filt
+ + "\" minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector preFilter=$my_filt minReturn=0.8}" + vectorToSearch,
+ "my_filt",
+ filt))) {
+ assertQ(
+ req,
+ "//result[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='2']",
+ "//result/doc[3]/str[@name='id'][.='7']");
+ }
+ }
+
+ @Test
+ public void vecSimQueryWithFilterQuery_multiPreFilterEquivilence() {
+ final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+ final SolrParams common = params("fl", "id");
+
+ // these requests should be equivalent
+ final String fx = "id:(3 4 9 2 1 )"; // 1 & 10 dropped from intersection
+ final String fy = "id:(3 4 9 2 10)";
+ final String minR = "minReturn=0.8"; // should exclude 9
+ for (SolrQueryRequest req :
+ Arrays.asList(
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector " + minR + "}" + vectorToSearch,
+ "fq",
+ fx,
+ "fq",
+ fy),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector preFilter=\""
+ + fx
+ + "\" preFilter=\""
+ + fy
+ + "\" "
+ + minR
+ + "}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector preFilter=$fx preFilter=$fy "
+ + minR
+ + "}"
+ + vectorToSearch,
+ "fx",
+ fx,
+ "fy",
+ fy),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector preFilter=$multi_filt " + minR + "}" + vectorToSearch,
+ "multi_filt",
+ fx,
+ "multi_filt",
+ fy))) {
+ assertQ(
+ req,
+ "//result[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.='4']",
+ "//result/doc[2]/str[@name='id'][.='2']",
+ "//result/doc[3]/str[@name='id'][.='3']");
+ }
+ }
+
+ @Test
+ public void vecSimQueryWithPreFilter_rejectIncludeExclude() {
+ final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQEx(
+ "vecSim preFilter localparm incompatible with include/exclude localparams",
+ "does not support combining preFilter localparam with either",
+ // shouldn't matter if global fq w/tag even exists, usage is an error
+ req(
+ "q",
+ "{!vectorSimilarity f=vector minReturn=0.8 preFilter='id:1' includeTags=xxx}"
+ + vectorToSearch),
+ SolrException.ErrorCode.BAD_REQUEST);
+ assertQEx(
+ "vecSim preFilter localparm incompatible with include/exclude localparams",
+ "does not support combining preFilter localparam with either",
+ // shouldn't matter if global fq w/tag even exists, usage is an error
+ req(
+ "q",
+ "{!vectorSimilarity f=vector minReturn=0.8 preFilter='id:1' excludeTags=xxx}"
+ + vectorToSearch),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void vecSimQueryWithFilterQuery_preFilterLocalParamOverridesGlobalFilters() {
+ final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ // trivial case: empty preFilter localparam means no pre-filtering
+ assertQ(
+ req(
+ "q", "{!vectorSimilarity f=vector preFilter='' minReturn=0.8}" + vectorToSearch,
+ "fq", "-id:4",
+ "fl", "id"),
+ "//result[@numFound='7']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='2']",
+ "//result/doc[3]/str[@name='id'][.='10']",
+ "//result/doc[4]/str[@name='id'][.='3']");
+
+ // localparam prefiltering, global fqs applied independently
+ assertQ(
+ req(
+ "q",
+ "{!vectorSimilarity f=vector preFilter='id:(3 4 9 2 7 8)' minReturn=0.8}"
+ + vectorToSearch,
+ "fq", "-id:4",
+ "fl", "id"),
+ "//result[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.='2']",
+ "//result/doc[2]/str[@name='id'][.='3']",
+ "//result/doc[3]/str[@name='id'][.='7']");
+ }
+
+ @Test
+ public void vecSimQueryWithFilterQuery_localParamIncludeExcludeTags() {
+ final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+ final SolrParams common =
+ params(
+ "fl", "id",
+ "fq", "{!tag=xx,aa}id:(5 6 7 8 9 10)",
+ "fq", "{!tag=yy,aa}id:(1 2 3 4 5 6 7)");
+
+ // These req's are equivalent: pre-filter everything
+ // So only 7,6,5 are viable for minReturn=0.8
+ for (SolrQueryRequest req :
+ Arrays.asList(
+ // default behavior is all fq's pre-filter,
+ req(common, "q", "{!vectorSimilarity f=vector minReturn=0.8}" + vectorToSearch),
+ // diff ways of explicitly requesting both fq params
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=aa minReturn=0.8}" + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=aa excludeTags='' minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=aa excludeTags=bogus minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=xx includeTags=yy minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=xx,yy,bogus minReturn=0.8}"
+ + vectorToSearch))) {
+ assertQ(
+ req,
+ "//result[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.='7']",
+ "//result/doc[2]/str[@name='id'][.='5']",
+ "//result/doc[3]/str[@name='id'][.='6']");
+ }
+ }
+
+ @Test
+ public void vecSimQueryWithFilterQuery_localParamsDisablesAllPreFiltering() {
+ final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+ final SolrParams common =
+ params(
+ "fl", "id",
+ "fq", "{!tag=xx,aa}id:(11 7 8 9 10)",
+ "fq", "{!tag=yy,aa}id:(1 2 3 4 12 7)");
+
+ // These req's are equivalent: pre-filter nothing
+ // So 1,4,2,10,3,7,5,6 are the minReturn=0.8
+ // Only 7 matches both of the the regular fq params
+ for (SolrQueryRequest req :
+ Arrays.asList(
+ // explicit local empty preFilter
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector preFilter='' minReturn=0.8}" + vectorToSearch),
+ // diff ways of explicitly including none of the global fq params
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags='' minReturn=0.8}" + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=bogus minReturn=0.8}" + vectorToSearch),
+ // diff ways of explicitly excluding all of the global fq params
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector excludeTags=aa minReturn=0.8}" + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=aa excludeTags=aa minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=aa excludeTags=xx,yy minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=xx,yy excludeTags=aa minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector excludeTags=xx,yy minReturn=0.8}" + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector excludeTags=aa minReturn=0.8}" + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector excludeTags=xx excludeTags=yy minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector excludeTags=xx excludeTags=yy,bogus minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector excludeTags=xx,yy,bogus minReturn=0.8}"
+ + vectorToSearch))) {
+ assertQ(req, "//result[@numFound='1']", "//result/doc[1]/str[@name='id'][.='7']");
+ }
+ }
+
+ @Test
+ public void vecSimQueryWithFilterQuery_localParamCombinedIncludeExcludeTags() {
+ final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+ final SolrParams common =
+ params(
+ "fl", "id",
+ "fq", "{!tag=xx,aa}id:(11 7 8 9 10)",
+ "fq", "{!tag=yy,aa}id:(1 2 3 4 12 7)");
+
+ // These req's are equivalent: prefilter only the 'yy' fq
+ // So 1,4,2,3,7 are in the minReturn=0.8
+ // Only 7 matches the regular 'xx' fq param
+ for (SolrQueryRequest req :
+ Arrays.asList(
+ // diff ways of only using the 'yy' filter
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=yy,bogus minReturn=0.8}" + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=yy excludeTags='' minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector excludeTags=xx,bogus minReturn=0.8}" + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=yy excludeTags=xx minReturn=0.8}"
+ + vectorToSearch),
+ req(
+ common,
+ "q",
+ "{!vectorSimilarity f=vector includeTags=aa excludeTags=xx minReturn=0.8}"
+ + vectorToSearch))) {
+ assertQ(req, "//result[@numFound='1']", "//result/doc[1]/str[@name='id'][.='7']");
+ }
+ }
+
+ @Test
+ public void vecSimQueryWithMultiSelectFaceting_excludeTags() {
+ // NOTE: faceting on id is not very realistic,
+ // but it confirms what we care about re:filters w/o needing extra fields.
+ final String facet_xpath = "//lst[@name='facet_fields']/lst[@name='id']/int";
+ final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ final SolrParams common =
+ params(
+ "fl", "id",
+ "indent", "true",
+ "q", "{!vectorSimilarity f=vector minReturn=0.9 excludeTags=facet_click v=$vec}",
+ "vec", vectorToSearch,
+ // mimicing "inStock:true"
+ "fq", "-id:(2 3)",
+ "facet", "true",
+ "facet.mincount", "1",
+ "facet.field", "{!ex=facet_click}id");
+
+ // initial query, with basic pre-filter and facet counts
+ assertQ(
+ req(common),
+ "//result[@numFound='5']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='4']",
+ "//result/doc[3]/str[@name='id'][.='10']",
+ "//result/doc[4]/str[@name='id'][.='7']",
+ "//result/doc[5]/str[@name='id'][.='5']",
+ "*[count(" + facet_xpath + ")=5]",
+ facet_xpath + "[@name='1'][.='1']",
+ facet_xpath + "[@name='4'][.='1']",
+ facet_xpath + "[@name='10'][.='1']",
+ facet_xpath + "[@name='7'][.='1']",
+ facet_xpath + "[@name='5'][.='1']");
+
+ // drill down on a single facet constraint
+ // multi-select means facet counts shouldn't change
+ assertQ(
+ req(common, "fq", "{!tag=facet_click}id:(4)"),
+ "//result[@numFound='1']",
+ "//result/doc[1]/str[@name='id'][.='4']",
+ "*[count(" + facet_xpath + ")=5]",
+ facet_xpath + "[@name='1'][.='1']",
+ facet_xpath + "[@name='4'][.='1']",
+ facet_xpath + "[@name='10'][.='1']",
+ facet_xpath + "[@name='7'][.='1']",
+ facet_xpath + "[@name='5'][.='1']");
+
+ // drill down on an additional facet constraint
+ // multi-select means facet counts shouldn't change
+ assertQ(
+ req(common, "fq", "{!tag=facet_click}id:(4 5)"),
+ "//result[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='4']",
+ "//result/doc[2]/str[@name='id'][.='5']",
+ "*[count(" + facet_xpath + ")=5]",
+ facet_xpath + "[@name='1'][.='1']",
+ facet_xpath + "[@name='4'][.='1']",
+ facet_xpath + "[@name='10'][.='1']",
+ facet_xpath + "[@name='7'][.='1']",
+ facet_xpath + "[@name='5'][.='1']");
+ }
+
+ /**
+ * See {@link org.apache.solr.search.ReRankQParserPlugin.ReRankQueryRescorer#combine(float,
+ * boolean, float)}} for more details.
+ */
+ @Test
+ public void vecSimQueryAsRerank_shouldAddSimilarityFunctionScore() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQ(
+ req(
+ CommonParams.Q,
+ "id:(3 4 9 2)",
+ "rq",
+ "{!rerank reRankQuery=$rqq reRankDocs=4 reRankWeight=1}",
+ "rqq",
+ "{!vectorSimilarity f=vector minReturn=0.8}" + vectorToSearch,
+ "fl",
+ "id"),
+ "//result[@numFound='4']",
+ "//result/doc[1]/str[@name='id'][.='4']",
+ "//result/doc[2]/str[@name='id'][.='2']",
+ "//result/doc[3]/str[@name='id'][.='3']",
+ "//result/doc[4]/str[@name='id'][.='9']");
+ }
+}
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc
index ac96ef827bf..35f5f094c68 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc
@@ -237,14 +237,12 @@ client.add(Arrays.asList(d1, d2));
--
== Query Time
-This is the Apache Solr query approach designed to support dense vector search:
-=== knn Query Parser
-The `knn` k-nearest neighbors query parser allows to find the k-nearest documents to the target vector according to indexed dense vectors in the given field. The set of documents can be Pre-Filtered to reduce the number of vector distance calculations that must be computed, and ensure the best `topK` are returned.
+Apache Solr provides two query parsers that work with dense vector fields, that each support different ways of matching documents based on vector similarity: The `knn` query parser, and the `vectorSimilarity` query parser.
-The score for a retrieved document is the approximate distance to the target vector(defined by the similarityFunction configured at indexing time).
+Both parsers return scores for retrieved documents that are the approximate distance to the target vector (defined by the similarityFunction configured at indexing time) and both support "Pre-Filtering" the document graph to reduce the number of candidate vectors evaluated (with out needing to compute their vector similarity distances).
-It takes the following parameters:
+Common parameters for both query parsers are:
`f`::
+
@@ -255,15 +253,6 @@ s|Required |Default: none
+
The `DenseVectorField` to search in.
-`topK`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: 10
-|===
-+
-How many k-nearest results to return.
-
`preFilter`::
+
[%autowidth,frame=none]
@@ -293,22 +282,73 @@ Indicates that only `fq` filters with the specified `tag` should be considered f
Indicates that `fq` filters with the specified `tag` should be excluded from consideration for implicit Pre-Filtering. Must not be combined with `preFilter`.
-Here's how to run a simple KNN search:
+=== knn Query Parser
+
+The `knn` k-nearest neighbors query parser matches k-nearest documents to the target vector.
+
+In addition to the common parameters described above, it takes the following parameters:
+
+`topK`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: 10
+|===
++
+How many k-nearest results to return.
+
+Here's an example of a simple `knn` search:
[source,text]
?q={!knn f=vector topK=10}[1.0, 2.0, 3.0, 4.0]
The search results retrieved are the k=10 nearest documents to the vector in input `[1.0, 2.0, 3.0, 4.0]`, ranked by the `similarityFunction` configured at indexing time.
+=== vectorSimilarity Query Parser
+
+The `vectorSimilarity` vector similarity query parser matches documents whose similarity with the target vector is a above a minimum threshold.
-==== Explicit KNN Pre-Filtering
+In addition to the common parameters described above, it takes the following parameters:
-The `knn` query parser's `preFilter` parameter can be specified to reduce the number of candidate documents evaluated for the k-nearest distance calculation:
+
+`minReturn`::
++
+[%autowidth,frame=none]
+|===
+s|Required |Default: none
+|===
++
+Minimum similarity threshold of nodes in the graph to be returned as matches
+
+`minTraverse`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: -Infinity
+|===
++
+Minimum similarity of nodes in the graph to continue traversal of their neighbors
+
+Here's an example of a simple `vectorSimilarity` search:
+
+[source,text]
+?q={!vectorSimilarity f=vector minReturn=0.7}[1.0, 2.0, 3.0, 4.0]
+
+The search results retrieved are all documents whose similarity with the input vector `[1.0, 2.0, 3.0, 4.0]` is at least `0.7` based on the `similarityFunction` configured at indexing time
+
+
+=== Graph Pre-Filtering
+
+Pre-Filtering the set of candidate documents considered when walking the graph can be specified either explicitly, or implicitly (based on existing `fq` params) depending on how and when these dense vector query parsers are used.
+
+==== Explicit Pre-Filtering
+
+The `preFilter` parameter can be specified explicitly to reduce the number of candidate documents evaluated for the distance calculation:
[source,text]
-?q={!knn f=vector topK=10 preFilter=inStock:true}[1.0, 2.0, 3.0, 4.0]
+?q={!vectorSimilarity f=vector minReturn=0.7 preFilter=inStock:true}[1.0, 2.0, 3.0, 4.0]
-In the above example, only documents matching the Pre-Filter `inStock:true` will be candidates for consideration when evaluating the k-nearest search against the specified vector.
+In the above example, only documents matching the Pre-Filter `inStock:true` will be candidates for consideration when evaluating the `vectorSimilarity` search against the specified vector.
The `preFilter` parameter may be blank (ex: `preFilter=""`) to indicate that no Pre-Filtering should be performed; or it may be multi-valued -- either through repetition, or via duplicated xref:local-params.adoc#parameter-dereferencing[Parameter References].
@@ -324,22 +364,22 @@ These two examples are equivalent:
&knnPreFilter=inStock:true
----
-==== Implicit KNN Pre-Filtering
+==== Implicit Pre-Filtering
-While the `preFilter` parameter may be explicitly specified on *_any_* usage of the `knn` query parser, the default Pre-Filtering behavior (when no `preFilter` parameter is specified) will vary based on how the `knn` query parser is used:
+While the `preFilter` parameter may be explicitly specified on *_any_* usage of the `knn` or `vectorSimilarity` query parsers, the default Pre-Filtering behavior (when no `preFilter` parameter is specified) will vary based on how the query parser is used:
-* When used as the main `q` param: `fq` filters in the request (that are not xref:common-query-parameters.adoc#cache-local-parameter[Solr Post Filters]) will be combined to form an implicit KNN Pre-Filter.
+* When used as the main `q` param: `fq` filters in the request (that are not xref:common-query-parameters.adoc#cache-local-parameter[Solr Post Filters]) will be combined to form an implicit Graph Pre-Filter.
** This default behavior optimizes the number of vector distance calculations considered, eliminating documents that would eventually be excluded by an `fq` filter anyway.
** `includeTags` and `excludeTags` may be used to limit the set of `fq` filters used in the Pre-Filter.
-* When used as an `fq` param, or as a subquery clause in a larger query: No implicit Pre-Filter is used.
+* When a vector search query parser is used as an `fq` param, or as a subquery clause in a larger query: No implicit Pre-Filter is used.
** `includeTags` and `excludeTags` must not be used in these situations.
-The example request below shows two usages of the `knn` query parser that will get _no_ implicit Pre-Filtering from any of the `fq` parameters, because neither usage is as the main `q` param:
+The example request below shows two usages of vector query parsers that will get _no_ implicit Pre-Filtering from any of the `fq` parameters, because neither usage is as the main `q` param:
[source,text]
----
-?q=(color_str:red OR {!knn f=color_vector topK=10 v="[1.0, 2.0, 3.0, 4.0]"})
+?q=(color_str:red OR {!vectorSimilarity f=color_vector minReturn=0.7 v="[1.0, 2.0, 3.0, 4.0]"})
&fq={!knn f=title_vector topK=10}[9.0, 8.0, 7.0, 6.0]
&fq=inStock:true
----
@@ -363,21 +403,24 @@ If we modify the above request to add tags to the `fq` parameters, we can specif
&fq={!tag=for_knn}inStock:true
----
-In this example, only the `inStock:true` filter will be used for KNN Pre-Filtering to find the the `topK=10` documents, and the `category:AAA` filter will be applied independently; possibly resulting in less then 10 total matches.
+In this example, only the `inStock:true` filter will be used for Pre-Filtering to find the the `topK=10` documents, and the `category:AAA` filter will be applied independently; possibly resulting in less then 10 total matches.
Some use cases where `includeTags` and/or `excludeTags` may be more useful then an explicit `preFilter` parameters:
-* You have some `fq` parameters that are xref:configuration-guide:requesthandlers-searchcomponents.adoc#paramsets-and-useparams[re-used on many requests] (even when you don't use the `knn` parser) that you wish to be used as KNN Pre-Filters when you _do_ use the `knn` query parser.
-* You typically want all `fq` params to be used as KNN Pre-Filters, but when users "drill down" on Facets, you want the `fq` parameters you add to be excluded from the KNN Pre-Filtering so that the result set gets smaller; instead of just computing a new `topK` set.
+* You have some `fq` parameters that are xref:configuration-guide:requesthandlers-searchcomponents.adoc#paramsets-and-useparams[re-used on many requests] (even when you don't use search dense vector fields) that you wish to be used as Pre-Filters when you _do_ search dense vector fields.
+* You typically want all `fq` params to be used as graph Pre-Filters on your `knn` queries, but when users "drill down" on Facets, you want the `fq` parameters you add to be excluded from the Pre-Filtering so that the result set gets smaller; instead of just computing a new `topK` set.
-==== Usage as Re-Ranking Query
-The `knn` query parser can be used to rerank first pass query results:
+=== Usage in Re-Ranking Query
+
+Both dense vector search query parsers can be used to rerank first pass query results:
+
[source,text]
&q=id:(3 4 9 2)&rq={!rerank reRankQuery=$rqq reRankDocs=4 reRankWeight=1}&rqq={!knn f=vector topK=10}[1.0, 2.0, 3.0, 4.0]
+
[IMPORTANT]
====
When using `knn` in re-ranking pay attention to the `topK` parameter.