Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15880: K Nearest Neighbors Search (#476)

Alessandro Benedettic309a8a65d88
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 030baeff36e..13a7417d1ee 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -94,6 +94,8 @@ New Features
* SOLR-15197: Support temporal graph queries with DAY and WEEKDAY windows (Joel Bernstein)
+* SOLR-15880: Introduce support for k nearest neighbors search (Alessandro Benedetti, Elia Porciani)
+
Improvements
----------------------
* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta)
diff --git a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
index eb1223b3a31..8e37ee7eba6 100644
--- a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
@@ -22,12 +22,16 @@ import java.util.Locale;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90Codec;
import org.apache.lucene.codecs.lucene90.Lucene90Codec.Mode;
+import org.apache.lucene.codecs.lucene90.Lucene90HnswVectorsFormat;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.schema.DenseVectorField;
+import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
@@ -114,6 +118,26 @@ public class SchemaCodecFactory extends CodecFactory implements SolrCoreAware {
}
return super.getDocValuesFormatForField(field);
}
+
+ @Override
+ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
+ final SchemaField schemaField = core.getLatestSchema().getFieldOrNull(field);
+ FieldType fieldType = (schemaField == null ? null : schemaField.getType());
+ if (fieldType instanceof DenseVectorField) {
+ DenseVectorField vectorType = (DenseVectorField) fieldType;
+ String knnVectorFormatName = vectorType.getCodecFormat();
+ if (knnVectorFormatName != null) {
+ if (knnVectorFormatName.equals(Lucene90HnswVectorsFormat.class.getSimpleName())) {
+ int maxConn = vectorType.getHnswMaxConn();
+ int beamWidth = vectorType.getHnswBeamWidth();
+ return new Lucene90HnswVectorsFormat(maxConn, beamWidth);
+ } else {
+ return KnnVectorsFormat.forName(knnVectorFormatName);
+ }
+ }
+ }
+ return super.getKnnVectorsFormatForField(field);
+ }
};
}
diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java
new file mode 100644
index 00000000000..b96a042d904
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import org.apache.lucene.codecs.lucene90.Lucene90HnswVectorsFormat;
+import org.apache.lucene.document.KnnVectorField;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.KnnVectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.hnsw.HnswGraph;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.search.QParser;
+import org.apache.solr.uninverting.UninvertingReader;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static java.util.Optional.ofNullable;
+import static org.apache.lucene.codecs.lucene90.Lucene90HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+import static org.apache.lucene.codecs.lucene90.Lucene90HnswVectorsFormat.DEFAULT_MAX_CONN;
+
+/**
+ * Provides a field type to support Lucene's {@link
+ * org.apache.lucene.document.KnnVectorField}.
+ * See {@link org.apache.lucene.search.KnnVectorQuery} for more details.
+ * It supports a fixed cardinality dimension for the vector and a fixed similarity function.
+ * The default similarity is EUCLIDEAN_HNSW (L2).
+ * The default index codec format is specified in the Lucene Codec constructor.
+ * For Lucene 9.0 e.g.
+ * See {@link org.apache.lucene.codecs.lucene90.Lucene90Codec}
+ * Currently only {@link org.apache.lucene.codecs.lucene90.Lucene90HnswVectorsFormat} is supported for
+ * advanced hyper-parameter customisation.
+ * See {@link org.apache.lucene.util.hnsw.HnswGraph} for more details about the implementation.
+ *
+ * <br>
+ * Only {@code Indexed} and {@code Stored} attributes are supported.
+ */
+public class DenseVectorField extends FloatPointField {
+
+ static final String KNN_VECTOR_DIMENSION = "vectorDimension";
+ static final String KNN_SIMILARITY_FUNCTION = "similarityFunction";
+
+ static final String CODEC_FORMAT = "codecFormat";
+ static final String HNSW_MAX_CONNECTIONS = "hnswMaxConnections";
+ static final String HNSW_BEAM_WIDTH = "hnswBeamWidth";
+
+ private int dimension;
+ private VectorSimilarityFunction similarityFunction;
+ private VectorSimilarityFunction DEFAULT_SIMILARITY = VectorSimilarityFunction.EUCLIDEAN;
+
+ private String codecFormat;
+ /**
+ * This parameter is coupled with the {@link Lucene90HnswVectorsFormat} format implementation.
+ * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to
+ * {@link Lucene90HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details.
+ */
+ private int hnswMaxConn;
+ /**
+ * This parameter is coupled with the {@link Lucene90HnswVectorsFormat} format implementation.
+ * The number of candidate neighbors to track while searching the graph for each newly inserted
+ * node. Defaults to to {@link Lucene90HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link
+ * HnswGraph} for details.
+ */
+ private int hnswBeamWidth;
+
+ public DenseVectorField() {
+ super();
+ }
+
+ public DenseVectorField(int dimension) {
+ super();
+ this.dimension = dimension;
+ this.similarityFunction = DEFAULT_SIMILARITY;
+ }
+
+ public DenseVectorField(int dimension, VectorSimilarityFunction similarityFunction) {
+ super();
+ this.dimension = dimension;
+ this.similarityFunction = similarityFunction;
+ }
+
+ @Override
+ public void init(IndexSchema schema, Map<String, String> args) {
+ this.dimension = ofNullable(args.get(KNN_VECTOR_DIMENSION))
+ .map(value -> Integer.parseInt(value))
+ .orElseThrow(() -> new SolrException(SolrException.ErrorCode.SERVER_ERROR, "the vector dimension is a mandatory parameter"));
+ args.remove(KNN_VECTOR_DIMENSION);
+
+ this.similarityFunction = ofNullable(args.get(KNN_SIMILARITY_FUNCTION))
+ .map(value -> VectorSimilarityFunction.valueOf(value.toUpperCase(Locale.ROOT)))
+ .orElse(DEFAULT_SIMILARITY);
+ args.remove(KNN_SIMILARITY_FUNCTION);
+
+ this.codecFormat = args.get(CODEC_FORMAT);
+ args.remove(CODEC_FORMAT);
+
+ this.hnswMaxConn = ofNullable(args.get(HNSW_MAX_CONNECTIONS))
+ .map(value -> Integer.parseInt(value))
+ .orElse(DEFAULT_MAX_CONN);
+ args.remove(HNSW_MAX_CONNECTIONS);
+
+ this.hnswBeamWidth = ofNullable(args.get(HNSW_BEAM_WIDTH))
+ .map(value -> Integer.parseInt(value))
+ .orElse(DEFAULT_BEAM_WIDTH);
+ args.remove(HNSW_BEAM_WIDTH);
+
+ this.properties &= ~MULTIVALUED;
+ this.properties &= ~UNINVERTIBLE;
+
+ super.init(schema, args);
+ }
+
+ public int getDimension() {
+ return dimension;
+ }
+
+ public VectorSimilarityFunction getSimilarityFunction() {
+ return similarityFunction;
+ }
+
+ public String getCodecFormat() {
+ return codecFormat;
+ }
+
+ public Integer getHnswMaxConn() {
+ return hnswMaxConn;
+ }
+
+ public Integer getHnswBeamWidth() {
+ return hnswBeamWidth;
+ }
+
+ @Override
+ public void checkSchemaField(final SchemaField field) throws SolrException {
+ super.checkSchemaField(field);
+ if (field.multiValued()) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ getClass().getSimpleName() + " fields can not be multiValued: " + field.getName());
+ }
+
+ if (field.hasDocValues()) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ getClass().getSimpleName() + " fields can not have docValues: " + field.getName());
+ }
+ }
+
+ public List<IndexableField> createFields(SchemaField field, Object value) {
+ ArrayList<IndexableField> fields = new ArrayList<>();
+ float[] parsedVector;
+ try {
+ parsedVector = parseVector(value);
+ } catch (RuntimeException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while creating field '" + field + "' from value '" + value + "', expected format:'[f1, f2, f3...fn]' e.g. [1.0, 3.4, 5.6]", e);
+ }
+
+ if (field.indexed()) {
+ fields.add(createField(field, parsedVector));
+ }
+ if (field.stored()) {
+ fields.ensureCapacity(parsedVector.length + 1);
+ for (float vectorElement : parsedVector) {
+ fields.add(getStoredField(field, vectorElement));
+ }
+ }
+ return fields;
+ }
+
+ @Override
+ public IndexableField createField(SchemaField field, Object parsedVector) {
+ if (parsedVector == null) return null;
+ float[] typedVector = (float[]) parsedVector;
+ return new KnnVectorField(field.getName(), typedVector, similarityFunction);
+ }
+
+ /**
+ * Index Time Parsing
+ * The inputValue is an ArrayList with a type that depends on the loader used:
+ * - {@link org.apache.solr.handler.loader.XMLLoader}, {@link org.apache.solr.handler.loader.CSVLoader} produces an ArrayList of String
+ * - {@link org.apache.solr.handler.loader.JsonLoader} produces an ArrayList of Double
+ * - {@link org.apache.solr.handler.loader.JavabinLoader} produces an ArrayList of Float
+ *
+ * @param inputValue - An {@link ArrayList} containing the elements of the vector
+ * @return the vector parsed
+ */
+ float[] parseVector(Object inputValue) {
+ if (!(inputValue instanceof List)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "incorrect vector format." +
+ " The expected format is an array :'[f1,f2..f3]' where each element f is a float");
+ }
+ List<?> inputVector = (List<?>) inputValue;
+ if (inputVector.size() != dimension) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "incorrect vector dimension." +
+ " The vector value has size "
+ + inputVector.size() + " while it is expected a vector with size " + dimension);
+ }
+
+ float[] vector = new float[dimension];
+ if (inputVector.get(0) instanceof CharSequence) {
+ for (int i = 0; i < dimension; i++) {
+ try {
+ vector[i] = Float.parseFloat(inputVector.get(i).toString());
+ } catch (NumberFormatException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "incorrect vector element: '" + inputVector.get(i) +
+ "'. The expected format is:'[f1,f2..f3]' where each element f is a float");
+ }
+ }
+ } else if (inputVector.get(0) instanceof Number) {
+ for (int i = 0; i < dimension; i++) {
+ vector[i] = ((Number) inputVector.get(i)).floatValue();
+ }
+ } else {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "incorrect vector format." +
+ " The expected format is an array :'[f1,f2..f3]' where each element f is a float");
+ }
+
+ return vector;
+ }
+
+ @Override
+ public UninvertingReader.Type getUninversionType(SchemaField sf) {
+ return null;
+ }
+
+ @Override
+ public ValueSource getValueSource(SchemaField field, QParser parser) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Function queries are not supported for Dense Vector fields.");
+ }
+
+ public Query getKnnVectorQuery(String fieldName, float[] vectorToSearch, int topK) {
+ return new KnnVectorQuery(fieldName, vectorToSearch, topK);
+ }
+
+ /**
+ * Not Supported. Please use the {!knn} query parser to run K nearest neighbors search queries.
+ */
+ @Override
+ public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Field Queries are not supported for Dense Vector fields. Please use the {!knn} query parser to run K nearest neighbors search queries.");
+ }
+
+ /**
+ * Not Supported
+ */
+ @Override
+ public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Range Queries are not supported for Dense Vector fields. Please use the {!knn} query parser to run K nearest neighbors search queries.");
+ }
+
+ /**
+ * Not Supported
+ */
+ @Override
+ public SortField getSortField(SchemaField field, boolean top) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot sort on a Dense Vector field");
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
index 4bf592ec344..1aa1aa840f4 100644
--- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
@@ -30,6 +30,7 @@ import org.apache.solr.search.join.FiltersQParserPlugin;
import org.apache.solr.search.join.GraphQParserPlugin;
import org.apache.solr.search.join.HashRangeQParserPlugin;
import org.apache.solr.search.mlt.MLTQParserPlugin;
+import org.apache.solr.search.neural.KnnQParserPlugin;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrInfoBean {
@@ -87,6 +88,7 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI
map.put(MinHashQParserPlugin.NAME, new MinHashQParserPlugin());
map.put(HashRangeQParserPlugin.NAME, new HashRangeQParserPlugin());
map.put(RankQParserPlugin.NAME, new RankQParserPlugin());
+ map.put(KnnQParserPlugin.NAME, new KnnQParserPlugin());
standardPlugins = Collections.unmodifiableMap(map);
}
diff --git a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java
new file mode 100644
index 00000000000..fe510776e9c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.neural;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.lucene.search.Query;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.DenseVectorField;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QueryParsing;
+
+public class KnnQParser extends QParser {
+
+ static final String TOP_K = "topK";// retrieve the top K results based on the distance similarity function
+ static final int DEFAULT_TOP_K = 10;
+
+ /**
+ * Constructor for the QParser
+ *
+ * @param qstr The part of the query string specific to this parser
+ * @param localParams The set of parameters that are specific to this QParser. See https://solr.apache.org/guide/local-parameters-in-queries.html
+ * @param params The rest of the {@link SolrParams}
+ * @param req The original {@link SolrQueryRequest}.
+ */
+ public KnnQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ super(qstr, localParams, params, req);
+ }
+
+ @Override
+ public Query parse() {
+ String denseVectorField = localParams.get(QueryParsing.F);
+ String vectorToSearch = localParams.get(QueryParsing.V);
+ int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K);
+
+ if (denseVectorField == null || denseVectorField.isEmpty()) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "the Dense Vector field 'f' is missing");
+ }
+
+ if (vectorToSearch == null || vectorToSearch.isEmpty()) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "the Dense Vector value 'v' to search is missing");
+ }
+
+ SchemaField schemaField = req.getCore().getLatestSchema().getField(denseVectorField);
+ FieldType fieldType = schemaField.getType();
+ if (!(fieldType instanceof DenseVectorField)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "only DenseVectorField is compatible with Knn Query Parser");
+ }
+
+ DenseVectorField denseVectorType = (DenseVectorField) fieldType;
+ float[] parsedVectorToSearch = parseVector(vectorToSearch, denseVectorType.getDimension());
+ return denseVectorType.getKnnVectorQuery(schemaField.getName(), parsedVectorToSearch, topK);
+ }
+
+ /**
+ * Parses a String vector.
+ *
+ * @param value with format: [f1, f2, f3, f4...fn]
+ * @return a float array
+ */
+ static private float[] parseVector(String value, int dimension) {
+ if (!value.startsWith("[") || !value.endsWith("]")) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "incorrect vector format." +
+ " The expected format is:'[f1,f2..f3]' where each element f is a float");
+ }
+
+ String[] elements = StringUtils.split(value.substring(1, value.length() - 1),',');
+ if (elements.length != dimension) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "incorrect vector dimension." +
+ " The vector value has size "
+ + elements.length + " while it is expected a vector with size " + dimension);
+ }
+ float[] vector = new float[dimension];
+ for (int i = 0; i < dimension; i++) {
+ try {
+ vector[i] = Float.parseFloat(elements[i]);
+ } catch (NumberFormatException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "incorrect vector element: '" + elements[i] +
+ "'. The expected format is:'[f1,f2..f3]' where each element f is a float");
+ }
+ }
+ return vector;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/neural/KnnQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/neural/KnnQParserPlugin.java
new file mode 100644
index 00000000000..fb515948918
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/neural/KnnQParserPlugin.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.neural;
+
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QParserPlugin;
+
+/**
+ * A neural query parser to run K-nearest neighbors search on Dense Vector fields.
+ * See Wiki page https://solr.apache.org/guide/neural-search.html
+ */
+public class KnnQParserPlugin extends QParserPlugin {
+ public static final String NAME = "knn";
+
+ @Override
+ public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ return new KnnQParser(qstr, localParams, params, req);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/neural/package-info.java b/solr/core/src/java/org/apache/solr/search/neural/package-info.java
new file mode 100644
index 00000000000..9fb9623314b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/neural/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * APIs and classes for implementing Neural (Dense Retrieval) QueryParsers.
+ */
+package org.apache.solr.search.neural;
+
+
diff --git a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
index 1f47269da38..56bccba7df0 100644
--- a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
+++ b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
@@ -31,6 +31,8 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.schema.CopyField;
+import org.apache.solr.schema.DenseVectorField;
+import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
@@ -152,12 +154,12 @@ public class DocumentBuilder {
String name = field.getName();
SchemaField sfield = schema.getFieldOrNull(name);
boolean used = false;
-
+
// Make sure it has the correct number
- if( sfield!=null && !sfield.multiValued() && field.getValueCount() > 1 ) {
+ if( sfield!=null && !sfield.multiValued() && field.getValueCount() > 1 && !(sfield.getType() instanceof DenseVectorField) ) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
- "ERROR: "+getID(doc, schema)+"multiple values encountered for non multiValued field " +
- sfield.getName() + ": " +field.getValue() );
+ "ERROR: "+getID(doc, schema)+"multiple values encountered for non multiValued field " +
+ sfield.getName() + ": " +field.getValue() );
}
List<CopyField> copyFields = schema.getCopyFieldsList(name);
@@ -166,48 +168,34 @@ public class DocumentBuilder {
// load each field value
boolean hasField = false;
try {
- Iterator<?> it = field.iterator();
- while (it.hasNext()) {
- Object v = it.next();
- if( v == null ) {
- continue;
- }
- hasField = true;
- if (sfield != null) {
- used = true;
- addField(out, sfield, v, forInPlaceUpdate);
- // record the field as having a value
- usedFields.add(sfield.getName());
- }
-
- // Check if we should copy this field value to any other fields.
- // This could happen whether it is explicit or not.
- if (copyFields != null) {
- for (CopyField cf : copyFields) {
- SchemaField destinationField = cf.getDestination();
-
- final boolean destHasValues = usedFields.contains(destinationField.getName());
-
- // check if the copy field is a multivalued or not
- if (!destinationField.multiValued() && destHasValues) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Multiple values encountered for non multiValued copy field " +
- destinationField.getName() + ": " + v);
- }
+ if (sfield != null && sfield.getType() instanceof DenseVectorField) {
+ Object vectorValue = field.getValue();
+ if (vectorValue != null) {
+ hasField = true;
+ used = addOriginalField(vectorValue, sfield, forInPlaceUpdate, out, usedFields);
- used = true;
-
- // Perhaps trim the length of a copy field
- Object val = v;
- if( val instanceof CharSequence && cf.getMaxChars() > 0 ) {
- val = cf.getLimitedValue(val.toString());
- }
+ // Check if we should copy this field value to any other fields.
+ // This could happen whether it is explicit or not.
+ if (copyFields != null) {
+ used |= addCopyFields(vectorValue, sfield.getType(), copyFields, forInPlaceUpdate, uniqueKeyFieldName, out, usedFields);
+ }
+ }
+ } else {
+ Iterator<?> it = field.iterator();
+ while (it.hasNext()) {
+ Object v = it.next();
+ if (v == null) {
+ continue;
+ }
+ hasField = true;
+ if (sfield != null) {
+ used = addOriginalField(v, sfield, forInPlaceUpdate, out, usedFields);
+ }
- // TODO ban copyField populating uniqueKeyField; too problematic to support
- addField(out, destinationField, val,
- destinationField.getName().equals(uniqueKeyFieldName) ? false : forInPlaceUpdate);
- // record the field as having a value
- usedFields.add(destinationField.getName());
+ // Check if we should copy this field value to any other fields.
+ // This could happen whether it is explicit or not.
+ if (copyFields != null) {
+ used |= addCopyFields(v, sfield.getType(), copyFields, forInPlaceUpdate, uniqueKeyFieldName, out, usedFields);
}
}
}
@@ -258,6 +246,49 @@ public class DocumentBuilder {
return out;
}
+ private static boolean addOriginalField( Object originalFieldValue, SchemaField sfield, boolean forInPlaceUpdate, Document out, Set<String> usedFields) {
+ addField(out, sfield, originalFieldValue, forInPlaceUpdate);
+ // record the field as having a value
+ usedFields.add(sfield.getName());
+ return true;
+ }
+
+ private static boolean addCopyFields(Object originalFieldValue, FieldType originalFieldType, List<CopyField> copyFields, boolean forInPlaceUpdate,
+ String uniqueKeyFieldName, Document out, Set<String> usedFields) {
+ boolean used = false;
+ for (CopyField cf : copyFields) {
+ SchemaField destinationField = cf.getDestination();
+
+ final boolean destHasValues = usedFields.contains(destinationField.getName());
+
+ // Dense Vector Fields can only be copied to same field type
+ if (originalFieldType instanceof DenseVectorField && !(destinationField.getType() instanceof DenseVectorField)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "The copy field destination must be a DenseVectorField: " +
+ destinationField.getName());
+ }
+
+ // check if the copy field is a multivalued or not
+ if (!destinationField.multiValued() && destHasValues) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Multiple values encountered for non multiValued copy field " +
+ destinationField.getName() + ": " + originalFieldValue);
+ }
+ // Perhaps trim the length of a copy field
+ if (originalFieldValue instanceof CharSequence && cf.getMaxChars() > 0) {
+ originalFieldValue = cf.getLimitedValue(originalFieldValue.toString());
+ }
+
+ // TODO ban copyField populating uniqueKeyField; too problematic to support
+ addField(out, destinationField, originalFieldValue,
+ destinationField.getName().equals(uniqueKeyFieldName) ? false : forInPlaceUpdate);
+ // record the field as having a originalFieldValue
+ usedFields.add(destinationField.getName());
+ used = true;
+ }
+ return used;
+ }
+
private static SolrException unexpectedNestedDocException(IndexSchema schema, boolean forInPlaceUpdate) {
if (! schema.isUsableForChildDocs()) {
return new SolrException(SolrException.ErrorCode.BAD_REQUEST,
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-dimension-null.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-dimension-null.xml
new file mode 100644
index 00000000000..69e80724bc2
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-dimension-null.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField -->
+
+<schema name="bad-schema-densevector-dimension-null" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" similarityFunction="euclidean"/>
+
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
+
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-dimension.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-dimension.xml
new file mode 100644
index 00000000000..5a2b4a1f8e0
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-dimension.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField -->
+
+<schema name="bad-schema-densevector-dimension" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4.6" similarityFunction="euclidean"/>
+
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" indexed="true" stored="true" multiValued="false" />
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-docvalues.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-docvalues.xml
new file mode 100644
index 00000000000..9e434a5e473
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-docvalues.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField -->
+
+<schema name="bad-schema-densevector-docvalues" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="euclidean" stored="true"/>
+
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" docValues="true"/>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
+
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-multivalued.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-multivalued.xml
new file mode 100644
index 00000000000..915b124892c
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-multivalued.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField -->
+
+<schema name="bad-schema-densevector-multivalued" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="euclidean" stored="true"/>
+
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" multiValued="true"/>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
+
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-similarity.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-similarity.xml
new file mode 100644
index 00000000000..6bd87e45073
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-similarity.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField -->
+
+<schema name="bad-schema-densevector-similarity" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="not_existent"/>
+
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-densevector-codec-hyperparamer.xml b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-codec-hyperparamer.xml
new file mode 100644
index 00000000000..bbd78e480fa
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-codec-hyperparamer.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField -->
+
+<schema name="schema-densevector-codec-hyperparamer" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector_default" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" codecFormat="Lucene90HnswVectorsFormat" hnswMaxConnections="10" hnswBeamWidth="40"/>
+ <fieldType name="knn_vector2" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" codecFormat="Lucene90HnswVectorsFormat" hnswMaxConnections="6" hnswBeamWidth="60"/>
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector_default" type="knn_vector_default" indexed="true" stored="true" />
+ <field name="vector" type="knn_vector" indexed="true" stored="true" />
+ <field name="vector2" type="knn_vector2" indexed="true" stored="true" />
+
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-densevector-similarity-null.xml b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-similarity-null.xml
new file mode 100644
index 00000000000..44850ddd2bb
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-similarity-null.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField -->
+
+<schema name="schema-densevector-similarity-null" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4"/>
+
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
+
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-densevector.xml b/solr/core/src/test-files/solr/collection1/conf/schema-densevector.xml
new file mode 100644
index 00000000000..f03c570cb9f
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-densevector.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField -->
+
+<schema name="schema-densevector" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine"/>
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml
index d6b14bbfcaf..570c19bc561 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml
@@ -50,7 +50,14 @@
<fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
<fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
<fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
-
+
+ <!-- Dense Vector Fields -->
+ <fieldType name="knn_vector_cosine" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine"/>
+ <fieldType name="knn_vector_dot_product" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="dot_product"/>
+ <fieldType name="knn_vector5" class="solr.DenseVectorField" vectorDimension="5" similarityFunction="dot_product"/>
+
+
+
<!-- Field type demonstrating an Analyzer failure -->
<fieldType name="failtype1" class="solr.TextField">
<analyzer type="index">
@@ -697,8 +704,14 @@
<field name="severity" type="severityType" docValues="true" indexed="true" stored="true" multiValued="false"/>
<field name="max_chars" type="string" indexed="false" stored="true"/>
-
-
+
+ <!-- Dense Vector-->
+ <field name="vector" type="knn_vector_cosine" indexed="true" stored="true"/>
+ <field name="vector2" type="knn_vector_dot_product" indexed="true" stored="true"/>
+ <field name="vector3" type="knn_vector_cosine" indexed="true" stored="true"/>
+ <field name="vector4" type="knn_vector_cosine" indexed="true" stored="true"/>
+ <field name="vector5" type="knn_vector5" indexed="true" stored="true"/>
+
<!-- Dynamic field definitions. If a field name is not found, dynamicFields
will be used if the name matches any of the patterns.
RESTRICTION: the glob-like pattern in the name attribute must have
@@ -904,6 +917,9 @@
<copyField source="*_path" dest="*_ancestor"/>
<copyField source="title" dest="max_chars" maxChars="10"/>
+ <copyField source="vector" dest="vector2"/>
+ <copyField source="vector3" dest="vector_f_p"/>
+ <copyField source="vector4" dest="vector5"/>
<!-- example of a custom similarity -->
<similarity class="solr.CustomSimilarityFactory">
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema12.xml b/solr/core/src/test-files/solr/collection1/conf/schema12.xml
index 5e67304be3e..336bfdc6661 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema12.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema12.xml
@@ -50,6 +50,9 @@
<fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
<fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
+ <!-- Dense Vector Fields -->
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine"/>
+
<!-- Field type demonstrating an Analyzer failure -->
<fieldType name="failtype1" class="solr.TextField">
<analyzer type="index">
@@ -663,7 +666,9 @@
<field name="shingle23" type="shingle23" indexed="true" stored="true"/>
-
+ <!-- Dense Vector-->
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+
<!--
DO NOT INCLUDE _version_ in this schema file
We explicitly want tests using this file to *NOT* have this field
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
index c5b90a007f1..7f7c8260ccc 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
@@ -43,6 +43,9 @@
<fieldType name="currency" class="solr.CurrencyField" currencyConfig="currency.xml" multiValued="false"/>
<fieldType name="rank" class="solr.RankField"/>
+ <!-- Dense Vector Fields -->
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine"/>
+
<!-- Field type demonstrating an Analyzer failure -->
<fieldType name="failtype1" class="solr.TextField">
<analyzer type="index">
@@ -559,6 +562,9 @@
<field name="min_hash_analysed" type="text_min_hash" multiValued="false" indexed="true" stored="false"/>
+ <!-- Dense Vector-->
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+
<dynamicField name="*_sI" type="string" indexed="true" stored="false"/>
<dynamicField name="*_sS" type="string" indexed="false" stored="true"/>
<dynamicField name="t_*" type="text" indexed="true" stored="true"/>
diff --git a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
index 4779a0bda5e..a2bfe1cfbb8 100644
--- a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
@@ -1009,4 +1009,13 @@ public class JsonLoaderTest extends SolrTestCaseJ4 {
}
+ @Test
+ public void JSONLoader_denseVector_shouldIndexCorrectly() throws Exception {
+ updateJ(json( "[{'id':'888','vector':[1.8,2.8,3.8,4.8]}]" ),
+ params("commit", "true"));
+
+ assertJQ(req("q","id:888", "fl","vector"),
+ "/response/docs/[0]=={'vector':[1.8,2.8,3.8,4.8]}");
+ }
+
}
diff --git a/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java b/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
index 292ffc9a55a..6e031fb7a71 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
@@ -327,6 +327,24 @@ public class TestCSVLoader extends SolrTestCaseJ4 {
}
+ @Test
+ public void CSVLoader_denseVector_shouldIndexCorrectly() throws Exception {
+ makeFile("id,vector\n"
+ + "999,\"1.3,2.3,3.3,4.3\"\n");
+
+ loadLocal("commit", "true",
+ "f.str_s.map", ":EMPTY",
+ "f.vector.split", "true");
+
+ assertQ(req("q", "id:999", "fl", "vector"), "*[count(//doc)=1]",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[1][.='" + 1.3 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[2][.='" + 2.3 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[3][.='" + 3.3 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[4][.='" + 4.3 + "']"
+ );
+ }
+}
-}
+
+
diff --git a/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java
index 4478379c94c..d8709d0056f 100644
--- a/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java
@@ -197,7 +197,28 @@ public class XmlUpdateRequestHandlerTest extends SolrTestCaseJ4 {
p.assertNoCommandsPending();
}
- private static class MockUpdateRequestProcessor extends UpdateRequestProcessor {
+ @Test
+ public void XMLLoader_denseVector_shouldIndexCorrectly() throws Exception {
+ assertU("<?xml version=\"1.0\" ?>\n" +
+ "<add><doc>" +
+ "<field name=\"id\">777</field>" +
+ "<field name=\"vector\">1.5</field>" +
+ "<field name=\"vector\">2.5</field>" +
+ "<field name=\"vector\">3.5</field>" +
+ "<field name=\"vector\">4.5</field>" +
+ "</doc></add>");
+
+ assertU("<commit/>");
+
+ assertQ(req("q", "id:777", "fl", "vector"), "*[count(//doc)=1]",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[1][.='" + 1.5 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[2][.='" + 2.5 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[3][.='" + 3.5 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[4][.='" + 4.5 + "']"
+ );
+ }
+
+ private static class MockUpdateRequestProcessor extends UpdateRequestProcessor {
private Queue<DeleteUpdateCommand> deleteCommands = new LinkedList<>();
diff --git a/solr/core/src/test/org/apache/solr/handler/loader/JavabinLoaderTest.java b/solr/core/src/test/org/apache/solr/handler/loader/JavabinLoaderTest.java
index 6e988879574..1f3e26f66fd 100644
--- a/solr/core/src/test/org/apache/solr/handler/loader/JavabinLoaderTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/loader/JavabinLoaderTest.java
@@ -19,9 +19,11 @@ package org.apache.solr.handler.loader;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrInputDocument;
@@ -31,6 +33,7 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.processor.BufferingRequestProcessor;
import org.junit.BeforeClass;
+import org.junit.Test;
public class JavabinLoaderTest extends SolrTestCaseJ4 {
@BeforeClass
@@ -88,4 +91,22 @@ public class JavabinLoaderTest extends SolrTestCaseJ4 {
// last doc should have the flag set
assertTrue(mockUpdateProcessor.addCommands.get(batch.size()-1).isLastDocInBatch);
}
+
+ @Test
+ public void javabinLoader_denseVector_shouldIndexCorrectly() throws Exception {
+ SolrInputDocument doc1 = new SolrInputDocument();
+ doc1.addField("id", "555");
+ doc1.addField("vector", Arrays.asList(1.4f, 2.4f, 3.4f, 4.4f));
+
+ EmbeddedSolrServer solrJClient = new EmbeddedSolrServer(h.getCoreContainer(), "collection1", EmbeddedSolrServer.RequestWriterSupplier.JavaBin);
+ solrJClient.add(doc1);
+ solrJClient.commit();
+
+ assertQ(req("q", "id:555", "fl", "vector"), "*[count(//doc)=1]",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[1][.='" + 1.4 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[2][.='" + 2.4 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[3][.='" + 3.4 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[4][.='" + 4.4 + "']"
+ );
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.java
new file mode 100644
index 00000000000..204a2452d05
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.java
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.core.AbstractBadConfigTestBase;
+import org.hamcrest.MatcherAssert;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static org.hamcrest.core.Is.is;
+
+public class DenseVectorFieldTest extends AbstractBadConfigTestBase {
+
+ private DenseVectorField toTest = new DenseVectorField();
+
+ @Test
+ public void fieldTypeDefinition_badVectorDimension_shouldThrowException() throws Exception {
+ assertConfigs("solrconfig-basic.xml", "bad-schema-densevector-dimension.xml",
+ "For input string: \"4.6\"");
+ }
+
+ @Test
+ public void fieldTypeDefinition_nullVectorDimension_shouldThrowException() throws Exception {
+ assertConfigs("solrconfig-basic.xml", "bad-schema-densevector-dimension-null.xml",
+ "the vector dimension is a mandatory parameter");
+ }
+
+ @Test
+ public void fieldTypeDefinition_badSimilarityDistance_shouldThrowException() throws Exception {
+ assertConfigs("solrconfig-basic.xml", "bad-schema-densevector-similarity.xml",
+ "No enum constant org.apache.lucene.index.VectorSimilarityFunction.NOT_EXISTENT");
+ }
+
+ @Test
+ public void fieldDefinition_docValues_shouldThrowException() throws Exception {
+ assertConfigs("solrconfig-basic.xml", "bad-schema-densevector-docvalues.xml",
+ "DenseVectorField fields can not have docValues: vector");
+ }
+
+ @Test
+ public void fieldDefinition_multiValued_shouldThrowException() throws Exception {
+ assertConfigs("solrconfig-basic.xml", "bad-schema-densevector-multivalued.xml",
+ "DenseVectorField fields can not be multiValued: vector");
+ }
+
+ @Test
+ public void fieldTypeDefinition_nullSimilarityDistance_shouldUseDefaultSimilarityEuclidean() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector-similarity-null.xml");
+ IndexSchema schema = h.getCore().getLatestSchema();
+
+ SchemaField vector = schema.getField("vector");
+ assertNotNull(vector);
+
+ DenseVectorField type = (DenseVectorField) vector.getType();
+ MatcherAssert.assertThat(type.getSimilarityFunction(), is(VectorSimilarityFunction.EUCLIDEAN));
+ MatcherAssert.assertThat(type.getDimension(), is(4));
+
+ assertTrue(vector.indexed());
+ assertTrue(vector.stored());
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void fieldDefinition_correctConfiguration_shouldLoadSchemaField() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+ IndexSchema schema = h.getCore().getLatestSchema();
+
+ SchemaField vector = schema.getField("vector");
+ assertNotNull(vector);
+
+ DenseVectorField type = (DenseVectorField) vector.getType();
+ MatcherAssert.assertThat(type.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));
+ MatcherAssert.assertThat(type.getDimension(), is(4));
+
+ assertTrue(vector.indexed());
+ assertTrue(vector.stored());
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void fieldDefinition_advancedCodecHyperParamer_shouldLoadSchemaField() throws Exception {
+ try {
+ initCore("solrconfig_codec.xml", "schema-densevector-codec-hyperparamer.xml");
+ IndexSchema schema = h.getCore().getLatestSchema();
+
+ SchemaField vector = schema.getField("vector");
+ assertNotNull(vector);
+
+ DenseVectorField type1 = (DenseVectorField) vector.getType();
+ MatcherAssert.assertThat(type1.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));
+ MatcherAssert.assertThat(type1.getDimension(), is(4));
+ MatcherAssert.assertThat(type1.getCodecFormat(), is("Lucene90HnswVectorsFormat"));
+ MatcherAssert.assertThat(type1.getHnswMaxConn(), is(10));
+ MatcherAssert.assertThat(type1.getHnswBeamWidth(), is(40));
+
+ SchemaField vector2 = schema.getField("vector2");
+ assertNotNull(vector2);
+
+ DenseVectorField type2 = (DenseVectorField) vector2.getType();
+ MatcherAssert.assertThat(type2.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));
+ MatcherAssert.assertThat(type2.getDimension(), is(4));
+ MatcherAssert.assertThat(type2.getCodecFormat(), is("Lucene90HnswVectorsFormat"));
+ MatcherAssert.assertThat(type2.getHnswMaxConn(), is(6));
+ MatcherAssert.assertThat(type2.getHnswBeamWidth(), is(60));
+
+ SchemaField vectorDefault = schema.getField("vector_default");
+ assertNotNull(vectorDefault);
+
+ DenseVectorField typeDefault = (DenseVectorField) vectorDefault.getType();
+ MatcherAssert.assertThat(typeDefault.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));
+ MatcherAssert.assertThat(typeDefault.getDimension(), is(4));
+ assertNull(typeDefault.getCodecFormat());
+ MatcherAssert.assertThat(typeDefault.getHnswMaxConn(), is(16));
+ MatcherAssert.assertThat(typeDefault.getHnswBeamWidth(), is(100));
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void parseVector_NotAList_shouldThrowException() {
+ RuntimeException thrown = Assert.assertThrows("Single string value should throw an exception", SolrException.class, () -> {
+ toTest.parseVector("string");
+ });
+ MatcherAssert.assertThat(thrown.getMessage(), is("incorrect vector format." +
+ " The expected format is an array :'[f1,f2..f3]' where each element f is a float"));
+
+ thrown = Assert.assertThrows("Single float value should throw an exception", SolrException.class, () -> {
+ toTest.parseVector(1.5f);
+ });
+ MatcherAssert.assertThat(thrown.getMessage(), is("incorrect vector format." +
+ " The expected format is an array :'[f1,f2..f3]' where each element f is a float"));
+ }
+
+ @Test
+ public void parseVector_notNumericList_shouldThrowException() {
+ toTest = new DenseVectorField(3);
+
+ RuntimeException thrown = Assert.assertThrows("Incorrect elements should throw an exception", SolrException.class, () -> {
+ toTest.parseVector(Arrays.asList(new DenseVectorField(3), new DenseVectorField(4), new DenseVectorField(5)));
+ });
+ MatcherAssert.assertThat(thrown.getMessage(), is("incorrect vector format. The expected format is an array :'[f1,f2..f3]' where each element f is a float"));
+ }
+
+ @Test
+ public void parseVector_incorrectVectorDimension_shouldThrowException() {
+ toTest = new DenseVectorField(3);
+
+ RuntimeException thrown = Assert.assertThrows("Incorrect vector dimension should throw an exception", SolrException.class, () -> {
+ toTest.parseVector(Arrays.asList(1.0f, 1.5f));
+ });
+ MatcherAssert.assertThat(thrown.getMessage(), is("incorrect vector dimension. The vector value has size 2 while it is expected a vector with size 3"));
+ }
+
+ @Test
+ public void parseVector_incorrectElement_shouldThrowException() {
+ toTest = new DenseVectorField(3);
+
+ RuntimeException thrown = Assert.assertThrows("Incorrect elements should throw an exception", SolrException.class, () -> {
+ toTest.parseVector(Arrays.asList("1.0f", "string", "string2"));
+ });
+ MatcherAssert.assertThat(thrown.getMessage(), is("incorrect vector element: 'string'. The expected format is:'[f1,f2..f3]' where each element f is a float"));
+ }
+
+ /**
+ * The inputValue is an ArrayList with a type that dipends on the loader used:
+ * - {@link org.apache.solr.handler.loader.XMLLoader}, {@link org.apache.solr.handler.loader.CSVLoader} produces an ArrayList of String
+ */
+ @Test
+ public void parseVector_StringArrayList_shouldParseFloatArray() {
+ toTest = new DenseVectorField(3);
+ float[] expected = new float[]{1.1f, 2.2f, 3.3f};
+
+ MatcherAssert.assertThat(toTest.parseVector(Arrays.asList("1.1", "2.2", "3.3")), is(expected));
+ }
+
+ /**
+ * The inputValue is an ArrayList with a type that dipends on the loader used:
+ * - {@link org.apache.solr.handler.loader.JsonLoader} produces an ArrayList of Double
+ */
+ @Test
+ public void parseVector_DoubleArrayList_shouldParseFloatArray() {
+ toTest = new DenseVectorField(3);
+ float[] expected = new float[]{1.7f, 5.4f, 6.6f};
+
+ MatcherAssert.assertThat(toTest.parseVector(Arrays.asList(1.7d, 5.4d, 6.6d)), is(expected));
+ }
+
+ /**
+ * The inputValue is an ArrayList with a type that dipends on the loader used:
+ * - {@link org.apache.solr.handler.loader.JavabinLoader} produces an ArrayList of Float
+ */
+ @Test
+ public void parseVector_FloatArrayList_shouldParseFloatArray() {
+ toTest = new DenseVectorField(3);
+ float[] expected = new float[]{5.5f, 7.7f, 9.8f};
+
+ MatcherAssert.assertThat(toTest.parseVector(Arrays.asList(5.5f, 7.7f, 9.8f)), is(expected));
+ }
+
+ @Test
+ public void indexing_incorrectVectorFormat_shouldThrowException() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ assertFailedU(adoc("id", "0", "vector", "5.4"));
+ assertFailedU(adoc("id", "0", "vector", "string"));
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void indexing_inconsistentVectorDimension_shouldThrowException() throws Exception {
+ try {
+ //vectorDimension = 4
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ SolrInputDocument toFailDoc1 = new SolrInputDocument();
+ toFailDoc1.addField("id", "0");
+ toFailDoc1.addField("vector", Arrays.asList(1, 2, 3));
+
+ SolrInputDocument toFailDoc2 = new SolrInputDocument();
+ toFailDoc2.addField("id", "0");
+ toFailDoc2.addField("vector", Arrays.asList(1, 2, 3, 4, 5));
+
+ assertFailedU(adoc(toFailDoc1));
+ assertFailedU(adoc(toFailDoc2));
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void indexing_correctDocument_shouldBeIndexed() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ SolrInputDocument correctDoc = new SolrInputDocument();
+ correctDoc.addField("id", "0");
+ correctDoc.addField("vector", Arrays.asList(1, 2, 3, 4));
+
+ assertU(adoc(correctDoc));
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void query_storedField_shouldBeReturnedInResults() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ SolrInputDocument doc1 = new SolrInputDocument();
+ doc1.addField("id", "0");
+ doc1.addField("vector", Arrays.asList(1.1f, 2.1f, 3.1f, 4.1f));
+ assertU(adoc(doc1));
+ assertU(commit());
+
+ assertJQ(req("q","id:0", "fl","vector"),
+ "/response/docs/[0]=={'vector':[1.1,2.1,3.1,4.1]}");
+
+ assertQ(req("q", "id:0", "fl", "vector"), "*[count(//doc)=1]",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[1][.='" + 1.1 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[2][.='" + 2.1 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[3][.='" + 3.1 + "']",
+ "//result/doc[1]/arr[@name=\"vector\"]/float[4][.='" + 4.1 + "']"
+ );
+ } finally {
+ deleteCore();
+ }
+ }
+
+ /**
+ * Not Supported
+ */
+ @Test
+ public void query_rangeSearch_shouldThrowException() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ assertQEx("Running Range queries on a dense vector field should raise an Exception",
+ "Cannot parse 'vector:[[1.0 2.0] TO [1.5 2.5]]'",
+ req("q", "vector:[[1.0 2.0] TO [1.5 2.5]]", "fl", "vector"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ assertQEx("Running Range queries on a dense vector field should raise an Exception",
+ "Range Queries are not supported for Dense Vector fields." +
+ " Please use the {!knn} query parser to run K nearest neighbors search queries.",
+ req("q", "vector:[1 TO 5]", "fl", "vector"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ } finally {
+ deleteCore();
+ }
+ }
+
+ /**
+ * Not Supported
+ */
+ @Test
+ public void query_existenceSearch_shouldThrowException() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ assertQEx("Running Existence queries on a dense vector field should raise an Exception",
+ "Range Queries are not supported for Dense Vector fields." +
+ " Please use the {!knn} query parser to run K nearest neighbors search queries.",
+ req("q", "vector:[* TO *]", "fl", "vector"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ } finally {
+ deleteCore();
+ }
+ }
+
+ /**
+ * Not Supported
+ */
+ @Test
+ public void query_fieldQuery_shouldThrowException() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ assertQEx("Running Field queries on a dense vector field should raise an Exception",
+ "Cannot parse 'vector:[1.0, 2.0, 3.0, 4.0]",
+ req("q", "vector:[1.0, 2.0, 3.0, 4.0]", "fl", "vector"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ assertQEx("Running Field queries on a dense vector field should raise an Exception",
+ "Field Queries are not supported for Dense Vector fields." +
+ " Please use the {!knn} query parser to run K nearest neighbors search queries.",
+ req("q", "vector:\"[1.0, 2.0, 3.0, 4.0]\"", "fl", "vector"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ assertQEx("Running Field queries on a dense vector field should raise an Exception",
+ "Field Queries are not supported for Dense Vector fields." +
+ " Please use the {!knn} query parser to run K nearest neighbors search queries.",
+ req("q", "vector:2.0", "fl", "vector"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ } finally {
+ deleteCore();
+ }
+ }
+
+ /**
+ * Not Supported
+ */
+ @Test
+ public void query_sortByVectorField_shouldThrowException() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ assertQEx("Sort over vectors should raise an Exception",
+ "Cannot sort on a Dense Vector field",
+ req("q", "*:*", "sort", "vector desc"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ } finally {
+ deleteCore();
+ }
+ }
+
+ /**
+ * Not Supported
+ */
+ @Test
+ public void query_functionQueryUsage_shouldThrowException() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ assertQEx("Running Function queries on a dense vector field should raise an Exception",
+ "Function queries are not supported for Dense Vector fields.",
+ req("q", "*:*", "fl", "id,field(vector)"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ } finally {
+ deleteCore();
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
index 95c311ae020..849b220d01a 100644
--- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
+++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryUtils;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
@@ -1149,6 +1150,22 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
}
}
+ public void testQueryKNN() throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "0");
+ doc.addField("vector", Arrays.asList(1, 2, 3, 4));
+ assertU(adoc(doc));
+ assertU(commit());
+
+ try {
+ assertQueryEquals("knn", "{!knn f=vector}[1.0,2.0,3.0,4.0]",
+ "{!knn f=vector v=[1.0,2.0,3.0,4.0]}");
+ } finally {
+ delQ("id:0");
+ assertU(commit());
+ }
+ }
+
/**
* NOTE: defType is not only used to pick the parser, but also to record
diff --git a/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java b/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java
new file mode 100644
index 00000000000..8d1f4d9ca07
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.neural;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.solr.search.neural.KnnQParser.DEFAULT_TOP_K;
+
+public class KnnQParserTest extends SolrTestCaseJ4 {
+ String IDField = "id";
+ String vectorField = "vector";
+ String vectorField2 = "vector2";
+
+ @Before
+ public void prepareIndex() throws Exception {
+ /* vectorDimension="4" similarityFunction="cosine" */
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ List<SolrInputDocument> docsToIndex = this.prepareDocs();
+ for(SolrInputDocument doc:docsToIndex){
+ assertU(adoc(doc));
+ }
+
+ assertU(commit());
+ }
+
+ private List<SolrInputDocument> prepareDocs() {
+ int docsCount = 13;
+ List<SolrInputDocument> docs = new ArrayList<>(docsCount);
+ for (int i = 1; i < docsCount + 1; i++) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField(IDField, i);
+ docs.add(doc);
+ }
+
+ docs.get(0).addField(vectorField, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector1= 1.0
+ docs.get(1).addField(vectorField, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector1= 0.998
+ docs.get(2).addField(vectorField, Arrays.asList(7.5f, 15.5f, 17.5f, 22.5f)); // cosine distance vector1= 0.992
+ docs.get(3).addField(vectorField, Arrays.asList(1.4f, 2.4f, 3.4f, 4.4f)); // cosine distance vector1= 0.999
+ docs.get(4).addField(vectorField, Arrays.asList(30f, 22f, 35f, 20f)); // cosine distance vector1= 0.862
+ docs.get(5).addField(vectorField, Arrays.asList(40f, 1f, 1f, 200f)); // cosine distance vector1= 0.756
+ docs.get(6).addField(vectorField, Arrays.asList(5f, 10f, 20f, 40f)); // cosine distance vector1= 0.970
+ docs.get(7).addField(vectorField, Arrays.asList(120f, 60f, 30f, 15f)); // cosine distance vector1= 0.515
+ docs.get(8).addField(vectorField, Arrays.asList(200f, 50f, 100f, 25f)); // cosine distance vector1= 0.554
+ docs.get(9).addField(vectorField, Arrays.asList(1.8f, 2.5f, 3.7f, 4.9f)); // cosine distance vector1= 0.997
+
+ docs.get(10).addField(vectorField2, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector2= 1
+ docs.get(11).addField(vectorField2, Arrays.asList(7.5f, 15.5f, 17.5f, 22.5f)); // cosine distance vector2= 0.992
+ docs.get(12).addField(vectorField2, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector2= 0.998
+
+ return docs;
+ }
+
+ @After
+ public void cleanUp() throws Exception {
+ clearIndex();
+ deleteCore();
+ }
+
+ @Test
+ public void incorrectTopK_shouldThrowException() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQEx("String topK should throw Exception",
+ "For input string: \"string\"",
+ req(CommonParams.Q, "{!knn f=vector topK=string}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ assertQEx("Double topK should throw Exception",
+ "For input string: \"4.5\"",
+ req(CommonParams.Q, "{!knn f=vector topK=4.5}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void topKMissing_shouldReturnDefaultTopK() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQ(req(CommonParams.Q, "{!knn f=vector}" + vectorToSearch, "fl", "id"),
+ "//result[@numFound='" + DEFAULT_TOP_K + "']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='4']",
+ "//result/doc[3]/str[@name='id'][.='2']",
+ "//result/doc[4]/str[@name='id'][.='10']",
+ "//result/doc[5]/str[@name='id'][.='3']",
+ "//result/doc[6]/str[@name='id'][.='7']",
+ "//result/doc[7]/str[@name='id'][.='5']",
+ "//result/doc[8]/str[@name='id'][.='6']",
+ "//result/doc[9]/str[@name='id'][.='9']",
+ "//result/doc[10]/str[@name='id'][.='8']"
+ );
+ }
+
+ @Test
+ public void topK_shouldReturnOnlyTopKResults() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQ(req(CommonParams.Q, "{!knn f=vector topK=5}" + vectorToSearch, "fl", "id"),
+ "//result[@numFound='5']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='4']",
+ "//result/doc[3]/str[@name='id'][.='2']",
+ "//result/doc[4]/str[@name='id'][.='10']",
+ "//result/doc[5]/str[@name='id'][.='3']"
+ );
+
+ assertQ(req(CommonParams.Q, "{!knn f=vector topK=3}" + vectorToSearch, "fl", "id"),
+ "//result[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='4']",
+ "//result/doc[3]/str[@name='id'][.='2']"
+ );
+ }
+
+ @Test
+ public void incorrectVectorFieldType_shouldThrowException() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQEx("Incorrect vector field type should throw Exception",
+ "only DenseVectorField is compatible with Knn Query Parser",
+ req(CommonParams.Q, "{!knn f=id topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void undefinedVectorField_shouldThrowException() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQEx("Undefined vector field should throw Exception",
+ "undefined field: \"notExistent\"",
+ req(CommonParams.Q, "{!knn f=notExistent topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void missingVectorField_shouldThrowException() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQEx("missing vector field should throw Exception",
+ "the Dense Vector field 'f' is missing",
+ req(CommonParams.Q, "{!knn topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void correctVectorField_shouldSearchOnThatField() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQ(req(CommonParams.Q, "{!knn f=vector2 topK=5}" + vectorToSearch, "fl", "id"),
+ "//result[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.='11']",
+ "//result/doc[2]/str[@name='id'][.='13']",
+ "//result/doc[3]/str[@name='id'][.='12']"
+ );
+ }
+
+ @Test
+ public void missingVectorToSearch_shouldThrowException() {
+ assertQEx("missing vector to search should throw Exception",
+ "the Dense Vector value 'v' to search is missing",
+ req(CommonParams.Q, "{!knn f=vector topK=10}", "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void incorrectVectorToSearchDimension_shouldThrowException() {
+ String vectorToSearch = "[2.0, 4.4, 3.]";
+ assertQEx("missing vector to search should throw Exception",
+ "incorrect vector dimension. The vector value has size 3 while it is expected a vector with size 4",
+ req(CommonParams.Q, "{!knn f=vector topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "[2.0, 4.4,,]";
+ assertQEx("incorrect vector to search should throw Exception",
+ "incorrect vector dimension. The vector value has size 2 while it is expected a vector with size 4",
+ req(CommonParams.Q, "{!knn f=vector topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void incorrectVectorToSearch_shouldThrowException() {
+ String vectorToSearch = "2.0, 4.4, 3.5, 6.4";
+ assertQEx("incorrect vector to search should throw Exception",
+ "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float",
+ req(CommonParams.Q, "{!knn f=vector topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "[2.0, 4.4, 3.5, 6.4";
+ assertQEx("incorrect vector to search should throw Exception",
+ "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float",
+ req(CommonParams.Q, "{!knn f=vector topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "2.0, 4.4, 3.5, 6.4]";
+ assertQEx("incorrect vector to search should throw Exception",
+ "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float",
+ req(CommonParams.Q, "{!knn f=vector topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "[2.0, 4.4, 3.5, stringElement]";
+ assertQEx("incorrect vector to search should throw Exception",
+ "incorrect vector element: ' stringElement'. The expected format is:'[f1,f2..f3]' where each element f is a float",
+ req(CommonParams.Q, "{!knn f=vector topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "[2.0, 4.4, , ]";
+ assertQEx("incorrect vector to search should throw Exception",
+ "incorrect vector element: ' '. The expected format is:'[f1,f2..f3]' where each element f is a float",
+ req(CommonParams.Q, "{!knn f=vector topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void correctQuery_shouldRankBySimilarityFunction() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQ(req(CommonParams.Q, "{!knn f=vector topK=10}" + vectorToSearch, "fl", "id"),
+ "//result[@numFound='10']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='4']",
+ "//result/doc[3]/str[@name='id'][.='2']",
+ "//result/doc[4]/str[@name='id'][.='10']",
+ "//result/doc[5]/str[@name='id'][.='3']",
+ "//result/doc[6]/str[@name='id'][.='7']",
+ "//result/doc[7]/str[@name='id'][.='5']",
+ "//result/doc[8]/str[@name='id'][.='6']",
+ "//result/doc[9]/str[@name='id'][.='9']",
+ "//result/doc[10]/str[@name='id'][.='8']"
+ );
+ }
+
+ @Test
+ public void knnQueryWithFilterQuery_shouldIntersectResults() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQ(req(CommonParams.Q, "{!knn f=vector topK=10}" + vectorToSearch, "fq", "id:(1 2 7 20)", "fl", "id"),
+ "//result[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.='1']",
+ "//result/doc[2]/str[@name='id'][.='2']",
+ "//result/doc[3]/str[@name='id'][.='7']"
+ );
+ /*
+ * This behavior is counter-intuitive.
+ * You would expect the query to apply to only the results filtered by the filter query.
+ * So you ideally would like to see the following ranked list as a result: [4,2,3,9].
+ * To get this please use the knn query parser as a reranker, example in a following test.
+ * This is how filter queries work(it's just intersection of the bitsets coming from each query and filter query):
+ * Ranked List from q=[1,4,2,10] <intersects> Set from fq={3,4,9,2} = [4,2]
+ * */
+ assertQ(req(CommonParams.Q, "{!knn f=vector topK=4}" + vectorToSearch, "fq", "id:(3 4 9 2)", "fl", "id"),
+ "//result[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='4']",
+ "//result/doc[2]/str[@name='id'][.='2']"
+ );
+ /* The ranking is now different as default solr score is used for the main query */
+ assertQ(req(CommonParams.Q, "id:(3 4 9 2)", "fq", "{!knn f=vector topK=4}" + vectorToSearch, "fl", "id"),
+ "//result[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='2']",
+ "//result/doc[2]/str[@name='id'][.='4']"
+ );
+ }
+
+
+ /*
+ * See {@link org.apache.solr.search.ReRankQParserPlugin.ReRankQueryRescorer.combine} for more details.
+ * */
+ @Test
+ public void knnQueryAsRerank_shouldAddSimilarityFunctionScore() {
+ String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]";
+
+ assertQ(req(CommonParams.Q, "id:(3 4 9 2)", "rq", "{!rerank reRankQuery=$rqq reRankDocs=4 reRankWeight=1}",
+ "rqq", "{!knn f=vector topK=4}" + vectorToSearch, "fl", "id"),
+ "//result[@numFound='4']",
+ "//result/doc[1]/str[@name='id'][.='4']",
+ "//result/doc[2]/str[@name='id'][.='2']",
+ "//result/doc[3]/str[@name='id'][.='3']",
+ "//result/doc[4]/str[@name='id'][.='9']"
+ );
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java b/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
index 1b0794b652c..534d213537a 100644
--- a/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
+++ b/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.update;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
@@ -23,7 +24,9 @@ import java.util.stream.StreamSupport;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrDocument;
@@ -33,11 +36,15 @@ import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.util.ByteArrayUtf8CharSequence;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.FieldType;
+import org.hamcrest.MatcherAssert;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.hamcrest.core.Is.is;
+
/**
*
*
@@ -299,4 +306,83 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
assertEquals(truncatedValue, out.get("max_chars"));
}
+ @Test
+ public void denseVector_shouldReturnOneIndexableFieldAndOneStoredFieldPerVectorElement() {
+ SolrCore core = h.getCore();
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "0");
+ doc.addField("vector", Arrays.asList(1.1f, 2.1f, 3.1f, 4.1f));
+
+ Document out = DocumentBuilder.toDocument(doc, core.getLatestSchema());
+
+ //from /solr/core/src/test-files/solr/collection1/conf/schema.xml
+ KnnVectorField expectedIndexableField = new KnnVectorField("vector", new float[]{1.1f, 2.1f, 3.1f, 4.1f}, VectorSimilarityFunction.COSINE);
+
+ MatcherAssert.assertThat(((KnnVectorField)out.getField("vector")).vectorValue(), is(expectedIndexableField.vectorValue()));
+
+ List<IndexableField> storedFields = StreamSupport.stream(out.spliterator(), false)
+ .filter(f -> (f.fieldType().stored() && f.name().equals("vector"))).collect(Collectors.toList());
+ MatcherAssert.assertThat(storedFields.size(), is(4));
+
+ MatcherAssert.assertThat(storedFields.get(0).numericValue(), is(1.1f));
+ MatcherAssert.assertThat(storedFields.get(1).numericValue(), is(2.1f));
+ MatcherAssert.assertThat(storedFields.get(2).numericValue(), is(3.1f));
+ MatcherAssert.assertThat(storedFields.get(3).numericValue(), is(4.1f));
+ }
+
+ @Test
+ public void denseVector_shouldWorkWithCopyFields() {
+ SolrCore core = h.getCore();
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "0");
+ doc.addField("vector", Arrays.asList(1.1f, 2.1f, 3.1f, 4.1f));
+
+ Document out = DocumentBuilder.toDocument(doc, core.getLatestSchema());
+
+ //from /solr/core/src/test-files/solr/collection1/conf/schema.xml
+ KnnVectorField exectedDestination = new KnnVectorField("vector2", new float[]{1.1f, 2.1f, 3.1f, 4.1f}, VectorSimilarityFunction.DOT_PRODUCT);
+
+ MatcherAssert.assertThat(((KnnVectorField)out.getField("vector2")).vectorValue(), is(exectedDestination.vectorValue()));
+
+ List<IndexableField> storedFields = StreamSupport.stream(out.spliterator(), false)
+ .filter(f -> (f.fieldType().stored() && f.name().equals("vector2"))).collect(Collectors.toList());
+ MatcherAssert.assertThat(storedFields.size(), is(4));
+
+ MatcherAssert.assertThat(storedFields.get(0).numericValue(), is(1.1f));
+ MatcherAssert.assertThat(storedFields.get(1).numericValue(), is(2.1f));
+ MatcherAssert.assertThat(storedFields.get(2).numericValue(), is(3.1f));
+ MatcherAssert.assertThat(storedFields.get(3).numericValue(), is(4.1f));
+ }
+
+ @Test
+ public void denseVector_incorrectCopyFieldDestinationType_shouldThrowException() {
+ SolrCore core = h.getCore();
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "0");
+ doc.addField("vector3", Arrays.asList(1.1f, 2.1f, 3.1f, 4.1f));
+
+ RuntimeException thrown = Assert.assertThrows("Incorrect destination field type should raise exception", SolrException.class, () -> {
+ DocumentBuilder.toDocument(doc, core.getLatestSchema());
+ });
+ MatcherAssert.assertThat(thrown.getMessage(), is("ERROR: [doc=0] Error adding field 'vector3'='[1.1, 2.1, 3.1, 4.1]' msg=The copy field destination must be a DenseVectorField: vector_f_p"));
+ }
+
+ @Test
+ public void denseVector_incorrectCopyFieldDestinationDimension_shouldThrowException() {
+ SolrCore core = h.getCore();
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "0");
+ doc.addField("vector4", Arrays.asList(1.1f, 2.1f, 3.1f, 4.1f));
+
+ RuntimeException thrown = Assert.assertThrows("Incorrect destination dimension should raise exception", SolrException.class, () -> {
+ DocumentBuilder.toDocument(doc, core.getLatestSchema());
+ });
+ MatcherAssert.assertThat(thrown.getMessage(), is("ERROR: [doc=0] Error adding field 'vector4'='[1.1, 2.1, 3.1, 4.1]' msg=Error while creating field 'vector5{type=knn_vector5,properties=indexed,stored}' from value '[1.1, 2.1, 3.1, 4.1]', expected format:'[f1, f2, f3...fn]' e.g. [1.0, 3.4, 5.6]"));
+ MatcherAssert.assertThat(thrown.getCause().getCause().getMessage(), is("incorrect vector dimension. The vector value has size 4 while it is expected a vector with size 5"));
+ }
+
}
diff --git a/solr/solr-ref-guide/src/dense-vector-search.adoc b/solr/solr-ref-guide/src/dense-vector-search.adoc
new file mode 100644
index 00000000000..273520fc373
--- /dev/null
+++ b/solr/solr-ref-guide/src/dense-vector-search.adoc
@@ -0,0 +1,314 @@
+= Dense Vector Search
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+The Apache Solr *Dense Vector Search* module adds support for indexing and searching dense numerical vectors.
+
+https://en.wikipedia.org/wiki/Deep_learning[Deep learning] can be used to produce a vector representation of both the query and the documents in a corpus of information.
+
+These neural network-based techniques are usually referred to as neural search, an industry derivation from the academic field of https://www.microsoft.com/en-us/research/uploads/prod/2017/06/fntir2018-neuralir-mitra.pdf[Neural information Retrieval].
+
+== Important Concepts
+
+=== Dense Vector Representation
+A traditional tokenized https://en.wikipedia.org/wiki/Inverted_index[inverted index] can be considered to model text as a "sparse" vector, in which each term in the corpus corresponds to one vector dimension. In such a model, the number of dimensions is generally quite high (corresponding to the term dictionary cardinality), and the vector for any given document contains mostly zeros (hence it is sparse, as only a handful of terms that exist in the overall index will be present in any given document).
+
+Dense vector representation contrasts with term-based sparse vector representation in that it distills approximate semantic meaning into a fixed (and limited) number of dimensions.
+
+The number of dimensions in this approach is generally much lower than the sparse case, and the vector for any given document is dense, as most of its dimensions are populated by non-zero values.
+
+In contrast to the sparse approach (for which tokenizers are used to generate sparse vectors directly from text input) the task of generating vectors must be handled in application logic external to Apache Solr.
+
+There may be cases where it makes sense to directly search data that natively exists as a vector (e.g., scientific data); but in a text search context, it is likely that users will leverage deep learning models such as https://en.wikipedia.org/wiki/BERT_(language_model)[BERT] to encode textual information as dense vectors, supplying the resulting vectors to Apache Solr explicitly at index and query time.
+
+For additional information you can refer to this https://sease.io/2021/12/using-bert-to-improve-search-relevance.html[blog post].
+
+=== Dense Retrieval
+Given a dense vector `v` that models the information need, the easiest approach for providing dense vector retrieval would be to calculate the distance (euclidean, dot product, etc.) between `v` and each vector `d` that represents a document in the corpus of information.
+
+This approach is quite expensive, so many approximate strategies are currently under active research.
+
+The strategy implemented in Apache Lucene and used by Apache Solr is based on Navigable Small-world graph.
+
+It provides efficient approximate nearest neighbor search for high dimensional vectors.
+
+See https://doi.org/10.1016/j.is.2013.10.006[Approximate nearest neighbor algorithm based on navigable small world graphs [2014]] and https://arxiv.org/abs/1603.09320[Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs [2018]] for details.
+
+
+== Index Time
+This is the Apache Solr field type designed to support dense vector search:
+
+=== DenseVectorField
+The dense vector field gives the possibility of indexing and searching dense vectors of float elements.
+
+For example:
+
+`[1.0, 2.5, 3.7, 4.1]`
+
+Here's how `DenseVectorField` should be configured in the schema:
+
+[source,xml]
+<fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine"/>
+<field name="vector" type="knn_vector" indexed="true" stored="true"/>
+
+`vectorDimension`::
++
+[%autowidth,frame=none]
+|===
+s|Required |Default: none
+|===
++
+The dimension of the dense vector to pass in.
++
+Accepted values:
+Any integer < = `1024`.
+
+`similarityFunction`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `euclidean`
+|===
++
+Vector similarity function; used in search to return top K most similar vectors to a target vector.
++
+Accepted values: `euclidean`, `dot_product` or `cosine`.
+
+* `euclidean`: https://en.wikipedia.org/wiki/Euclidean_distance[Euclidean distance]
+* `dot_product`: https://en.wikipedia.org/wiki/Dot_product[Dot product]
+
+[NOTE]
+this similarity is intended as an optimized way to perform cosine similarity. In order to use it, all vectors must be of unit length, including both document and query vectors. Using dot product with vectors that are not unit length can result in errors or poor search results.
+
+* `cosine`: https://en.wikipedia.org/wiki/Cosine_similarity[Cosine similarity]
+
+[NOTE]
+the preferred way to perform cosine similarity is to normalize all vectors to unit length, and instead use DOT_PRODUCT. You should only use this function if you need to preserve the original vectors and cannot normalize them in advance.
+
+To use the following advanced parameters that customise the codec format
+and the hyper-parameter of the HNSW algorithm make sure you set this configuration in `solrconfig.xml`:
+[source,xml]
+<config>
+<codecFactory class="solr.SchemaCodecFactory"/>
+...
+
+Here's how `DenseVectorField` can be configured with the advanced codec hyper-parameters:
+
+[source,xml]
+<fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" codecFormat="Lucene90HnswVectorsFormat" hnswMaxConnections="10" hnswBeamWidth="40"/>
+<field name="vector" type="knn_vector" indexed="true" stored="true"/>
+
+`codecFormat`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `Lucene90HnswVectorsFormat`
+|===
++
+(advanced) Specifies the knn codec implementation to use
++
+
+Accepted values: `Lucene90HnswVectorsFormat`.
+
+Please note that the `codecFormat` accepted values may change in future releases.
+
+
+
+[NOTE]
+Lucene index back-compatibility is only supported for the default codec.
+If you choose to customize the `codecFormat` in your schema, upgrading to a future version of Solr may require you to either switch back to the default codec and optimize your index to rewrite it into the default codec before upgrading, or re-build your entire index from scratch after upgrading.
+
+`hnswMaxConnections`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `16`
+|===
++
+(advanced) This parameter is specific for the `Lucene90HnswVectorsFormat` codec format:
++
+Controls how many of the nearest neighbor candidates are connected to the new node.
++
+It has the same meaning as `M` from the 2018 paper.
++
+Accepted values:
+Any integer.
+
+`hnswBeamWidth`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `100`
+|===
++
+(advanced) This parameter is specific for the `Lucene90HnswVectorsFormat` codec format:
++
+It is the number of nearest neighbor candidates to track while searching the graph for each newly inserted node.
++
+It has the same meaning as `efConstruction` from the 2018 paper.
++
+Accepted values:
+Any integer.
+
+`DenseVectorField` supports the attributes: `indexed`, `stored`.
+
+[NOTE]
+currently multivalue is not supported
+
+Here's how a `DenseVectorField` should be indexed:
+
+[.dynamic-tabs]
+--
+[example.tab-pane#json]
+====
+[.tab-label]*JSON*
+[source,json]
+----
+[{ "id": "1",
+"vector": [1.0, 2.5, 3.7, 4.1]
+},
+{ "id": "2",
+"vector": [1.5, 5.5, 6.7, 65.1]
+}
+]
+----
+====
+
+[example.tab-pane#xml]
+====
+[.tab-label]*XML*
+[source,xml]
+----
+<add>
+<doc>
+<field name="id">1</field>
+<field name="vector">1.0</field>
+<field name="vector">2.5</field>
+<field name="vector">3.7</field>
+<field name="vector">4.1</field>
+</doc>
+<doc>
+<field name="id">2</field>
+<field name="vector">1.5</field>
+<field name="vector">5.5</field>
+<field name="vector">6.7</field>
+<field name="vector">65.1</field>
+</doc>
+</add>
+----
+====
+
+[example.tab-pane#solrj]
+====
+[.tab-label]*SolrJ*
+[source,java,indent=0]
+----
+final SolrClient client = getSolrClient();
+
+final SolrInputDocument d1 = new SolrInputDocument();
+d1.setField("id", "1");
+d1.setField("vector", Arrays.asList(1.0f, 2.5f, 3.7f, 4.1f));
+
+
+final SolrInputDocument d2 = new SolrInputDocument();
+d2.setField("id", "2");
+d2.setField("vector", Arrays.asList(1.5f, 5.5f, 6.7f, 65.1f));
+
+client.add(Arrays.asList(d1, d2));
+----
+====
+--
+
+== Query Time
+This is the Apache Solr query approach designed to support dense vector search:
+
+=== knn Query Parser
+The `knn` k-nearest neighbors query parser allows to find the k-nearest documents to the target vector according to indexed dense vectors in the given field.
+
+The score for a retrieved document is the approximate distance to the target vector(defined by the similarityFunction configured at indexing time).
+
+It takes the following parameters:
+
+`f`::
++
+[%autowidth,frame=none]
+|===
+s|Required |Default: none
+|===
++
+The `DenseVectorField` to search in.
+
+`topK`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: 10
+|===
++
+How many k-nearest results to return.
+
+Here's how to run a KNN search:
+
+[source,text]
+&q={!knn f=vector topK=10}[1.0, 2.0, 3.0, 4.0]
+
+The search results retrieved are the k-nearest to the vector in input `[1.0, 2.0, 3.0, 4.0]`, ranked by the similarityFunction configured at indexing time.
+
+==== Usage with Filter Queries
+The `knn` query parser can be used in filter queries:
+[source,text]
+&q=id:(1 2 3)&fq={!knn f=vector topK=10}[1.0, 2.0, 3.0, 4.0]
+
+The `knn` query parser can be used with filter queries:
+[source,text]
+&q={!knn f=vector topK=10}[1.0, 2.0, 3.0, 4.0]&fq=id:(1 2 3)
+
+[IMPORTANT]
+====
+When using `knn` in these scenarios make sure you have a clear understanding of how filter queries work in Apache Solr:
+
+The Ranked List of document IDs resulting from the main query `q` is intersected with the set of document IDs deriving from each filter query `fq`.
+
+e.g.
+
+Ranked List from `q`=`[ID1, ID4, ID2, ID10]` <intersects> Set from `fq`=`{ID3, ID2, ID9, ID4}` = `[ID4,ID2]`
+====
+
+
+==== Usage as Re-Ranking Query
+The `knn` query parser can be used to rerank first pass query results:
+[source,text]
+&q=id:(3 4 9 2)&rq={!rerank reRankQuery=$rqq reRankDocs=4 reRankWeight=1}&rqq={!knn f=vector topK=10}[1.0, 2.0, 3.0, 4.0]
+
+[IMPORTANT]
+====
+When using `knn` in re-ranking pay attention to the `topK` parameter.
+
+The second pass score(deriving from knn) is calculated only if the document `d` from the first pass is within
+the k-nearest neighbors(*in the whole index*) of the target vector to search.
+
+This means the second pass `knn` is executed on the whole index anyway, which is a current limitation.
+
+The final ranked list of results will have the first pass score(main query `q`) added to the second pass score(the approximated similarityFunction distance to the target vector to search) multiplied by a multiplicative factor(reRankWeight).
+
+Details about using the ReRank Query Parser can be found in the <<query-re-ranking.adoc#,Query Re-Ranking>> section.
+====
+
+== Additional Resources
+
+* Blog: https://sease.io/2022/01/apache-solr-neural-search.html
+* Blog: https://sease.io/2022/01/apache-solr-neural-search-knn-benchmark.html
diff --git a/solr/solr-ref-guide/src/field-types-included-with-solr.adoc b/solr/solr-ref-guide/src/field-types-included-with-solr.adoc
index 74268c0fc4a..6c1463179b5 100644
--- a/solr/solr-ref-guide/src/field-types-included-with-solr.adoc
+++ b/solr/solr-ref-guide/src/field-types-included-with-solr.adoc
@@ -37,6 +37,8 @@ The {solr-javadocs}/core/org/apache/solr/schema/package-summary.html[`org.apache
|DateRangeField |Supports indexing date ranges, to include point in time date instances as well (single-millisecond durations). See the section <<date-formatting-math.adoc#,Date Formatting and Date Math>> for more detail on using this field type. Consider using this field type even if it's just for date instances, particularly when the queries typically fall on UTC year/month/day/hour, etc., boundaries.
+|DenseVectorField |Supports indexing dense vectors of float values. See the section <<dense-vector-search.adoc#,Dense Vector Search>> for more information.
+
|DatePointField |Date field. Represents a point in time with millisecond precision, encoded using a "Dimensional Points" based data structure that allows for very efficient searches for specific values, or ranges of values. See the section <<date-formatting-math.adoc#,Working with Dates>> for more details on the supported syntax. For single valued fields, `docValues="true"` must be used to enable sorting.
|DoublePointField |Double field (64-bit IEEE floating point). This class encodes double values using a "Dimensional Points" based data structure that allows for very efficient searches for specific values, or ranges of values. For single valued fields, `docValues="true"` must be used to enable sorting.
diff --git a/solr/solr-ref-guide/src/other-parsers.adoc b/solr/solr-ref-guide/src/other-parsers.adoc
index f0459ed8bc5..9934d88a2ef 100644
--- a/solr/solr-ref-guide/src/other-parsers.adoc
+++ b/solr/solr-ref-guide/src/other-parsers.adoc
@@ -915,6 +915,14 @@ These parameters would be defined in `solrconfig.xml`, in the `defaults` section
For more information about the possibilities of nested queries, see Yonik Seeley's blog post https://lucidworks.com/2009/03/31/nested-queries-in-solr/[Nested Queries in Solr].
+== Neural Query Parsers
+
+There is currently one Query Parser in Solr to provide Neural Search: `knn`.
+
+KNN stands for k-nearest neighbors.
+
+Details are documented further in the section <<dense-vector-search.adoc#,Dense Vector Search>>.
+
== Payload Query Parsers
diff --git a/solr/solr-ref-guide/src/query-syntax-and-parsers.adoc b/solr/solr-ref-guide/src/query-syntax-and-parsers.adoc
index a46f7e3f4cc..9737524b296 100644
--- a/solr/solr-ref-guide/src/query-syntax-and-parsers.adoc
+++ b/solr/solr-ref-guide/src/query-syntax-and-parsers.adoc
@@ -10,6 +10,7 @@
block-join-query-parser, \
join-query-parser, \
spatial-search, \
+ dense-vector-search, \
other-parsers, \
sql-query, \
query-screen, \
@@ -59,6 +60,7 @@ If you have custom parsing needs, you may want to extend that class to create yo
| <<searching-nested-documents.adoc#,Searching Nested Documents>>: Constructing nested and hierarchical queries.
| <<block-join-query-parser.adoc#,Block Join Query Parser>>: Query parser dedicated to searching nested documents.
| <<join-query-parser.adoc#,Join Query Parser>>: Query parser to facilitate joins.
+| <<dense-vector-search.adoc#,Dense Vector Search>>: Solr's dense retrieval capabilities.
| <<spatial-search.adoc#,Spatial Search>>: Solr's spatial search capabilities.
| <<other-parsers.adoc#,Other Parsers>>: More parsers designed for use in specific situations.
| <<sql-query.adoc#,SQL Query>>: SQL language support for Solr.