Apache Solr open-source search software
SOLR-17780: Add support for scalar quantized dense vectors (#3385)
Kevin Liang8d6b6d6f9425
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex dee3965495b..218419c6c7c 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -19,6 +19,8 @@ New Features* SOLR-17864: Standarized patterns for system property naming across Solr. i.e enable.packages is now solr.packages.enabled. (janhoy, Eric Pugh)+* SOLR-17780: Add support for scalar quantized dense vectors (Kevin Liang via Alessandro Benedetti)+Improvements---------------------diff --git a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.javaindex a40b9a51161..dfb86bf7886 100644--- a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java+++ b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java@@ -126,17 +126,14 @@ public class SchemaCodecFactory extends CodecFactory implements SolrCoreAware {public KnnVectorsFormat getKnnVectorsFormatForField(String field) {final SchemaField schemaField = core.getLatestSchema().getFieldOrNull(field);FieldType fieldType = (schemaField == null ? null : schemaField.getType());- if (fieldType instanceof DenseVectorField vectorType) {- String knnAlgorithm = vectorType.getKnnAlgorithm();- if (DenseVectorField.HNSW_ALGORITHM.equals(knnAlgorithm)) {- int maxConn = vectorType.getHnswMaxConn();- int beamWidth = vectorType.getHnswBeamWidth();- var delegate = new Lucene99HnswVectorsFormat(maxConn, beamWidth);- return new SolrDelegatingKnnVectorsFormat(delegate, vectorType.getDimension());- } else {+ if (fieldType instanceof DenseVectorField vectorField) {+ final String knnAlgorithm = vectorField.getKnnAlgorithm();+ if (!DenseVectorField.HNSW_ALGORITHM.equals(knnAlgorithm)) {throw new SolrException(ErrorCode.SERVER_ERROR, knnAlgorithm + " KNN algorithm is not supported");}+ return new SolrDelegatingKnnVectorsFormat(+ vectorField.buildKnnVectorsFormat(), vectorField.getDimension());}return super.getKnnVectorsFormatForField(field);}diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.javaindex 773c1e6337d..c355e167a76 100644--- a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java+++ b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java@@ -26,6 +26,7 @@ import java.util.List;import java.util.Locale;import java.util.Map;import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.document.FieldType;import org.apache.lucene.document.KnnByteVectorField;import org.apache.lucene.document.KnnFloatVectorField;@@ -346,6 +347,10 @@ public class DenseVectorField extends FloatPointField {}}+ public KnnVectorsFormat buildKnnVectorsFormat() {+ return new Lucene99HnswVectorsFormat(hnswMaxConn, hnswBeamWidth);+ }+@Overridepublic UninvertingReader.Type getUninversionType(SchemaField sf) {return null;diff --git a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.javanew file mode 100644index 00000000000..885d09cea17--- /dev/null+++ b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java@@ -0,0 +1,135 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.schema;++import static java.util.Optional.ofNullable;+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER;++import java.util.Map;+import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat;+import org.apache.lucene.index.VectorEncoding;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.solr.common.SolrException;++public class ScalarQuantizedDenseVectorField extends DenseVectorField {+ public static final String BITS_PARAM = "bits"; //+ public static final String CONFIDENCE_INTERVAL_PARAM = "confidenceInterval";+ public static final String DYNAMIC_CONFIDENCE_INTERVAL_PARAM = "dynamicConfidenceInterval";+ public static final String COMPRESS_PARAM =+ "compress"; // can only be enabled when bits = 4 per Lucene codec spec++ static final int DEFAULT_BITS = 7; // use signed byte as default when unspecified+ static final Float DEFAULT_CONFIDENCE_INTERVAL = null; // use dimension scaled confidence interval++ /**+ * Number of bits to use for storage Must be 4 (half-byte) or 7 (signed-byte) per Lucene codec+ * spec+ */+ private int bits;++ /**+ * Confidence interval to use for scalar quantization Default is calculated as+ * `1-1/(vector_dimensions + 1)`+ */+ private Float confidenceInterval;++ /**+ * When enabled, in conjunction with 4 bit size, will pair values into single bytes for 50%+ * reduction in memory usage (comes at the cost of some decode speed penalty)+ */+ private boolean compress;++ public ScalarQuantizedDenseVectorField() {+ super();+ }++ public ScalarQuantizedDenseVectorField(+ int dimension,+ VectorSimilarityFunction similarityFunction,+ VectorEncoding vectorEncoding,+ int bits,+ Float confidenceInterval,+ boolean compress) {+ super(dimension, similarityFunction, vectorEncoding);+ this.bits = bits;+ this.confidenceInterval = confidenceInterval;+ this.compress = compress;+ }++ @Override+ public void init(IndexSchema schema, Map<String, String> args) {+ this.bits = ofNullable(args.remove(BITS_PARAM)).map(Integer::parseInt).orElse(DEFAULT_BITS);++ this.compress =+ ofNullable(args.remove(COMPRESS_PARAM)).map(Boolean::parseBoolean).orElse(false);++ this.confidenceInterval =+ ofNullable(args.remove(CONFIDENCE_INTERVAL_PARAM))+ .map(Float::parseFloat)+ .orElse(DEFAULT_CONFIDENCE_INTERVAL);++ if (ofNullable(args.remove(DYNAMIC_CONFIDENCE_INTERVAL_PARAM))+ .map(Boolean::parseBoolean)+ .orElse(false)) {+ this.confidenceInterval = Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL;+ }++ super.init(schema, args);+ }++ @Override+ public KnnVectorsFormat buildKnnVectorsFormat() {+ return new Lucene99HnswScalarQuantizedVectorsFormat(+ getHnswMaxConn(),+ getHnswBeamWidth(),+ DEFAULT_NUM_MERGE_WORKER,+ getBits(),+ useCompression(),+ getConfidenceInterval(),+ null);+ }++ @Override+ public void checkSchemaField(final SchemaField field) throws SolrException {+ super.checkSchemaField(field);++ try {+ // format does not expose any validation, however by constructing+ // we will run lucene level argument validation+ buildKnnVectorsFormat();+ } catch (IllegalArgumentException e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR,+ getClass().getSimpleName() + " " + e.getMessage() + ": " + field.getName(),+ e);+ }+ }++ public int getBits() {+ return bits;+ }++ public boolean useCompression() {+ return compress;+ }++ public Float getConfidenceInterval() {+ return confidenceInterval;+ }+}diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-bits.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-bits.xmlnew file mode 100644index 00000000000..c0a740a74d7--- /dev/null+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-bits.xml@@ -0,0 +1,64 @@+<?xml version="1.0" ?>+<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<!-- Test schema file for DenseVectorField types -->++<schema name="bad-schema-densevector-quantized-bits" version="1.0">+ <fieldType name="string" class="solr.StrField" multiValued="true"/>+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>+ <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/>+ <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/>+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>++ <!-- Scalar Quantized vectors -->+ <!-- use an invalid bit value -->+ <fieldType name="knn_vector_scalar_quantized_bits" class="solr.ScalarQuantizedDenseVectorField"+ vectorDimension="4"+ similarityFunction="cosine"+ bits="6"/>+ <field name="v_scalar_bits" type="knn_vector_scalar_quantized_bits" indexed="true" stored="true" />++ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />+ <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" />+ <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" />++ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>++ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />+ <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>+ <copyField source="*" dest="_text_"/>+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">+ <analyzer type="index">+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ <analyzer type="query">+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>+ <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ </fieldType>++ <uniqueKey>id</uniqueKey>+</schema>diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-compress.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-compress.xmlnew file mode 100644index 00000000000..1fd4242390f--- /dev/null+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-compress.xml@@ -0,0 +1,67 @@+<?xml version="1.0" ?>+<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<!-- Test schema file for DenseVectorField types -->++<schema name="bad-schema-densevector-quantized-compress" version="1.0">+ <fieldType name="string" class="solr.StrField" multiValued="true"/>+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>+ <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/>+ <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/>+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>++ <!-- Scalar Quantized vectors -->+ <!-- attempt to use compression with an invalid bit size alignment -->+ <fieldType name="knn_vector_scalar_quantized_compressed" class="solr.ScalarQuantizedDenseVectorField"+ vectorDimension="4"+ similarityFunction="cosine"+ bits="7"+ compress="true"/>+ <field name="v_scalar_compressed" type="knn_vector_scalar_quantized_compressed" indexed="true" stored="true" />++ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />+ <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" />+ <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" />++++ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>++ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />+ <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>+ <copyField source="*" dest="_text_"/>+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">+ <analyzer type="index">+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ <analyzer type="query">+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>+ <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ </fieldType>++ <uniqueKey>id</uniqueKey>+</schema>diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-high.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-high.xmlnew file mode 100644index 00000000000..4c6bbbb17aa--- /dev/null+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-high.xml@@ -0,0 +1,66 @@+<?xml version="1.0" ?>+<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<!-- Test schema file for DenseVectorField types -->++<schema name="bad-schema-densevector-quantized-confidence-interval-high" version="1.0">+ <fieldType name="string" class="solr.StrField" multiValued="true"/>+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>+ <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/>+ <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/>+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>++ <!-- Scalar Quantized vectors -->+ <!-- attempt to use compression with an invalid bit size alignment -->+ <fieldType name="knn_vector_scalar_quantized_ci_high" class="solr.ScalarQuantizedDenseVectorField"+ vectorDimension="4"+ similarityFunction="cosine"+ confidenceInterval="1.5"/>+ <field name="v_scalar_ci" type="knn_vector_scalar_quantized_ci_high" indexed="true" stored="true" />++ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />+ <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" />+ <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" />++++ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>++ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />+ <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>+ <copyField source="*" dest="_text_"/>+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">+ <analyzer type="index">+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ <analyzer type="query">+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>+ <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ </fieldType>++ <uniqueKey>id</uniqueKey>+</schema>diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-low.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-low.xmlnew file mode 100644index 00000000000..9de85797208--- /dev/null+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-low.xml@@ -0,0 +1,66 @@+<?xml version="1.0" ?>+<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<!-- Test schema file for DenseVectorField types -->++<schema name="bad-schema-densevector-quantized-confidence-interval-low" version="1.0">+ <fieldType name="string" class="solr.StrField" multiValued="true"/>+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>+ <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/>+ <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/>+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>++ <!-- Scalar Quantized vectors -->+ <!-- attempt to use compression with an invalid bit size alignment -->+ <fieldType name="knn_vector_scalar_quantized_ci_low" class="solr.ScalarQuantizedDenseVectorField"+ vectorDimension="4"+ similarityFunction="cosine"+ confidenceInterval="0.8"/>+ <field name="v_scalar_ci" type="knn_vector_scalar_quantized_ci_low" indexed="true" stored="true" />++ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />+ <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" />+ <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" />++++ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>++ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />+ <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>+ <copyField source="*" dest="_text_"/>+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">+ <analyzer type="index">+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ <analyzer type="query">+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>+ <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ </fieldType>++ <uniqueKey>id</uniqueKey>+</schema>diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-densevector-quantized.xml b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-quantized.xmlnew file mode 100644index 00000000000..b2e9b0d27f3--- /dev/null+++ b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-quantized.xml@@ -0,0 +1,86 @@+<?xml version="1.0" ?>+<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<!-- Test schema file for DenseVectorField types -->++<schema name="schema-densevector-quantized" version="1.0">+ <fieldType name="string" class="solr.StrField" multiValued="true"/>+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>+ <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/>+ <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/>+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>++ <!-- Scalar Quantized vectors -->+ <fieldType name="knn_vector_scalar_quantized_default" class="solr.ScalarQuantizedDenseVectorField"+ vectorDimension="4"+ similarityFunction="cosine"+ />+ <fieldType name="knn_vector_scalar_quantized_half_byte" class="solr.ScalarQuantizedDenseVectorField"+ vectorDimension="4"+ similarityFunction="cosine"+ bits="4"/>+ <fieldType name="knn_vector_scalar_quantized_compressed" class="solr.ScalarQuantizedDenseVectorField"+ vectorDimension="4"+ similarityFunction="cosine"+ bits="4"+ compress="true"/>+ <fieldType name="knn_vector_scalar_quantized_confidence_interval" class="solr.ScalarQuantizedDenseVectorField"+ vectorDimension="4"+ similarityFunction="cosine"+ confidenceInterval="0.91"/>+ <fieldType name="knn_vector_scalar_quantized_confidence_dynamic" class="solr.ScalarQuantizedDenseVectorField"+ vectorDimension="4"+ similarityFunction="cosine"+ dynamicConfidenceInterval="true"/>+++ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />+ <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" />+ <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" />++ <field name="v_scalar_default" type="knn_vector_scalar_quantized_default" indexed="true" stored="true" />+ <field name="v_scalar_half_byte" type="knn_vector_scalar_quantized_half_byte" indexed="true" stored="true" />+ <field name="v_scalar_compressed" type="knn_vector_scalar_quantized_compressed" indexed="true" stored="true" />+ <field name="v_scalar_confidence" type="knn_vector_scalar_quantized_confidence_interval" indexed="true" stored="true" />+ <field name="v_scalar_dynamic" type="knn_vector_scalar_quantized_confidence_dynamic" indexed="true" stored="true" />++ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>++ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />+ <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>+ <copyField source="*" dest="_text_"/>+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">+ <analyzer type="index">+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ <analyzer type="query">+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>+ <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ </fieldType>++ <uniqueKey>id</uniqueKey>+</schema>diff --git a/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java b/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.javaindex 8f27aa930c9..d8c5b60c821 100644--- a/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java+++ b/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java@@ -48,7 +48,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 {// TODO: replace with EmbeddedSolrServerTestRule (at a glance, I'm not sure how just yet)initCore("solrconfig-minimal.xml", "schema-minimal.xml");- // Some basic assertions that are fundemental to the entire test...+ // Some basic assertions that are fundamental to the entire test...assertNull("Someone broke test config so it declares a CodecFactory, making this test useless",@@ -84,7 +84,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 {* Methods with these names must exist, take a single String arg, and return instances of the same* concrete class as the corrisponding method in the Lucene default codec*- * @see #testPerFieldMethodEquivilence+ * @see #testPerFieldMethodEquivalence* @see #testCodecDeclaredMethodCoverage*/public static Set<String> PER_FIELD_FORMAT_METHOD_NAMES =@@ -97,10 +97,10 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 {** <p>The primary purpose of this Set is to list the no-arg versions of per-field methods that are* already tested via in {@link #PER_FIELD_FORMAT_METHOD_NAMES} (Because the solr "per-field"- * wrapper wll be different then the Lucene "per-field" default impl of these methods)+ * wrapper will be different from the Lucene "per-field" default impl of these methods)** <p>The other reason to include methods in this set, is if they already have bespoke tests- * (perhaps because thye require arugments?) and can't be automatically tested via reflection+ * (perhaps because they require arguments?) and can't be automatically tested via reflection** @see #testCodecDeclaredMethodCoverage*/@@ -114,7 +114,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 {** @see #PER_FIELD_FORMAT_METHOD_NAMES*/- public void testPerFieldMethodEquivilence() throws Exception {+ public void testPerFieldMethodEquivalence() throws Exception {final String fieldNameShouldNotMatter = TestUtil.randomSimpleString(random(), 64);final Codec expectedCodec = LUCENE_DEFAULT_CODEC;@@ -138,7 +138,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 {* runtime type.** <p>Any method found which requires arguments will cause a failure unless it is in {@link- * #PER_FIELD_FORMAT_METHOD_NAMES} (which are tested by {@link #testPerFieldMethodEquivilence}) or+ * #PER_FIELD_FORMAT_METHOD_NAMES} (which are tested by {@link #testPerFieldMethodEquivalence}) or* {@link #IGNORED_CODEC_DECLARED_METHOD_NAMES} (Exempt from testing, or should have their own* bespoke test)*/diff --git a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.javanew file mode 100644index 00000000000..70a239600f6--- /dev/null+++ b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java@@ -0,0 +1,163 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.schema;++import static org.hamcrest.core.Is.is;++import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.solr.core.AbstractBadConfigTestBase;+import org.junit.Before;+import org.junit.Test;++public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBase {+ @Before+ public void init() {}++ @Test+ public void fieldTypeDefinition_invalidBitSize_shouldThrowException() throws Exception {+ assertConfigs(+ "solrconfig-basic.xml",+ "bad-schema-densevector-quantized-bits.xml",+ "ScalarQuantizedDenseVectorField bits must be one of: 4, 7; bits=6: v_scalar_bits");+ }++ @Test+ public void fieldTypeDefinition_improperCompressionUse_shouldThrowException() throws Exception {+ assertConfigs(+ "solrconfig-basic.xml",+ "bad-schema-densevector-quantized-compress.xml",+ "ScalarQuantizedDenseVectorField compress=true only applies when bits=4: v_scalar_compressed");+ }++ @Test+ public void fieldTypeDefinition_confidenceIntervalTooLow_shouldThrowException() throws Exception {+ assertConfigs(+ "solrconfig-basic.xml",+ "bad-schema-densevector-quantized-confidence-interval-low.xml",+ "ScalarQuantizedDenseVectorField confidenceInterval must be between 0.9 and 1.0 or 0; confidenceInterval=0.8: v_scalar_ci");+ }++ @Test+ public void fieldTypeDefinition_confidenceIntervalTooHigh_shouldThrowException()+ throws Exception {+ assertConfigs(+ "solrconfig-basic.xml",+ "bad-schema-densevector-quantized-confidence-interval-high.xml",+ "ScalarQuantizedDenseVectorField confidenceInterval must be between 0.9 and 1.0 or 0; confidenceInterval=1.5: v_scalar_ci");+ }++ @Test+ public void fieldDefinition_default_shouldLoadSchemaField() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");++ IndexSchema schema = h.getCore().getLatestSchema();++ SchemaField defaultVectorField = schema.getField("v_scalar_default");+ assertNotNull(defaultVectorField);++ ScalarQuantizedDenseVectorField defaultVectorType =+ (ScalarQuantizedDenseVectorField) defaultVectorField.getType();+ assertThat(defaultVectorType.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));+ assertThat(defaultVectorType.getDimension(), is(4));+ assertThat(defaultVectorType.getKnnAlgorithm(), is("hnsw"));+ assertThat(defaultVectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));+ assertThat(+ defaultVectorType.getConfidenceInterval(),+ is(ScalarQuantizedDenseVectorField.DEFAULT_CONFIDENCE_INTERVAL));+ assertThat(defaultVectorType.useCompression(), is(false));+ } finally {+ deleteCore();+ }+ }++ @Test+ public void fieldDefinition_halfByteSize_shouldLoadSchemaField() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");++ IndexSchema schema = h.getCore().getLatestSchema();++ SchemaField vectorField = schema.getField("v_scalar_half_byte");+ assertNotNull(vectorField);++ ScalarQuantizedDenseVectorField vectorType =+ (ScalarQuantizedDenseVectorField) vectorField.getType();+ assertThat(vectorType.getBits(), is(4));+ } finally {+ deleteCore();+ }+ }++ @Test+ public void fieldDefinition_compressed_shouldLoadSchemaField() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");++ IndexSchema schema = h.getCore().getLatestSchema();++ SchemaField vectorField = schema.getField("v_scalar_compressed");+ assertNotNull(vectorField);++ ScalarQuantizedDenseVectorField vectorType =+ (ScalarQuantizedDenseVectorField) vectorField.getType();+ assertThat(vectorType.getBits(), is(4));+ assertThat(vectorType.useCompression(), is(true));+ } finally {+ deleteCore();+ }+ }++ @Test+ public void fieldDefinition_customConfidenceInterval_shouldLoadSchemaField() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");++ IndexSchema schema = h.getCore().getLatestSchema();++ SchemaField vectorField = schema.getField("v_scalar_confidence");+ assertNotNull(vectorField);++ ScalarQuantizedDenseVectorField vectorType =+ (ScalarQuantizedDenseVectorField) vectorField.getType();+ assertThat(vectorType.getConfidenceInterval(), is(0.91F));+ } finally {+ deleteCore();+ }+ }++ @Test+ public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");++ IndexSchema schema = h.getCore().getLatestSchema();++ SchemaField vectorField = schema.getField("v_scalar_dynamic");+ assertNotNull(vectorField);++ ScalarQuantizedDenseVectorField vectorType =+ (ScalarQuantizedDenseVectorField) vectorField.getType();+ assertThat(+ vectorType.getConfidenceInterval(),+ is(Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL));+ } finally {+ deleteCore();+ }+ }+}diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adocindex 379e502fe1a..e66501faae3 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc@@ -240,6 +240,82 @@ client.add(Arrays.asList(d1, d2));==========+=== ScalarQuantizedDenseVectorField+Because dense vectors can have a costly size, it may be worthwhile to use a technique called "quantization"+which creates a compressed representation of the original vectors. This allows more of the index to be stored in faster memory+at the cost of some precision.++This dense vector type uses a conversion that projects a 32 bit float precision feature down to an 8 bit int (or smaller)+by linearly mapping the float range of each dimension down to evenly sized "buckets" of values that fit into an int. For example:+with 8 bits we can store up to 256 discrete values, so a float dimension with values from 0.0 to 1.0 may be mapped as++[0.0, 0.0039) => 0, [0.0039, 0.0078) => 1 ... etc++As a specific type of DenseVectorField, this field type supports all the same configurable properties outlined above as well+as some additional ones.++Here is how a ScalarQuantizedDenseVectorField can be defined in the schema:++[source,xml]+<fieldType name="scalar_quantized_vector" class="solr.ScalarQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine"/>+<field name="vector" type="scalar_quantized_vector" indexed="true" stored="true"/>++`bits`::+++[%autowidth,frame=none]+|===+s|Optional |Default: `7`+|===+++The number of bits to use for each quantized dimension value+++Accepted values: 4 (half byte) or 7 (unsigned byte).++`confidenceInterval`::+++[%autowidth,frame=none]+|===+s|Optional |Default: `dimension-scaled`+|===+++Statistically, outlier values are rarely meaningfully relevant to searches, so to increase the size of each bucket for+quantization (and therefore information gain) we can scale the quantization intervals to the middle n % of values and place the remaining+outliers in the outermost intervals.+++For example: 0.9 means scale interval sizes to the middle 90% of values+++If this param is omitted a default is used; scaled to the number of dimensions according to `1-1/(vector_dimensions + 1)`+++Accepted values: `FLOAT32` (within 0.9 and 1.0)++`dynamicConfidenceInterval`::+++[%autowidth,frame=none]+|===+s|Optional |Default: `false`+|===+++If set to true, enables dynamically determining confidence interval (per dimension) by sampling values each time a merge occurs.+++`NOTE: when this is enabled, it will take precedence over any value configured for confidenceInterval`+++Accepted values: `BOOLEAN`++`compress`::+++[%autowidth,frame=none]+|===+s|Optional |Default: `false`+|===+++If set to true, this will further pack multiple dimension values within a one byte alignment. This further decreases the+quantized vector disk storage size by 50% at some decode penalty. This does not affect the raw vector which is always+preserved when `stored` is true.+++`NOTE: this can only be enabled when bits=4`+++Accepted values: `BOOLEAN`+== Query TimeApache Solr provides three query parsers that work with dense vector fields, that each support different ways of matching documents based on vector similarity: The `knn` query parser, the `vectorSimilarity` query parser and the `knn_text_to_vector` query parser.