Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-17780: Add support for scalar quantized dense vectors (#3385)

Kevin Liang8d6b6d6f9425
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index dee3965495b..218419c6c7c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -19,6 +19,8 @@ New Features
* SOLR-17864: Standarized patterns for system property naming across Solr. i.e enable.packages is now solr.packages.enabled. (janhoy, Eric Pugh)
+* SOLR-17780: Add support for scalar quantized dense vectors (Kevin Liang via Alessandro Benedetti)
+
Improvements
---------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
index a40b9a51161..dfb86bf7886 100644
--- a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
@@ -126,17 +126,14 @@ public class SchemaCodecFactory extends CodecFactory implements SolrCoreAware {
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
final SchemaField schemaField = core.getLatestSchema().getFieldOrNull(field);
FieldType fieldType = (schemaField == null ? null : schemaField.getType());
- if (fieldType instanceof DenseVectorField vectorType) {
- String knnAlgorithm = vectorType.getKnnAlgorithm();
- if (DenseVectorField.HNSW_ALGORITHM.equals(knnAlgorithm)) {
- int maxConn = vectorType.getHnswMaxConn();
- int beamWidth = vectorType.getHnswBeamWidth();
- var delegate = new Lucene99HnswVectorsFormat(maxConn, beamWidth);
- return new SolrDelegatingKnnVectorsFormat(delegate, vectorType.getDimension());
- } else {
+ if (fieldType instanceof DenseVectorField vectorField) {
+ final String knnAlgorithm = vectorField.getKnnAlgorithm();
+ if (!DenseVectorField.HNSW_ALGORITHM.equals(knnAlgorithm)) {
throw new SolrException(
ErrorCode.SERVER_ERROR, knnAlgorithm + " KNN algorithm is not supported");
}
+ return new SolrDelegatingKnnVectorsFormat(
+ vectorField.buildKnnVectorsFormat(), vectorField.getDimension());
}
return super.getKnnVectorsFormatForField(field);
}
diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java
index 773c1e6337d..c355e167a76 100644
--- a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java
+++ b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KnnByteVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
@@ -346,6 +347,10 @@ public class DenseVectorField extends FloatPointField {
}
}
+ public KnnVectorsFormat buildKnnVectorsFormat() {
+ return new Lucene99HnswVectorsFormat(hnswMaxConn, hnswBeamWidth);
+ }
+
@Override
public UninvertingReader.Type getUninversionType(SchemaField sf) {
return null;
diff --git a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java
new file mode 100644
index 00000000000..885d09cea17
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import static java.util.Optional.ofNullable;
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER;
+
+import java.util.Map;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.solr.common.SolrException;
+
+public class ScalarQuantizedDenseVectorField extends DenseVectorField {
+ public static final String BITS_PARAM = "bits"; //
+ public static final String CONFIDENCE_INTERVAL_PARAM = "confidenceInterval";
+ public static final String DYNAMIC_CONFIDENCE_INTERVAL_PARAM = "dynamicConfidenceInterval";
+ public static final String COMPRESS_PARAM =
+ "compress"; // can only be enabled when bits = 4 per Lucene codec spec
+
+ static final int DEFAULT_BITS = 7; // use signed byte as default when unspecified
+ static final Float DEFAULT_CONFIDENCE_INTERVAL = null; // use dimension scaled confidence interval
+
+ /**
+ * Number of bits to use for storage Must be 4 (half-byte) or 7 (signed-byte) per Lucene codec
+ * spec
+ */
+ private int bits;
+
+ /**
+ * Confidence interval to use for scalar quantization Default is calculated as
+ * `1-1/(vector_dimensions + 1)`
+ */
+ private Float confidenceInterval;
+
+ /**
+ * When enabled, in conjunction with 4 bit size, will pair values into single bytes for 50%
+ * reduction in memory usage (comes at the cost of some decode speed penalty)
+ */
+ private boolean compress;
+
+ public ScalarQuantizedDenseVectorField() {
+ super();
+ }
+
+ public ScalarQuantizedDenseVectorField(
+ int dimension,
+ VectorSimilarityFunction similarityFunction,
+ VectorEncoding vectorEncoding,
+ int bits,
+ Float confidenceInterval,
+ boolean compress) {
+ super(dimension, similarityFunction, vectorEncoding);
+ this.bits = bits;
+ this.confidenceInterval = confidenceInterval;
+ this.compress = compress;
+ }
+
+ @Override
+ public void init(IndexSchema schema, Map<String, String> args) {
+ this.bits = ofNullable(args.remove(BITS_PARAM)).map(Integer::parseInt).orElse(DEFAULT_BITS);
+
+ this.compress =
+ ofNullable(args.remove(COMPRESS_PARAM)).map(Boolean::parseBoolean).orElse(false);
+
+ this.confidenceInterval =
+ ofNullable(args.remove(CONFIDENCE_INTERVAL_PARAM))
+ .map(Float::parseFloat)
+ .orElse(DEFAULT_CONFIDENCE_INTERVAL);
+
+ if (ofNullable(args.remove(DYNAMIC_CONFIDENCE_INTERVAL_PARAM))
+ .map(Boolean::parseBoolean)
+ .orElse(false)) {
+ this.confidenceInterval = Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL;
+ }
+
+ super.init(schema, args);
+ }
+
+ @Override
+ public KnnVectorsFormat buildKnnVectorsFormat() {
+ return new Lucene99HnswScalarQuantizedVectorsFormat(
+ getHnswMaxConn(),
+ getHnswBeamWidth(),
+ DEFAULT_NUM_MERGE_WORKER,
+ getBits(),
+ useCompression(),
+ getConfidenceInterval(),
+ null);
+ }
+
+ @Override
+ public void checkSchemaField(final SchemaField field) throws SolrException {
+ super.checkSchemaField(field);
+
+ try {
+ // format does not expose any validation, however by constructing
+ // we will run lucene level argument validation
+ buildKnnVectorsFormat();
+ } catch (IllegalArgumentException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ getClass().getSimpleName() + " " + e.getMessage() + ": " + field.getName(),
+ e);
+ }
+ }
+
+ public int getBits() {
+ return bits;
+ }
+
+ public boolean useCompression() {
+ return compress;
+ }
+
+ public Float getConfidenceInterval() {
+ return confidenceInterval;
+ }
+}
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-bits.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-bits.xml
new file mode 100644
index 00000000000..c0a740a74d7
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-bits.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField types -->
+
+<schema name="bad-schema-densevector-quantized-bits" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />
+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>
+ <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/>
+ <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/>
+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>
+
+ <!-- Scalar Quantized vectors -->
+ <!-- use an invalid bit value -->
+ <fieldType name="knn_vector_scalar_quantized_bits" class="solr.ScalarQuantizedDenseVectorField"
+ vectorDimension="4"
+ similarityFunction="cosine"
+ bits="6"/>
+ <field name="v_scalar_bits" type="knn_vector_scalar_quantized_bits" indexed="true" stored="true" />
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />
+ <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" />
+ <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" />
+
+ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+
+ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />
+ <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>
+ <copyField source="*" dest="_text_"/>
+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+ <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-compress.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-compress.xml
new file mode 100644
index 00000000000..1fd4242390f
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-compress.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField types -->
+
+<schema name="bad-schema-densevector-quantized-compress" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />
+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>
+ <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/>
+ <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/>
+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>
+
+ <!-- Scalar Quantized vectors -->
+ <!-- attempt to use compression with an invalid bit size alignment -->
+ <fieldType name="knn_vector_scalar_quantized_compressed" class="solr.ScalarQuantizedDenseVectorField"
+ vectorDimension="4"
+ similarityFunction="cosine"
+ bits="7"
+ compress="true"/>
+ <field name="v_scalar_compressed" type="knn_vector_scalar_quantized_compressed" indexed="true" stored="true" />
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />
+ <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" />
+ <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" />
+
+
+
+ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+
+ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />
+ <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>
+ <copyField source="*" dest="_text_"/>
+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+ <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-high.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-high.xml
new file mode 100644
index 00000000000..4c6bbbb17aa
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-high.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField types -->
+
+<schema name="bad-schema-densevector-quantized-confidence-interval-high" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />
+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>
+ <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/>
+ <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/>
+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>
+
+ <!-- Scalar Quantized vectors -->
+ <!-- attempt to use compression with an invalid bit size alignment -->
+ <fieldType name="knn_vector_scalar_quantized_ci_high" class="solr.ScalarQuantizedDenseVectorField"
+ vectorDimension="4"
+ similarityFunction="cosine"
+ confidenceInterval="1.5"/>
+ <field name="v_scalar_ci" type="knn_vector_scalar_quantized_ci_high" indexed="true" stored="true" />
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />
+ <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" />
+ <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" />
+
+
+
+ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+
+ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />
+ <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>
+ <copyField source="*" dest="_text_"/>
+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+ <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-low.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-low.xml
new file mode 100644
index 00000000000..9de85797208
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-low.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField types -->
+
+<schema name="bad-schema-densevector-quantized-confidence-interval-low" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />
+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>
+ <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/>
+ <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/>
+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>
+
+ <!-- Scalar Quantized vectors -->
+ <!-- attempt to use compression with an invalid bit size alignment -->
+ <fieldType name="knn_vector_scalar_quantized_ci_low" class="solr.ScalarQuantizedDenseVectorField"
+ vectorDimension="4"
+ similarityFunction="cosine"
+ confidenceInterval="0.8"/>
+ <field name="v_scalar_ci" type="knn_vector_scalar_quantized_ci_low" indexed="true" stored="true" />
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />
+ <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" />
+ <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" />
+
+
+
+ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+
+ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />
+ <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>
+ <copyField source="*" dest="_text_"/>
+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+ <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-densevector-quantized.xml b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-quantized.xml
new file mode 100644
index 00000000000..b2e9b0d27f3
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-quantized.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField types -->
+
+<schema name="schema-densevector-quantized" version="1.0">
+ <fieldType name="string" class="solr.StrField" multiValued="true"/>
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />
+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>
+ <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/>
+ <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/>
+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>
+
+ <!-- Scalar Quantized vectors -->
+ <fieldType name="knn_vector_scalar_quantized_default" class="solr.ScalarQuantizedDenseVectorField"
+ vectorDimension="4"
+ similarityFunction="cosine"
+ />
+ <fieldType name="knn_vector_scalar_quantized_half_byte" class="solr.ScalarQuantizedDenseVectorField"
+ vectorDimension="4"
+ similarityFunction="cosine"
+ bits="4"/>
+ <fieldType name="knn_vector_scalar_quantized_compressed" class="solr.ScalarQuantizedDenseVectorField"
+ vectorDimension="4"
+ similarityFunction="cosine"
+ bits="4"
+ compress="true"/>
+ <fieldType name="knn_vector_scalar_quantized_confidence_interval" class="solr.ScalarQuantizedDenseVectorField"
+ vectorDimension="4"
+ similarityFunction="cosine"
+ confidenceInterval="0.91"/>
+ <fieldType name="knn_vector_scalar_quantized_confidence_dynamic" class="solr.ScalarQuantizedDenseVectorField"
+ vectorDimension="4"
+ similarityFunction="cosine"
+ dynamicConfidenceInterval="true"/>
+
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="vector" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector2" type="knn_vector" indexed="true" stored="true"/>
+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />
+ <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" />
+ <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" />
+
+ <field name="v_scalar_default" type="knn_vector_scalar_quantized_default" indexed="true" stored="true" />
+ <field name="v_scalar_half_byte" type="knn_vector_scalar_quantized_half_byte" indexed="true" stored="true" />
+ <field name="v_scalar_compressed" type="knn_vector_scalar_quantized_compressed" indexed="true" stored="true" />
+ <field name="v_scalar_confidence" type="knn_vector_scalar_quantized_confidence_interval" indexed="true" stored="true" />
+ <field name="v_scalar_dynamic" type="knn_vector_scalar_quantized_confidence_dynamic" indexed="true" stored="true" />
+
+ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+
+ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />
+ <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>
+ <copyField source="*" dest="_text_"/>
+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+ <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java b/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java
index 8f27aa930c9..d8c5b60c821 100644
--- a/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java
+++ b/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java
@@ -48,7 +48,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 {
// TODO: replace with EmbeddedSolrServerTestRule (at a glance, I'm not sure how just yet)
initCore("solrconfig-minimal.xml", "schema-minimal.xml");
- // Some basic assertions that are fundemental to the entire test...
+ // Some basic assertions that are fundamental to the entire test...
assertNull(
"Someone broke test config so it declares a CodecFactory, making this test useless",
@@ -84,7 +84,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 {
* Methods with these names must exist, take a single String arg, and return instances of the same
* concrete class as the corrisponding method in the Lucene default codec
*
- * @see #testPerFieldMethodEquivilence
+ * @see #testPerFieldMethodEquivalence
* @see #testCodecDeclaredMethodCoverage
*/
public static Set<String> PER_FIELD_FORMAT_METHOD_NAMES =
@@ -97,10 +97,10 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 {
*
* <p>The primary purpose of this Set is to list the no-arg versions of per-field methods that are
* already tested via in {@link #PER_FIELD_FORMAT_METHOD_NAMES} (Because the solr "per-field"
- * wrapper wll be different then the Lucene "per-field" default impl of these methods)
+ * wrapper will be different from the Lucene "per-field" default impl of these methods)
*
* <p>The other reason to include methods in this set, is if they already have bespoke tests
- * (perhaps because thye require arugments?) and can't be automatically tested via reflection
+ * (perhaps because they require arguments?) and can't be automatically tested via reflection
*
* @see #testCodecDeclaredMethodCoverage
*/
@@ -114,7 +114,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 {
*
* @see #PER_FIELD_FORMAT_METHOD_NAMES
*/
- public void testPerFieldMethodEquivilence() throws Exception {
+ public void testPerFieldMethodEquivalence() throws Exception {
final String fieldNameShouldNotMatter = TestUtil.randomSimpleString(random(), 64);
final Codec expectedCodec = LUCENE_DEFAULT_CODEC;
@@ -138,7 +138,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 {
* runtime type.
*
* <p>Any method found which requires arguments will cause a failure unless it is in {@link
- * #PER_FIELD_FORMAT_METHOD_NAMES} (which are tested by {@link #testPerFieldMethodEquivilence}) or
+ * #PER_FIELD_FORMAT_METHOD_NAMES} (which are tested by {@link #testPerFieldMethodEquivalence}) or
* {@link #IGNORED_CODEC_DECLARED_METHOD_NAMES} (Exempt from testing, or should have their own
* bespoke test)
*/
diff --git a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java
new file mode 100644
index 00000000000..70a239600f6
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import static org.hamcrest.core.Is.is;
+
+import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.solr.core.AbstractBadConfigTestBase;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBase {
+ @Before
+ public void init() {}
+
+ @Test
+ public void fieldTypeDefinition_invalidBitSize_shouldThrowException() throws Exception {
+ assertConfigs(
+ "solrconfig-basic.xml",
+ "bad-schema-densevector-quantized-bits.xml",
+ "ScalarQuantizedDenseVectorField bits must be one of: 4, 7; bits=6: v_scalar_bits");
+ }
+
+ @Test
+ public void fieldTypeDefinition_improperCompressionUse_shouldThrowException() throws Exception {
+ assertConfigs(
+ "solrconfig-basic.xml",
+ "bad-schema-densevector-quantized-compress.xml",
+ "ScalarQuantizedDenseVectorField compress=true only applies when bits=4: v_scalar_compressed");
+ }
+
+ @Test
+ public void fieldTypeDefinition_confidenceIntervalTooLow_shouldThrowException() throws Exception {
+ assertConfigs(
+ "solrconfig-basic.xml",
+ "bad-schema-densevector-quantized-confidence-interval-low.xml",
+ "ScalarQuantizedDenseVectorField confidenceInterval must be between 0.9 and 1.0 or 0; confidenceInterval=0.8: v_scalar_ci");
+ }
+
+ @Test
+ public void fieldTypeDefinition_confidenceIntervalTooHigh_shouldThrowException()
+ throws Exception {
+ assertConfigs(
+ "solrconfig-basic.xml",
+ "bad-schema-densevector-quantized-confidence-interval-high.xml",
+ "ScalarQuantizedDenseVectorField confidenceInterval must be between 0.9 and 1.0 or 0; confidenceInterval=1.5: v_scalar_ci");
+ }
+
+ @Test
+ public void fieldDefinition_default_shouldLoadSchemaField() throws Exception {
+ try {
+ initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
+
+ IndexSchema schema = h.getCore().getLatestSchema();
+
+ SchemaField defaultVectorField = schema.getField("v_scalar_default");
+ assertNotNull(defaultVectorField);
+
+ ScalarQuantizedDenseVectorField defaultVectorType =
+ (ScalarQuantizedDenseVectorField) defaultVectorField.getType();
+ assertThat(defaultVectorType.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));
+ assertThat(defaultVectorType.getDimension(), is(4));
+ assertThat(defaultVectorType.getKnnAlgorithm(), is("hnsw"));
+ assertThat(defaultVectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));
+ assertThat(
+ defaultVectorType.getConfidenceInterval(),
+ is(ScalarQuantizedDenseVectorField.DEFAULT_CONFIDENCE_INTERVAL));
+ assertThat(defaultVectorType.useCompression(), is(false));
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void fieldDefinition_halfByteSize_shouldLoadSchemaField() throws Exception {
+ try {
+ initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
+
+ IndexSchema schema = h.getCore().getLatestSchema();
+
+ SchemaField vectorField = schema.getField("v_scalar_half_byte");
+ assertNotNull(vectorField);
+
+ ScalarQuantizedDenseVectorField vectorType =
+ (ScalarQuantizedDenseVectorField) vectorField.getType();
+ assertThat(vectorType.getBits(), is(4));
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void fieldDefinition_compressed_shouldLoadSchemaField() throws Exception {
+ try {
+ initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
+
+ IndexSchema schema = h.getCore().getLatestSchema();
+
+ SchemaField vectorField = schema.getField("v_scalar_compressed");
+ assertNotNull(vectorField);
+
+ ScalarQuantizedDenseVectorField vectorType =
+ (ScalarQuantizedDenseVectorField) vectorField.getType();
+ assertThat(vectorType.getBits(), is(4));
+ assertThat(vectorType.useCompression(), is(true));
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void fieldDefinition_customConfidenceInterval_shouldLoadSchemaField() throws Exception {
+ try {
+ initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
+
+ IndexSchema schema = h.getCore().getLatestSchema();
+
+ SchemaField vectorField = schema.getField("v_scalar_confidence");
+ assertNotNull(vectorField);
+
+ ScalarQuantizedDenseVectorField vectorType =
+ (ScalarQuantizedDenseVectorField) vectorField.getType();
+ assertThat(vectorType.getConfidenceInterval(), is(0.91F));
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() throws Exception {
+ try {
+ initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
+
+ IndexSchema schema = h.getCore().getLatestSchema();
+
+ SchemaField vectorField = schema.getField("v_scalar_dynamic");
+ assertNotNull(vectorField);
+
+ ScalarQuantizedDenseVectorField vectorType =
+ (ScalarQuantizedDenseVectorField) vectorField.getType();
+ assertThat(
+ vectorType.getConfidenceInterval(),
+ is(Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL));
+ } finally {
+ deleteCore();
+ }
+ }
+}
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc
index 379e502fe1a..e66501faae3 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc
@@ -240,6 +240,82 @@ client.add(Arrays.asList(d1, d2));
====
======
+=== ScalarQuantizedDenseVectorField
+Because dense vectors can have a costly size, it may be worthwhile to use a technique called "quantization"
+which creates a compressed representation of the original vectors. This allows more of the index to be stored in faster memory
+at the cost of some precision.
+
+This dense vector type uses a conversion that projects a 32 bit float precision feature down to an 8 bit int (or smaller)
+by linearly mapping the float range of each dimension down to evenly sized "buckets" of values that fit into an int. For example:
+with 8 bits we can store up to 256 discrete values, so a float dimension with values from 0.0 to 1.0 may be mapped as
+
+[0.0, 0.0039) => 0, [0.0039, 0.0078) => 1 ... etc
+
+As a specific type of DenseVectorField, this field type supports all the same configurable properties outlined above as well
+as some additional ones.
+
+Here is how a ScalarQuantizedDenseVectorField can be defined in the schema:
+
+[source,xml]
+<fieldType name="scalar_quantized_vector" class="solr.ScalarQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine"/>
+<field name="vector" type="scalar_quantized_vector" indexed="true" stored="true"/>
+
+`bits`::
++
+[%autowidth,frame=none]
+|===
+s|Optional |Default: `7`
+|===
++
+The number of bits to use for each quantized dimension value
++
+Accepted values: 4 (half byte) or 7 (unsigned byte).
+
+`confidenceInterval`::
++
+[%autowidth,frame=none]
+|===
+s|Optional |Default: `dimension-scaled`
+|===
++
+Statistically, outlier values are rarely meaningfully relevant to searches, so to increase the size of each bucket for
+quantization (and therefore information gain) we can scale the quantization intervals to the middle n % of values and place the remaining
+outliers in the outermost intervals.
++
+For example: 0.9 means scale interval sizes to the middle 90% of values
++
+If this param is omitted a default is used; scaled to the number of dimensions according to `1-1/(vector_dimensions + 1)`
++
+Accepted values: `FLOAT32` (within 0.9 and 1.0)
+
+`dynamicConfidenceInterval`::
++
+[%autowidth,frame=none]
+|===
+s|Optional |Default: `false`
+|===
++
+If set to true, enables dynamically determining confidence interval (per dimension) by sampling values each time a merge occurs.
++
+`NOTE: when this is enabled, it will take precedence over any value configured for confidenceInterval`
++
+Accepted values: `BOOLEAN`
+
+`compress`::
++
+[%autowidth,frame=none]
+|===
+s|Optional |Default: `false`
+|===
++
+If set to true, this will further pack multiple dimension values within a one byte alignment. This further decreases the
+quantized vector disk storage size by 50% at some decode penalty. This does not affect the raw vector which is always
+preserved when `stored` is true.
++
+`NOTE: this can only be enabled when bits=4`
++
+Accepted values: `BOOLEAN`
+
== Query Time
Apache Solr provides three query parsers that work with dense vector fields, that each support different ways of matching documents based on vector similarity: The `knn` query parser, the `vectorSimilarity` query parser and the `knn_text_to_vector` query parser.