Apache Solr open-source search software
SOLR-18376: remove ScalarQuantizedDenseVectorField compress and confidenceInterval (#4756)
Serhiy Bzhezytskyyf08f90999d21
diff --git a/changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.yml b/changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.ymlnew file mode 100644index 00000000000..ffd10e67b13--- /dev/null+++ b/changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.yml@@ -0,0 +1,8 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: Remove the deprecated ScalarQuantizedDenseVectorField compress and confidenceInterval fields and their accessors useCompression and getConfidenceInterval, since Solr 10.1 no longer used by the underlying Lucene codec; the compress, confidenceInterval and dynamicConfidenceInterval schema params are still accepted and ignored with a deprecation warning, so an existing schema that sets them keeps loading. The public four-argument constructor loses its two corresponding parameters and now takes dimension, similarityFunction, vectorEncoding and bits.+type: removed+authors:+ - name: Serhiy Bzhezytskyy+links:+ - name: SOLR-18376+ url: https://issues.apache.org/jira/browse/SOLR-18376diff --git a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.javaindex 26a33be48a2..66f0387505d 100644--- a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java+++ b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java@@ -18,7 +18,6 @@ package org.apache.solr.schema;import static java.util.Optional.ofNullable;-import com.google.common.annotations.VisibleForTesting;import java.util.Map;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;@@ -36,7 +35,6 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {"compress"; // can only be enabled when bits = 4 per Lucene codec specstatic final int DEFAULT_BITS = 7; // use signed byte as default when unspecified- static final Float DEFAULT_CONFIDENCE_INTERVAL = null; // use dimension scaled confidence interval/*** Number of bits to use for storage Must be 4 (half-byte) or 7 (signed-byte) per Lucene codec@@ -44,24 +42,6 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {*/private int bits;- /**- * Confidence interval to use for scalar quantization Default is calculated as- * `1-1/(vector_dimensions + 1)`- *- * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.- */- @Deprecated(since = "10.1")- private Float confidenceInterval;-- /**- * When enabled, in conjunction with 4 bit size, will pair values into single bytes for 50%- * reduction in memory usage (comes at the cost of some decode speed penalty)- *- * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.- */- @Deprecated(since = "10.1")- private boolean compress;-public ScalarQuantizedDenseVectorField() {super();}@@ -70,13 +50,9 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {int dimension,VectorSimilarityFunction similarityFunction,VectorEncoding vectorEncoding,- int bits,- Float confidenceInterval,- boolean compress) {+ int bits) {super(dimension, similarityFunction, vectorEncoding);this.bits = bits;- this.confidenceInterval = confidenceInterval;- this.compress = compress;}@Override@@ -84,11 +60,12 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {this.bits = ofNullable(args.remove(BITS_PARAM)).map(Integer::parseInt).orElse(DEFAULT_BITS);// These params ("compress", "confidenceInterval", "dynamicConfidenceInterval") are deprecated- // since Solr 10.1. Lucene 10.4's scalar-quantized vector format no longer consumes them- // directly. They are parsed for backward compatibility but are no-ops going forward.+ // since Solr 10.1: Lucene's scalar-quantized vector format no longer consumes them, and as of+ // 11.0 nothing here retains their values either. They are still consumed from the args - and+ // only warned about - so that an existing schema setting them keeps loading instead of failing+ // FieldType.setArgs' "invalid arguments" check.String compressStr = args.remove(COMPRESS_PARAM);if (compressStr != null) {- this.compress = Boolean.parseBoolean(compressStr);DeprecationLog.log(COMPRESS_PARAM,"The '"@@ -99,20 +76,16 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {String confidenceIntervalStr = args.remove(CONFIDENCE_INTERVAL_PARAM);if (confidenceIntervalStr != null) {- this.confidenceInterval = Float.parseFloat(confidenceIntervalStr);DeprecationLog.log(CONFIDENCE_INTERVAL_PARAM,"The '"+ CONFIDENCE_INTERVAL_PARAM+ "' parameter for ScalarQuantizedDenseVectorField is deprecated since Solr 10.1"+ " and will be ignored. Please remove it from your schema.");- } else {- this.confidenceInterval = DEFAULT_CONFIDENCE_INTERVAL;}String dynamicConfidenceIntervalStr = args.remove(DYNAMIC_CONFIDENCE_INTERVAL_PARAM);if (Boolean.parseBoolean(dynamicConfidenceIntervalStr)) {- this.confidenceInterval = 0f;DeprecationLog.log(DYNAMIC_CONFIDENCE_INTERVAL_PARAM,"The '"@@ -156,22 +129,4 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {public int getBits() {return bits;}-- /**- * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.- */- @Deprecated(since = "10.1")- @VisibleForTesting- boolean useCompression() {- return compress;- }-- /**- * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.- */- @Deprecated(since = "10.1")- @VisibleForTesting- Float getConfidenceInterval() {- return confidenceInterval;- }}diff --git a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.javaindex 94ebc63003b..cb1f24b88d5 100644--- a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java+++ b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java@@ -47,10 +47,6 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBaassertThat(defaultVectorType.getDimension(), is(4));assertThat(defaultVectorType.getKnnAlgorithm(), is("hnsw"));assertThat(defaultVectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));- assertThat(- defaultVectorType.getConfidenceInterval(),- is(ScalarQuantizedDenseVectorField.DEFAULT_CONFIDENCE_INTERVAL));- assertThat(defaultVectorType.useCompression(), is(false));} finally {deleteCore();}@@ -75,7 +71,7 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBa}@Test- public void fieldDefinition_compressed_shouldLoadSchemaField() throws Exception {+ public void fieldDefinition_deprecatedCompress_shouldStillLoadSchemaField() throws Exception {try {initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");@@ -87,14 +83,14 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBaScalarQuantizedDenseVectorField vectorType =(ScalarQuantizedDenseVectorField) vectorField.getType();assertThat(vectorType.getBits(), is(4));- assertThat(vectorType.useCompression(), is(true));} finally {deleteCore();}}@Test- public void fieldDefinition_customConfidenceInterval_shouldLoadSchemaField() throws Exception {+ public void fieldDefinition_deprecatedConfidenceInterval_shouldStillLoadSchemaField()+ throws Exception {try {initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");@@ -105,14 +101,16 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBaScalarQuantizedDenseVectorField vectorType =(ScalarQuantizedDenseVectorField) vectorField.getType();- assertThat(vectorType.getConfidenceInterval(), is(0.91F));+ assertThat(vectorType.getDimension(), is(4));+ assertThat(vectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));} finally {deleteCore();}}@Test- public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() throws Exception {+ public void fieldDefinition_deprecatedDynamicConfidenceInterval_shouldStillLoadSchemaField()+ throws Exception {try {initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");@@ -123,7 +121,8 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBaScalarQuantizedDenseVectorField vectorType =(ScalarQuantizedDenseVectorField) vectorField.getType();- assertThat(vectorType.getConfidenceInterval(), is(0f));+ assertThat(vectorType.getDimension(), is(4));+ assertThat(vectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));} finally {deleteCore();}