Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-18376: remove ScalarQuantizedDenseVectorField compress and confidenceInterval (#4756)

Serhiy Bzhezytskyyf08f90999d21
diff --git a/changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.yml b/changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.yml
new file mode 100644
index 00000000000..ffd10e67b13
--- /dev/null
+++ b/changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.yml
@@ -0,0 +1,8 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Remove the deprecated ScalarQuantizedDenseVectorField compress and confidenceInterval fields and their accessors useCompression and getConfidenceInterval, since Solr 10.1 no longer used by the underlying Lucene codec; the compress, confidenceInterval and dynamicConfidenceInterval schema params are still accepted and ignored with a deprecation warning, so an existing schema that sets them keeps loading. The public four-argument constructor loses its two corresponding parameters and now takes dimension, similarityFunction, vectorEncoding and bits.
+type: removed
+authors:
+ - name: Serhiy Bzhezytskyy
+links:
+ - name: SOLR-18376
+ url: https://issues.apache.org/jira/browse/SOLR-18376
diff --git a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java
index 26a33be48a2..66f0387505d 100644
--- a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java
+++ b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java
@@ -18,7 +18,6 @@ package org.apache.solr.schema;
import static java.util.Optional.ofNullable;
-import com.google.common.annotations.VisibleForTesting;
import java.util.Map;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
@@ -36,7 +35,6 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {
"compress"; // can only be enabled when bits = 4 per Lucene codec spec
static final int DEFAULT_BITS = 7; // use signed byte as default when unspecified
- static final Float DEFAULT_CONFIDENCE_INTERVAL = null; // use dimension scaled confidence interval
/**
* Number of bits to use for storage Must be 4 (half-byte) or 7 (signed-byte) per Lucene codec
@@ -44,24 +42,6 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {
*/
private int bits;
- /**
- * Confidence interval to use for scalar quantization Default is calculated as
- * `1-1/(vector_dimensions + 1)`
- *
- * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.
- */
- @Deprecated(since = "10.1")
- private Float confidenceInterval;
-
- /**
- * When enabled, in conjunction with 4 bit size, will pair values into single bytes for 50%
- * reduction in memory usage (comes at the cost of some decode speed penalty)
- *
- * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.
- */
- @Deprecated(since = "10.1")
- private boolean compress;
-
public ScalarQuantizedDenseVectorField() {
super();
}
@@ -70,13 +50,9 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {
int dimension,
VectorSimilarityFunction similarityFunction,
VectorEncoding vectorEncoding,
- int bits,
- Float confidenceInterval,
- boolean compress) {
+ int bits) {
super(dimension, similarityFunction, vectorEncoding);
this.bits = bits;
- this.confidenceInterval = confidenceInterval;
- this.compress = compress;
}
@Override
@@ -84,11 +60,12 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {
this.bits = ofNullable(args.remove(BITS_PARAM)).map(Integer::parseInt).orElse(DEFAULT_BITS);
// These params ("compress", "confidenceInterval", "dynamicConfidenceInterval") are deprecated
- // since Solr 10.1. Lucene 10.4's scalar-quantized vector format no longer consumes them
- // directly. They are parsed for backward compatibility but are no-ops going forward.
+ // since Solr 10.1: Lucene's scalar-quantized vector format no longer consumes them, and as of
+ // 11.0 nothing here retains their values either. They are still consumed from the args - and
+ // only warned about - so that an existing schema setting them keeps loading instead of failing
+ // FieldType.setArgs' "invalid arguments" check.
String compressStr = args.remove(COMPRESS_PARAM);
if (compressStr != null) {
- this.compress = Boolean.parseBoolean(compressStr);
DeprecationLog.log(
COMPRESS_PARAM,
"The '"
@@ -99,20 +76,16 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {
String confidenceIntervalStr = args.remove(CONFIDENCE_INTERVAL_PARAM);
if (confidenceIntervalStr != null) {
- this.confidenceInterval = Float.parseFloat(confidenceIntervalStr);
DeprecationLog.log(
CONFIDENCE_INTERVAL_PARAM,
"The '"
+ CONFIDENCE_INTERVAL_PARAM
+ "' parameter for ScalarQuantizedDenseVectorField is deprecated since Solr 10.1"
+ " and will be ignored. Please remove it from your schema.");
- } else {
- this.confidenceInterval = DEFAULT_CONFIDENCE_INTERVAL;
}
String dynamicConfidenceIntervalStr = args.remove(DYNAMIC_CONFIDENCE_INTERVAL_PARAM);
if (Boolean.parseBoolean(dynamicConfidenceIntervalStr)) {
- this.confidenceInterval = 0f;
DeprecationLog.log(
DYNAMIC_CONFIDENCE_INTERVAL_PARAM,
"The '"
@@ -156,22 +129,4 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {
public int getBits() {
return bits;
}
-
- /**
- * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.
- */
- @Deprecated(since = "10.1")
- @VisibleForTesting
- boolean useCompression() {
- return compress;
- }
-
- /**
- * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.
- */
- @Deprecated(since = "10.1")
- @VisibleForTesting
- Float getConfidenceInterval() {
- return confidenceInterval;
- }
}
diff --git a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java
index 94ebc63003b..cb1f24b88d5 100644
--- a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java
+++ b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java
@@ -47,10 +47,6 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBa
assertThat(defaultVectorType.getDimension(), is(4));
assertThat(defaultVectorType.getKnnAlgorithm(), is("hnsw"));
assertThat(defaultVectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));
- assertThat(
- defaultVectorType.getConfidenceInterval(),
- is(ScalarQuantizedDenseVectorField.DEFAULT_CONFIDENCE_INTERVAL));
- assertThat(defaultVectorType.useCompression(), is(false));
} finally {
deleteCore();
}
@@ -75,7 +71,7 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBa
}
@Test
- public void fieldDefinition_compressed_shouldLoadSchemaField() throws Exception {
+ public void fieldDefinition_deprecatedCompress_shouldStillLoadSchemaField() throws Exception {
try {
initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
@@ -87,14 +83,14 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBa
ScalarQuantizedDenseVectorField vectorType =
(ScalarQuantizedDenseVectorField) vectorField.getType();
assertThat(vectorType.getBits(), is(4));
- assertThat(vectorType.useCompression(), is(true));
} finally {
deleteCore();
}
}
@Test
- public void fieldDefinition_customConfidenceInterval_shouldLoadSchemaField() throws Exception {
+ public void fieldDefinition_deprecatedConfidenceInterval_shouldStillLoadSchemaField()
+ throws Exception {
try {
initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
@@ -105,14 +101,16 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBa
ScalarQuantizedDenseVectorField vectorType =
(ScalarQuantizedDenseVectorField) vectorField.getType();
- assertThat(vectorType.getConfidenceInterval(), is(0.91F));
+ assertThat(vectorType.getDimension(), is(4));
+ assertThat(vectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));
} finally {
deleteCore();
}
}
@Test
- public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() throws Exception {
+ public void fieldDefinition_deprecatedDynamicConfidenceInterval_shouldStillLoadSchemaField()
+ throws Exception {
try {
initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
@@ -123,7 +121,8 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBa
ScalarQuantizedDenseVectorField vectorType =
(ScalarQuantizedDenseVectorField) vectorField.getType();
- assertThat(vectorType.getConfidenceInterval(), is(0f));
+ assertThat(vectorType.getDimension(), is(4));
+ assertThat(vectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));
} finally {
deleteCore();
}