Skip to content

Apache Lucene open-source search software

Bring new quantized format inline with old quantized format (#15622)

Benjamin Trent849ce20c4560
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
index 8888ed7e5b6..69deea50e15 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
@@ -31,7 +31,11 @@ import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
-/** Vector scorer over OptimizedScalarQuantized vectors */
+/**
+ * Vector scorer over OptimizedScalarQuantized vectors
+ *
+ * @lucene.experimental
+ */
public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
private final FlatVectorsScorer nonQuantizedDelegate;
@@ -103,7 +107,7 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target);
}
- RandomVectorScorerSupplier getRandomVectorScorerSupplier(
+ public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
VectorSimilarityFunction similarityFunction,
QuantizedByteVectorValues scoringVectors,
QuantizedByteVectorValues targetVectors) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
index 50cede6736b..7557b5e4c57 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
@@ -199,7 +199,7 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
this.queryBitsPerDim = queryBitsPerDim;
}
- boolean isAsymmetric() {
+ public boolean isAsymmetric() {
return bits != queryBits;
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
index dad76d4b614..9cfbf8acc6b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
@@ -133,6 +133,11 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
return vectorValue;
}
+ @Override
+ public IndexInput getSlice() {
+ return slice;
+ }
+
@Override
public float getCentroidDP() {
return centroidDp;
@@ -169,7 +174,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
@Override
public int getVectorByteLength() {
- return dimension;
+ return vectorValue.length;
}
static void packNibbles(byte[] unpacked, byte[] packed) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java
index 02b9c074899..70b48a4df3c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java
@@ -18,12 +18,14 @@ package org.apache.lucene.codecs.lucene104;
import java.io.IOException;
import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
+import org.apache.lucene.codecs.lucene95.HasIndexSlice;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.search.VectorScorer;
+import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
/** Scalar quantized byte vector values */
-abstract class QuantizedByteVectorValues extends ByteVectorValues {
+public abstract class QuantizedByteVectorValues extends ByteVectorValues implements HasIndexSlice {
/**
* Retrieve the corrective terms for the given vector ordinal. For the dot-product family of
@@ -82,4 +84,9 @@ abstract class QuantizedByteVectorValues extends ByteVectorValues {
@Override
public abstract QuantizedByteVectorValues copy() throws IOException;
+
+ @Override
+ public IndexInput getSlice() {
+ return null;
+ }
}