Apache Lucene open-source search software
Refactor QuantizedVectorsReader to be able to access lucene 104 quantized values through this API. (#15783)
Ignacio Vera99bf168ca270
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex e5fc7bee5a6..77f55c11805 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -261,6 +261,9 @@ Other* GITHUB#15767: Fix random test failures in LegacyBaseDocValuesFormatTestCase caused by missing commit. (Alan Woodward)+* GITHUB#15783: Refactor QuantizedVectorsReader to be able to access lucene 104 quantized values through this API.+ (Ignacio Vera)+======================= Lucene 10.4.0 =======================API Changesdiff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.javaindex 1f0cd5c631d..377ef2ddc06 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java@@ -48,7 +48,7 @@ import org.apache.lucene.store.IndexInput;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.RamUsageEstimator;import org.apache.lucene.util.hnsw.RandomVectorScorer;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;import org.apache.lucene.util.quantization.QuantizedVectorsReader;import org.apache.lucene.util.quantization.ScalarQuantizer;@@ -327,7 +327,7 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade}@Override- public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {+ public LegacyQuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {final FieldEntry fieldEntry = getFieldEntry(field);return OffHeapQuantizedByteVectorValues.load(fieldEntry.ordToDoc,@@ -422,10 +422,10 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReadeprivate static final class QuantizedVectorValues extends FloatVectorValues {private final FloatVectorValues rawVectorValues;- private final QuantizedByteVectorValues quantizedVectorValues;+ private final LegacyQuantizedByteVectorValues quantizedVectorValues;QuantizedVectorValues(- FloatVectorValues rawVectorValues, QuantizedByteVectorValues quantizedVectorValues) {+ FloatVectorValues rawVectorValues, LegacyQuantizedByteVectorValues quantizedVectorValues) {this.rawVectorValues = rawVectorValues;this.quantizedVectorValues = quantizedVectorValues;}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.javaindex 22fe9f2c569..c64ea69d9e5 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java@@ -29,14 +29,14 @@ import org.apache.lucene.store.IndexInput;import org.apache.lucene.util.Bits;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.packed.DirectMonotonicReader;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;import org.apache.lucene.util.quantization.ScalarQuantizer;/*** Read the quantized vector values and their score correction values from the index input. This* supports both iterated and random access.*/-public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVectorValues {+public abstract class OffHeapQuantizedByteVectorValues extends LegacyQuantizedByteVectorValues {final int dimension;final int size;diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.javaindex b36f153eaeb..54424b2915d 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java@@ -61,7 +61,8 @@ import org.apache.lucene.util.VectorUtil;import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.BaseQuantizedByteVectorValues;+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;import org.apache.lucene.util.quantization.QuantizedVectorsReader;import org.apache.lucene.util.quantization.ScalarQuantizer;@@ -713,7 +714,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite*/public static DocsWithFieldSet writeQuantizedVectorData(IndexOutput output,- QuantizedByteVectorValues quantizedByteVectorValues,+ LegacyQuantizedByteVectorValues quantizedByteVectorValues,byte bits,boolean compress)throws IOException {@@ -883,10 +884,10 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite}static class QuantizedByteVectorValueSub extends DocIDMerger.Sub {- private final QuantizedByteVectorValues values;+ private final LegacyQuantizedByteVectorValues values;private final KnnVectorValues.DocIndexIterator iterator;- QuantizedByteVectorValueSub(MergeState.DocMap docMap, QuantizedByteVectorValues values) {+ QuantizedByteVectorValueSub(MergeState.DocMap docMap, LegacyQuantizedByteVectorValues values) {super(docMap);this.values = values;iterator = values.iterator();@@ -903,8 +904,8 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite}}- /** Returns a merged view over all the segment's {@link QuantizedByteVectorValues}. */- static class MergedQuantizedVectorValues extends QuantizedByteVectorValues {+ /** Returns a merged view over all the segment's {@link BaseQuantizedByteVectorValues}. */+ static class MergedQuantizedVectorValues extends LegacyQuantizedByteVectorValues {public static MergedQuantizedVectorValues mergeQuantizedByteVectorValues(FieldInfo fieldInfo, MergeState mergeState, ScalarQuantizer scalarQuantizer)throws IOException {@@ -941,7 +942,8 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWritenew QuantizedByteVectorValueSub(mergeState.docMaps[i],new OffsetCorrectedQuantizedByteVectorValues(- reader.getQuantizedVectorValues(fieldInfo.name),+ (LegacyQuantizedByteVectorValues)+ reader.getQuantizedVectorValues(fieldInfo.name),fieldInfo.getVectorSimilarityFunction(),scalarQuantizer,reader.getQuantizationState(fieldInfo.name)));@@ -1038,7 +1040,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite}}- static class QuantizedFloatVectorValues extends QuantizedByteVectorValues {+ static class QuantizedFloatVectorValues extends LegacyQuantizedByteVectorValues {private final FloatVectorValues values;private final ScalarQuantizer quantizer;private final byte[] quantizedVector;@@ -1143,14 +1145,15 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite}}- static final class OffsetCorrectedQuantizedByteVectorValues extends QuantizedByteVectorValues {+ static final class OffsetCorrectedQuantizedByteVectorValues+ extends LegacyQuantizedByteVectorValues {- private final QuantizedByteVectorValues in;+ private final LegacyQuantizedByteVectorValues in;private final VectorSimilarityFunction vectorSimilarityFunction;private final ScalarQuantizer scalarQuantizer, oldScalarQuantizer;OffsetCorrectedQuantizedByteVectorValues(- QuantizedByteVectorValues in,+ LegacyQuantizedByteVectorValues in,VectorSimilarityFunction vectorSimilarityFunction,ScalarQuantizer scalarQuantizer,ScalarQuantizer oldScalarQuantizer) {diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.javaindex 6d967942811..c42ebfebfbf 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java@@ -52,7 +52,7 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.SameThreadExecutorService;import org.apache.lucene.util.VectorUtil;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;import org.apache.lucene.util.quantization.ScalarQuantizer;import org.junit.Before;@@ -277,8 +277,8 @@ public class TestLucene99HnswQuantizedVectorsFormat extends BaseKnnVectorsFormatknnVectorsReader = knnVectorsReader.unwrapReaderForField("f");if (knnVectorsReader instanceof Lucene99HnswVectorsReader hnswReader) {assertNotNull(hnswReader.getQuantizationState("f"));- QuantizedByteVectorValues quantizedByteVectorValues =- hnswReader.getQuantizedVectorValues("f");+ LegacyQuantizedByteVectorValues quantizedByteVectorValues =+ (LegacyQuantizedByteVectorValues) hnswReader.getQuantizedVectorValues("f");for (int ord = 0; ord < quantizedByteVectorValues.size(); ord++) {byte[] vector = quantizedByteVectorValues.vectorValue(ord);float offset = quantizedByteVectorValues.getScoreCorrectionConstant(ord);diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.javaindex 9829fe4d509..ac07c6bcfde 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java@@ -48,7 +48,7 @@ import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.VectorUtil;import org.apache.lucene.util.hnsw.RandomVectorScorer;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;import org.apache.lucene.util.quantization.ScalarQuantizer;public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {@@ -98,8 +98,8 @@ public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {Lucene99ScalarQuantizedVectorScorer scorer =new Lucene99ScalarQuantizedVectorScorer(new DefaultFlatVectorScorer());- QuantizedByteVectorValues values =- new QuantizedByteVectorValues() {+ LegacyQuantizedByteVectorValues values =+ new LegacyQuantizedByteVectorValues() {@Overridepublic int dimension() {return 32;@@ -126,7 +126,7 @@ public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {}@Override- public QuantizedByteVectorValues copy() throws IOException {+ public LegacyQuantizedByteVectorValues copy() throws IOException {return this;}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.javaindex 2244b71d7bf..630fe5d9d5c 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java@@ -49,7 +49,7 @@ import org.apache.lucene.store.IndexOutput;import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.VectorUtil;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;import org.apache.lucene.util.quantization.ScalarQuantizer;import org.junit.Before;@@ -180,7 +180,7 @@ public class TestLucene99ScalarQuantizedVectorsFormat extends BaseKnnVectorsFormknnVectorsReader = knnVectorsReader.unwrapReaderForField("f");if (knnVectorsReader instanceof Lucene99ScalarQuantizedVectorsReader quantizedReader) {assertNotNull(quantizedReader.getQuantizationState("f"));- QuantizedByteVectorValues quantizedByteVectorValues =+ LegacyQuantizedByteVectorValues quantizedByteVectorValues =quantizedReader.getQuantizedVectorValues("f");int docId = -1;KnnVectorValues.DocIndexIterator iter = quantizedByteVectorValues.iterator();diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.javaindex 7a874ba5500..0d07bd5f52c 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java@@ -43,7 +43,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.Version;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.BaseQuantizedByteVectorValues;public class TestInt7HnswBackwardsCompatibility extends BackwardsCompatibilityTestBase {@@ -157,7 +157,7 @@ public class TestInt7HnswBackwardsCompatibility extends BackwardsCompatibilityTeassertTrue(forField instanceof Lucene99HnswVectorsReader);- QuantizedByteVectorValues quantized =+ BaseQuantizedByteVectorValues quantized =((Lucene99HnswVectorsReader) forField).getQuantizedVectorValues(KNN_VECTOR_FIELD);assertNotNull(diff --git a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/ScalarQuantizedVectorScorer.javaindex ad4bb7e3c6d..aa8a1aac45b 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/ScalarQuantizedVectorScorer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/ScalarQuantizedVectorScorer.java@@ -25,7 +25,7 @@ import org.apache.lucene.util.VectorUtil;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;import org.apache.lucene.util.quantization.ScalarQuantizedVectorSimilarity;import org.apache.lucene.util.quantization.ScalarQuantizer;@@ -63,7 +63,7 @@ public class ScalarQuantizedVectorScorer implements FlatVectorsScorer {public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)throws IOException {- if (vectorValues instanceof QuantizedByteVectorValues quantizedByteVectorValues) {+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantizedByteVectorValues) {return new ScalarQuantizedRandomVectorScorerSupplier(similarityFunction,quantizedByteVectorValues.getScalarQuantizer(),@@ -77,7 +77,7 @@ public class ScalarQuantizedVectorScorer implements FlatVectorsScorer {public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)throws IOException {- if (vectorValues instanceof QuantizedByteVectorValues quantizedByteVectorValues) {+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantizedByteVectorValues) {ScalarQuantizer scalarQuantizer = quantizedByteVectorValues.getScalarQuantizer();byte[] targetBytes = new byte[target.length];float offsetCorrection =@@ -121,14 +121,14 @@ public class ScalarQuantizedVectorScorer implements FlatVectorsScorer {public static class ScalarQuantizedRandomVectorScorerSupplierimplements RandomVectorScorerSupplier {- private final QuantizedByteVectorValues values;+ private final LegacyQuantizedByteVectorValues values;private final ScalarQuantizedVectorSimilarity similarity;private final VectorSimilarityFunction vectorSimilarityFunction;public ScalarQuantizedRandomVectorScorerSupplier(VectorSimilarityFunction similarityFunction,ScalarQuantizer scalarQuantizer,- QuantizedByteVectorValues values) {+ LegacyQuantizedByteVectorValues values) {this.similarity =ScalarQuantizedVectorSimilarity.fromVectorSimilarity(similarityFunction,@@ -141,7 +141,7 @@ public class ScalarQuantizedVectorScorer implements FlatVectorsScorer {private ScalarQuantizedRandomVectorScorerSupplier(ScalarQuantizedVectorSimilarity similarity,VectorSimilarityFunction vectorSimilarityFunction,- QuantizedByteVectorValues values) {+ LegacyQuantizedByteVectorValues values) {this.similarity = similarity;this.values = values;this.vectorSimilarityFunction = vectorSimilarityFunction;@@ -149,7 +149,7 @@ public class ScalarQuantizedVectorScorer implements FlatVectorsScorer {@Overridepublic UpdateableRandomVectorScorer scorer() throws IOException {- final QuantizedByteVectorValues vectorsCopy = values.copy();+ final LegacyQuantizedByteVectorValues vectorsCopy = values.copy();byte[] queryVector = new byte[values.dimension()];return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(vectorsCopy) {float queryOffset = 0;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.javaindex 0bb271436d1..4fd3dd6dbcf 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java@@ -28,7 +28,6 @@ import java.util.concurrent.ExecutorService;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.KnnVectorsReader;import org.apache.lucene.codecs.KnnVectorsWriter;-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;@@ -36,6 +35,7 @@ import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.search.TaskExecutor;import org.apache.lucene.util.hnsw.HnswGraph;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;/*** A vectors format that uses HNSW graph to store and search for vectors. But vectors are binarydiff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.javaindex 69deea50e15..53d3363e64b 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java@@ -30,6 +30,8 @@ import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;/*** Vector scorer over OptimizedScalarQuantized vectors@@ -61,7 +63,7 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {if (vectorValues instanceof QuantizedByteVectorValues qv) {FlatVectorsScorer.checkDimensions(target.length, qv.dimension());OptimizedScalarQuantizer quantizer = qv.getQuantizer();- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding scalarEncoding = qv.getScalarEncoding();+ ScalarEncoding scalarEncoding = qv.getScalarEncoding();byte[] scratch = new byte[scalarEncoding.getDiscreteDimensions(qv.dimension())];final byte[] targetQuantized;if (scalarEncoding.isAsymmetric() == false) {@@ -81,10 +83,8 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {target, scratch, scalarEncoding.getQueryBits(), qv.getCentroid());// for asymmetric encodings with 4-bit query, we need to transpose the nibbles for fast// scoring comparisons- if (scalarEncoding- == Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE- || scalarEncoding- == Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.DIBIT_QUERY_NIBBLE) {+ if (scalarEncoding == ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE+ || scalarEncoding == ScalarEncoding.DIBIT_QUERY_NIBBLE) {OptimizedScalarQuantizer.transposeHalfByte(scratch, targetQuantized);}return new RandomVectorScorer.AbstractRandomVectorScorer(qv) {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.javaindex a5a1c1de97c..fb8221deffb 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java@@ -17,7 +17,6 @@package org.apache.lucene.codecs.lucene104;import java.io.IOException;-import java.util.Optional;import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;@@ -25,6 +24,7 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;/*** The quantization format used here is a per-vector optimized scalar quantization. These ideas are@@ -112,163 +112,6 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {private final ScalarEncoding encoding;- /**- * Allowed encodings for scalar quantization.- *- * <p>This specifies how many bits are used per dimension and also dictates packing of dimensions- * into a byte stream.- */- public enum ScalarEncoding {- /** Each dimension is quantized to 8 bits and treated as an unsigned value. */- UNSIGNED_BYTE(0, (byte) 8, 8),- /** Each dimension is quantized to 4 bits two values are packed into each output byte. */- PACKED_NIBBLE(1, (byte) 4, 4),- /**- * Each dimension is quantized to 7 bits and treated as a signed value.- *- * <p>This is intended for backwards compatibility with older iterations of scalar quantization.- * This setting will produce an index the same size as {@link #UNSIGNED_BYTE} but will produce- * less accurate vector comparisons.- */- SEVEN_BIT(2, (byte) 7, 8),- /**- * Each dimension is quantized to a single bit and packed into bytes. During query time, the- * query vector is quantized to 4 bits per dimension.- *- * <p>This is the most space efficient encoding, and will produce an index 8x smaller than- * {@link #UNSIGNED_BYTE}. However, this comes at the cost of accuracy.- */- SINGLE_BIT_QUERY_NIBBLE(3, (byte) 1, 1, (byte) 4, 4),- /**- * Each dimension is quantized to 2 bits (dibit) and packed into bytes. During query time, the- * query vector is quantized to 4 bits per dimension.- *- * <p>This encoding produces an index 4x smaller than {@link #UNSIGNED_BYTE}, offering a balance- * between the compression of {@link #SINGLE_BIT_QUERY_NIBBLE} and the accuracy of {@link- * #PACKED_NIBBLE}.- */- DIBIT_QUERY_NIBBLE(4, (byte) 2, 2, (byte) 4, 4) {- @Override- public int getDiscreteDimensions(int dimensions) {- int queryDiscretized = (dimensions * 4 + 7) / 8 * 8 / 4;- // we want to force dibit packing to byte boundaries assuming single bit striping- // so we discretize to the same as single bit encoding- int docDiscretized = (dimensions + 7) / 8 * 8;- int maxDiscretized = Math.max(queryDiscretized, docDiscretized);- assert maxDiscretized % (8.0 / 4) == 0- : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;- assert maxDiscretized % (8.0 / 2) == 0- : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;- return maxDiscretized;- }-- @Override- public int getDocPackedLength(int dimensions) {- int discretized = getDiscreteDimensions(dimensions);- // DIBIT should be stored as two single bits striped- return 2 * ((discretized + 7) / 8);- }- };-- public static ScalarEncoding fromNumBits(int bits) {- for (ScalarEncoding encoding : values()) {- if (encoding.bits == bits) {- return encoding;- }- }- throw new IllegalArgumentException("No encoding for " + bits + " bits");- }-- /** The number used to identify this encoding on the wire, rather than relying on ordinal. */- private final int wireNumber;-- private final byte bits, queryBits;- private final int bitsPerDim, queryBitsPerDim;-- ScalarEncoding(int wireNumber, byte bits, int bitsPerDim) {- this.wireNumber = wireNumber;- this.bits = bits;- this.queryBits = bits;- this.bitsPerDim = bitsPerDim;- this.queryBitsPerDim = bitsPerDim;- }-- ScalarEncoding(int wireNumber, byte bits, int bitsPerDim, byte queryBits, int queryBitsPerDim) {- this.wireNumber = wireNumber;- this.bits = bits;- this.queryBits = queryBits;- this.bitsPerDim = bitsPerDim;- this.queryBitsPerDim = queryBitsPerDim;- }-- public boolean isAsymmetric() {- return bits != queryBits;- }-- int getWireNumber() {- return wireNumber;- }-- /** Return the number of bits used per dimension. */- public byte getBits() {- return bits;- }-- public byte getQueryBits() {- return queryBits;- }-- /** Return the number of dimensions rounded up to fit into whole bytes. */- public int getDiscreteDimensions(int dimensions) {- if (queryBits == bits) {- int totalBits = dimensions * bitsPerDim;- return (totalBits + 7) / 8 * 8 / bitsPerDim;- }- int queryDiscretized = (dimensions * queryBitsPerDim + 7) / 8 * 8 / queryBitsPerDim;- int docDiscretized = (dimensions * bitsPerDim + 7) / 8 * 8 / bitsPerDim;- int maxDiscretized = Math.max(queryDiscretized, docDiscretized);- assert maxDiscretized % (8.0 / queryBitsPerDim) == 0- : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;- assert maxDiscretized % (8.0 / bitsPerDim) == 0- : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;- return maxDiscretized;- }-- /** Return the number of dimensions that can be packed into a single byte. */- public int getDocBitsPerDim() {- return this.bitsPerDim;- }-- public int getQueryBitsPerDim() {- return this.queryBitsPerDim;- }-- /** Return the number of bytes required to store a packed vector of the given dimensions. */- public int getDocPackedLength(int dimensions) {- int discretized = getDiscreteDimensions(dimensions);- // how many bytes do we need to store the quantized vector?- int totalBits = discretized * bitsPerDim;- return (totalBits + 7) / 8;- }-- public int getQueryPackedLength(int dimensions) {- int discretized = getDiscreteDimensions(dimensions);- // how many bytes do we need to store the quantized vector?- int totalBits = discretized * queryBitsPerDim;- return (totalBits + 7) / 8;- }-- /** Returns the encoding for the given wire number, or empty if unknown. */- public static Optional<ScalarEncoding> fromWireNumber(int wireNumber) {- for (ScalarEncoding encoding : values()) {- if (encoding.wireNumber == wireNumber) {- return Optional.of(encoding);- }- }- return Optional.empty();- }- }-/** Creates a new instance with UNSIGNED_BYTE encoding. */public Lucene104ScalarQuantizedVectorsFormat() {this(ScalarEncoding.UNSIGNED_BYTE);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.javaindex 329921542fa..9097768689c 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java@@ -27,7 +27,6 @@ import java.util.Objects;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.KnnVectorsReader;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;import org.apache.lucene.index.ByteVectorValues;import org.apache.lucene.index.CorruptIndexException;@@ -52,6 +51,8 @@ import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.RamUsageEstimator;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;import org.apache.lucene.util.quantization.QuantizedVectorsReader;import org.apache.lucene.util.quantization.ScalarQuantizer;@@ -389,8 +390,7 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader}@Override- public org.apache.lucene.util.quantization.QuantizedByteVectorValues getQuantizedVectorValues(- String field) throws IOException {+ public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {FieldEntry fi = fields.get(field);if (fi == null) {return null;@@ -404,41 +404,19 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader+ " expected: "+ VectorEncoding.FLOAT32);}- var qv =- OffHeapScalarQuantizedVectorValues.load(- fi.ordToDocDISIReaderConfiguration,- fi.dimension,- fi.size,- new OptimizedScalarQuantizer(fi.similarityFunction),- fi.scalarEncoding,- fi.similarityFunction,- vectorScorer,- fi.centroid,- fi.centroidDP,- fi.vectorDataOffset,- fi.vectorDataLength,- quantizedVectorData);- return new org.apache.lucene.util.quantization.QuantizedByteVectorValues() {- @Override- public float getScoreCorrectionConstant(int ord) throws IOException {- return 0;- }-- @Override- public byte[] vectorValue(int ord) throws IOException {- return qv.vectorValue(ord);- }-- @Override- public int dimension() {- return qv.dimension();- }-- @Override- public int size() {- return qv.size();- }- };+ return OffHeapScalarQuantizedVectorValues.load(+ fi.ordToDocDISIReaderConfiguration,+ fi.dimension,+ fi.size,+ new OptimizedScalarQuantizer(fi.similarityFunction),+ fi.scalarEncoding,+ fi.similarityFunction,+ vectorScorer,+ fi.centroid,+ fi.centroidDP,+ fi.vectorDataOffset,+ fi.vectorDataLength,+ quantizedVectorData);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.javaindex 4206130d4c2..d591aa0a7e7 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java@@ -36,7 +36,6 @@ import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.KnnVectorsReader;import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter;import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;import org.apache.lucene.index.DocsWithFieldSet;import org.apache.lucene.index.FieldInfo;@@ -59,6 +58,8 @@ import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;/*** Writes quantized vector values and metadata to index segments in the format for Lucene 10.4.diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.javaindex ec3dbbea087..8fa87a271d2 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java@@ -34,6 +34,7 @@ import org.apache.lucene.util.Bits;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.packed.DirectMonotonicReader;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;/*** Reads quantized vector values from the index input and returns float vector values after@@ -68,14 +69,14 @@ abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValuesprivate int lastOrd = -1;final float[] correctiveValues;int quantizedComponentSum;- final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding;+ final ScalarEncoding encoding;final float[] centroid;OffHeapScalarQuantizedFloatVectorValues(int dimension,int size,float[] centroid,- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,+ ScalarEncoding encoding,VectorSimilarityFunction similarityFunction,FlatVectorsScorer vectorsScorer,IndexInput slice) {@@ -178,7 +179,7 @@ abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValuesOrdToDocDISIReaderConfiguration configuration,int dimension,int size,- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,+ ScalarEncoding encoding,VectorSimilarityFunction similarityFunction,FlatVectorsScorer vectorsScorer,float[] centroid,@@ -219,7 +220,7 @@ abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValuesint dimension,int size,float[] centroid,- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,+ ScalarEncoding encoding,VectorSimilarityFunction similarityFunction,FlatVectorsScorer vectorsScorer,IndexInput slice) {@@ -281,7 +282,7 @@ abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValuesint dimension,int size,float[] centroid,- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,+ ScalarEncoding encoding,IndexInput dataIn,VectorSimilarityFunction similarityFunction,FlatVectorsScorer vectorsScorer,@@ -372,7 +373,7 @@ abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValuesdimension,0,null,- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.UNSIGNED_BYTE,+ ScalarEncoding.UNSIGNED_BYTE,similarityFunction,vectorsScorer,null);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.javaindex 9cfbf8acc6b..a91ace8ff9f 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java@@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene104;import java.io.IOException;import java.nio.ByteBuffer;import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;import org.apache.lucene.codecs.lucene90.IndexedDISI;import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;import org.apache.lucene.index.VectorSimilarityFunction;@@ -30,6 +29,7 @@ import org.apache.lucene.util.Bits;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.packed.DirectMonotonicReader;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;/*** Scalar quantized vector values loaded from off-heapdiff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.javadeleted file mode 100644index 72f35bfc3c7..00000000000--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java+++ /dev/null@@ -1,96 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.codecs.lucene104;--import java.io.IOException;-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;-import org.apache.lucene.codecs.lucene95.HasIndexSlice;-import org.apache.lucene.index.ByteVectorValues;-import org.apache.lucene.search.VectorScorer;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;--/**- * Scalar quantized byte vector values- *- * @lucene.experimental- */-public abstract class QuantizedByteVectorValues extends ByteVectorValues implements HasIndexSlice {-- /**- * Retrieve the corrective terms for the given vector ordinal. For the dot-product family of- * distances, the corrective terms are, in order- *- * <ul>- * <li>the lower optimized interval- * <li>the upper optimized interval- * <li>the dot-product of the non-centered vector with the centroid- * <li>the sum of quantized components- * </ul>- *- * For euclidean:- *- * <ul>- * <li>the lower optimized interval- * <li>the upper optimized interval- * <li>the l2norm of the centered vector- * <li>the sum of quantized components- * </ul>- *- * @param vectorOrd the vector ordinal- * @return the corrective terms- * @throws IOException if an I/O error occurs- */- public abstract OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int vectorOrd)- throws IOException;-- /**- * @return the quantizer used to quantize the vectors- */- public abstract OptimizedScalarQuantizer getQuantizer();-- /**- * @return the scalar encoding used to pack the stored vectors.- */- public abstract ScalarEncoding getScalarEncoding();-- /**- * @return the centroid used to center the vectors prior to quantization- */- public abstract float[] getCentroid() throws IOException;-- /**- * @return the dot product of the centroid.- */- public abstract float getCentroidDP() throws IOException;-- /**- * Return a {@link VectorScorer} for the given query vector.- *- * @param query the query vector- * @return a {@link VectorScorer} instance or null- */- public abstract VectorScorer scorer(float[] query) throws IOException;-- @Override- public abstract QuantizedByteVectorValues copy() throws IOException;-- @Override- public IndexInput getSlice() {- return null;- }-}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.javaindex e55e015652c..0931048d7ac 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java@@ -59,7 +59,7 @@ import org.apache.lucene.util.hnsw.HnswGraphSearcher;import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.packed.DirectMonotonicReader;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.BaseQuantizedByteVectorValues;import org.apache.lucene.util.quantization.QuantizedVectorsReader;import org.apache.lucene.util.quantization.ScalarQuantizer;@@ -419,7 +419,7 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader}@Override- public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {+ public BaseQuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {if (flatVectorsReader instanceof QuantizedVectorsReader) {return ((QuantizedVectorsReader) flatVectorsReader).getQuantizedVectorValues(field);}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.javaindex a668c01de6b..57f5b77f290 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java@@ -28,7 +28,8 @@ import org.apache.lucene.util.VectorUtil;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.BaseQuantizedByteVectorValues;+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;import org.apache.lucene.util.quantization.ScalarQuantizer;/**@@ -49,7 +50,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)throws IOException {- if (vectorValues instanceof QuantizedByteVectorValues quantizedByteVectorValues) {+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantizedByteVectorValues) {return new ScalarQuantizedRandomVectorScorerSupplier(quantizedByteVectorValues, similarityFunction);}@@ -61,7 +62,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)throws IOException {- if (vectorValues instanceof QuantizedByteVectorValues quantizedByteVectorValues) {+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantizedByteVectorValues) {ScalarQuantizer scalarQuantizer = quantizedByteVectorValues.getScalarQuantizer();byte[] targetBytes = new byte[target.length];float offsetCorrection =@@ -94,7 +95,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {float offsetCorrection,VectorSimilarityFunction sim,float constMultiplier,- QuantizedByteVectorValues values) {+ LegacyQuantizedByteVectorValues values) {FlatVectorsScorer.checkDimensions(values.dimension(), targetBytes.length);return switch (sim) {case EUCLIDEAN -> new Euclidean(values, constMultiplier, targetBytes);@@ -120,7 +121,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {byte[] targetBytes,float offsetCorrection,float constMultiplier,- QuantizedByteVectorValues values,+ LegacyQuantizedByteVectorValues values,FloatToFloatFunction scoreAdjustmentFunction) {if (values.getScalarQuantizer().getBits() <= 4) {if (values.getVectorByteLength() != values.dimension() && values.getSlice() != null) {@@ -138,9 +139,10 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {extends UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer {private final float constMultiplier;private final byte[] targetBytes;- private final QuantizedByteVectorValues values;+ private final BaseQuantizedByteVectorValues values;- private Euclidean(QuantizedByteVectorValues values, float constMultiplier, byte[] targetBytes) {+ private Euclidean(+ BaseQuantizedByteVectorValues values, float constMultiplier, byte[] targetBytes) {super(values);this.values = values;this.constMultiplier = constMultiplier;@@ -165,13 +167,13 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {private static class DotProductextends UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer {private final float constMultiplier;- private final QuantizedByteVectorValues values;+ private final LegacyQuantizedByteVectorValues values;private final byte[] targetBytes;private float offsetCorrection;private final FloatToFloatFunction scoreAdjustmentFunction;public DotProduct(- QuantizedByteVectorValues values,+ LegacyQuantizedByteVectorValues values,float constMultiplier,byte[] targetBytes,float offsetCorrection,@@ -210,14 +212,14 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {private static class CompressedInt4DotProductextends UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer {private final float constMultiplier;- private final QuantizedByteVectorValues values;+ private final LegacyQuantizedByteVectorValues values;private final byte[] compressedVector;private final byte[] targetBytes;private float offsetCorrection;private final FloatToFloatFunction scoreAdjustmentFunction;private CompressedInt4DotProduct(- QuantizedByteVectorValues values,+ LegacyQuantizedByteVectorValues values,float constMultiplier,byte[] targetBytes,float offsetCorrection,@@ -257,13 +259,13 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {private static class Int4DotProductextends UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer {private final float constMultiplier;- private final QuantizedByteVectorValues values;+ private final LegacyQuantizedByteVectorValues values;private final byte[] targetBytes;private float offsetCorrection;private final FloatToFloatFunction scoreAdjustmentFunction;public Int4DotProduct(- QuantizedByteVectorValues values,+ LegacyQuantizedByteVectorValues values,float constMultiplier,byte[] targetBytes,float offsetCorrection,@@ -299,11 +301,11 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {implements RandomVectorScorerSupplier {private final VectorSimilarityFunction vectorSimilarityFunction;- private final QuantizedByteVectorValues values;- private final QuantizedByteVectorValues targetVectors;+ private final LegacyQuantizedByteVectorValues values;+ private final LegacyQuantizedByteVectorValues targetVectors;public ScalarQuantizedRandomVectorScorerSupplier(- QuantizedByteVectorValues values, VectorSimilarityFunction vectorSimilarityFunction)+ LegacyQuantizedByteVectorValues values, VectorSimilarityFunction vectorSimilarityFunction)throws IOException {this.values = values;this.targetVectors = values.copy();diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/BaseQuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/util/quantization/BaseQuantizedByteVectorValues.javanew file mode 100644index 00000000000..5e18fbaee1e--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/BaseQuantizedByteVectorValues.java@@ -0,0 +1,47 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.util.quantization;++import java.io.IOException;+import org.apache.lucene.codecs.lucene95.HasIndexSlice;+import org.apache.lucene.index.ByteVectorValues;+import org.apache.lucene.search.VectorScorer;+import org.apache.lucene.store.IndexInput;++/**+ * A version of {@link ByteVectorValues} for Scalar quantization scores.+ *+ * @lucene.experimental+ */+public abstract class BaseQuantizedByteVectorValues extends ByteVectorValues+ implements HasIndexSlice {++ /**+ * Return a {@link VectorScorer} for the given query vector.+ *+ * @param query the query vector+ * @return a {@link VectorScorer} instance or null+ */+ public VectorScorer scorer(float[] query) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public IndexInput getSlice() {+ return null;+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/LegacyQuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/util/quantization/LegacyQuantizedByteVectorValues.javanew file mode 100644index 00000000000..9b472f8afc0--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/LegacyQuantizedByteVectorValues.java@@ -0,0 +1,39 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.util.quantization;++import java.io.IOException;++/**+ * A version of {@link BaseQuantizedByteVectorValues}, but additionally retrieving score correction+ * offset for Scalar quantization scores.+ *+ * @lucene.experimental+ */+public abstract class LegacyQuantizedByteVectorValues extends BaseQuantizedByteVectorValues {++ public ScalarQuantizer getScalarQuantizer() {+ throw new UnsupportedOperationException();+ }++ public abstract float getScoreCorrectionConstant(int ord) throws IOException;++ @Override+ public LegacyQuantizedByteVectorValues copy() throws IOException {+ return this;+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.javaindex b90ab8276dd..25f47ca38fa 100644--- a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java@@ -17,42 +17,219 @@package org.apache.lucene.util.quantization;import java.io.IOException;-import org.apache.lucene.codecs.lucene95.HasIndexSlice;-import org.apache.lucene.index.ByteVectorValues;-import org.apache.lucene.search.VectorScorer;-import org.apache.lucene.store.IndexInput;+import java.util.Optional;/**- * A version of {@link ByteVectorValues}, but additionally retrieving score correction offset for- * Scalar quantization scores.+ * Scalar quantized byte vector values** @lucene.experimental*/-public abstract class QuantizedByteVectorValues extends ByteVectorValues implements HasIndexSlice {+public abstract class QuantizedByteVectorValues extends BaseQuantizedByteVectorValues {- public ScalarQuantizer getScalarQuantizer() {- throw new UnsupportedOperationException();- }+ /**+ * Allowed encodings for scalar quantization.+ *+ * <p>This specifies how many bits are used per dimension and also dictates packing of dimensions+ * into a byte stream.+ */+ public enum ScalarEncoding {+ /** Each dimension is quantized to 8 bits and treated as an unsigned value. */+ UNSIGNED_BYTE(0, (byte) 8, 8),+ /** Each dimension is quantized to 4 bits two values are packed into each output byte. */+ PACKED_NIBBLE(1, (byte) 4, 4),+ /**+ * Each dimension is quantized to 7 bits and treated as a signed value.+ *+ * <p>This is intended for backwards compatibility with older iterations of scalar quantization.+ * This setting will produce an index the same size as {@link #UNSIGNED_BYTE} but will produce+ * less accurate vector comparisons.+ */+ SEVEN_BIT(2, (byte) 7, 8),+ /**+ * Each dimension is quantized to a single bit and packed into bytes. During query time, the+ * query vector is quantized to 4 bits per dimension.+ *+ * <p>This is the most space efficient encoding, and will produce an index 8x smaller than+ * {@link #UNSIGNED_BYTE}. However, this comes at the cost of accuracy.+ */+ SINGLE_BIT_QUERY_NIBBLE(3, (byte) 1, 1, (byte) 4, 4),+ /**+ * Each dimension is quantized to 2 bits (dibit) and packed into bytes. During query time, the+ * query vector is quantized to 4 bits per dimension.+ *+ * <p>This encoding produces an index 4x smaller than {@link #UNSIGNED_BYTE}, offering a balance+ * between the compression of {@link #SINGLE_BIT_QUERY_NIBBLE} and the accuracy of {@link+ * #PACKED_NIBBLE}.+ */+ DIBIT_QUERY_NIBBLE(4, (byte) 2, 2, (byte) 4, 4) {+ @Override+ public int getDiscreteDimensions(int dimensions) {+ int queryDiscretized = (dimensions * 4 + 7) / 8 * 8 / 4;+ // we want to force dibit packing to byte boundaries assuming single bit striping+ // so we discretize to the same as single bit encoding+ int docDiscretized = (dimensions + 7) / 8 * 8;+ int maxDiscretized = Math.max(queryDiscretized, docDiscretized);+ assert maxDiscretized % (8.0 / 4) == 0+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;+ assert maxDiscretized % (8.0 / 2) == 0+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;+ return maxDiscretized;+ }++ @Override+ public int getDocPackedLength(int dimensions) {+ int discretized = getDiscreteDimensions(dimensions);+ // DIBIT should be stored as two single bits striped+ return 2 * ((discretized + 7) / 8);+ }+ };++ public static ScalarEncoding fromNumBits(int bits) {+ for (ScalarEncoding encoding : values()) {+ if (encoding.bits == bits) {+ return encoding;+ }+ }+ throw new IllegalArgumentException("No encoding for " + bits + " bits");+ }++ /** The number used to identify this encoding on the wire, rather than relying on ordinal. */+ private final int wireNumber;++ private final byte bits, queryBits;+ private final int bitsPerDim, queryBitsPerDim;++ ScalarEncoding(int wireNumber, byte bits, int bitsPerDim) {+ this.wireNumber = wireNumber;+ this.bits = bits;+ this.queryBits = bits;+ this.bitsPerDim = bitsPerDim;+ this.queryBitsPerDim = bitsPerDim;+ }++ ScalarEncoding(int wireNumber, byte bits, int bitsPerDim, byte queryBits, int queryBitsPerDim) {+ this.wireNumber = wireNumber;+ this.bits = bits;+ this.queryBits = queryBits;+ this.bitsPerDim = bitsPerDim;+ this.queryBitsPerDim = queryBitsPerDim;+ }++ public boolean isAsymmetric() {+ return bits != queryBits;+ }++ public int getWireNumber() {+ return wireNumber;+ }++ /** Return the number of bits used per dimension. */+ public byte getBits() {+ return bits;+ }++ public byte getQueryBits() {+ return queryBits;+ }++ /** Return the number of dimensions rounded up to fit into whole bytes. */+ public int getDiscreteDimensions(int dimensions) {+ if (queryBits == bits) {+ int totalBits = dimensions * bitsPerDim;+ return (totalBits + 7) / 8 * 8 / bitsPerDim;+ }+ int queryDiscretized = (dimensions * queryBitsPerDim + 7) / 8 * 8 / queryBitsPerDim;+ int docDiscretized = (dimensions * bitsPerDim + 7) / 8 * 8 / bitsPerDim;+ int maxDiscretized = Math.max(queryDiscretized, docDiscretized);+ assert maxDiscretized % (8.0 / queryBitsPerDim) == 0+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;+ assert maxDiscretized % (8.0 / bitsPerDim) == 0+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;+ return maxDiscretized;+ }++ /** Return the number of dimensions that can be packed into a single byte. */+ public int getDocBitsPerDim() {+ return this.bitsPerDim;+ }++ public int getQueryBitsPerDim() {+ return this.queryBitsPerDim;+ }++ /** Return the number of bytes required to store a packed vector of the given dimensions. */+ public int getDocPackedLength(int dimensions) {+ int discretized = getDiscreteDimensions(dimensions);+ // how many bytes do we need to store the quantized vector?+ int totalBits = discretized * bitsPerDim;+ return (totalBits + 7) / 8;+ }++ public int getQueryPackedLength(int dimensions) {+ int discretized = getDiscreteDimensions(dimensions);+ // how many bytes do we need to store the quantized vector?+ int totalBits = discretized * queryBitsPerDim;+ return (totalBits + 7) / 8;+ }- public abstract float getScoreCorrectionConstant(int ord) throws IOException;+ /** Returns the encoding for the given wire number, or empty if unknown. */+ public static Optional<ScalarEncoding> fromWireNumber(int wireNumber) {+ for (ScalarEncoding encoding : values()) {+ if (encoding.wireNumber == wireNumber) {+ return Optional.of(encoding);+ }+ }+ return Optional.empty();+ }+ }/**- * Return a {@link VectorScorer} for the given query vector.+ * Retrieve the corrective terms for the given vector ordinal. For the dot-product family of+ * distances, the corrective terms are, in order+ *+ * <ul>+ * <li>the lower optimized interval+ * <li>the upper optimized interval+ * <li>the dot-product of the non-centered vector with the centroid+ * <li>the sum of quantized components+ * </ul>+ *+ * For euclidean:*- * @param query the query vector- * @return a {@link VectorScorer} instance or null+ * <ul>+ * <li>the lower optimized interval+ * <li>the upper optimized interval+ * <li>the l2norm of the centered vector+ * <li>the sum of quantized components+ * </ul>+ *+ * @param vectorOrd the vector ordinal+ * @return the corrective terms+ * @throws IOException if an I/O error occurs*/- public VectorScorer scorer(float[] query) throws IOException {- throw new UnsupportedOperationException();- }+ public abstract OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int vectorOrd)+ throws IOException;- @Override- public QuantizedByteVectorValues copy() throws IOException {- return this;- }+ /**+ * @return the quantizer used to quantize the vectors+ */+ public abstract OptimizedScalarQuantizer getQuantizer();++ /**+ * @return the scalar encoding used to pack the stored vectors.+ */+ public abstract ScalarEncoding getScalarEncoding();++ /**+ * @return the centroid used to center the vectors prior to quantization+ */+ public abstract float[] getCentroid() throws IOException;++ /**+ * @return the dot product of the centroid.+ */+ public abstract float getCentroidDP() throws IOException;@Override- public IndexInput getSlice() {- return null;- }+ public abstract QuantizedByteVectorValues copy() throws IOException;}diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.javaindex 7c49b1f10d8..3bc264fd91a 100644--- a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.java@@ -27,7 +27,7 @@ import org.apache.lucene.util.Accountable;*/public interface QuantizedVectorsReader extends Closeable, Accountable {- QuantizedByteVectorValues getQuantizedVectorValues(String fieldName) throws IOException;+ BaseQuantizedByteVectorValues getQuantizedVectorValues(String fieldName) throws IOException;ScalarQuantizer getQuantizationState(String fieldName);}diff --git a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.javaindex ade4a248c0d..c8a9efb07ac 100644--- a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java+++ b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java@@ -26,7 +26,7 @@ import org.apache.lucene.index.KnnVectorValues;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.BaseQuantizedByteVectorValues;public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorer {@@ -65,7 +65,7 @@ public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorerprivate RandomVectorScorerSupplier getByteScorerSupplier(ByteVectorValues vectorValues, VectorSimilarityFunction similarityType) throws IOException {// a quantized values here is a wrapping or delegation issue- assert !(vectorValues instanceof QuantizedByteVectorValues);+ assert !(vectorValues instanceof BaseQuantizedByteVectorValues);// currently only supports binary vectorsif (vectorValues instanceof ByteVectorValues bvv&& bvv instanceof HasIndexSlice byteVectorValues@@ -104,7 +104,7 @@ public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorerthrows IOException {FlatVectorsScorer.checkDimensions(queryVector.length, vectorValues.dimension());// a quantized values here is a wrapping or delegation issue- assert !(vectorValues instanceof QuantizedByteVectorValues);+ assert !(vectorValues instanceof BaseQuantizedByteVectorValues);if (vectorValues instanceof ByteVectorValues bvv&& bvv instanceof HasIndexSlice byteVectorValues&& byteVectorValues.getSlice() != null) {diff --git a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentScalarQuantizedVectorScorer.java b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentScalarQuantizedVectorScorer.javaindex ad97360fee4..98e7bd9ed5c 100644--- a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentScalarQuantizedVectorScorer.java+++ b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentScalarQuantizedVectorScorer.java@@ -35,7 +35,7 @@ import org.apache.lucene.util.VectorUtil;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;import org.apache.lucene.util.quantization.ScalarQuantizer;class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsScorer {@@ -51,7 +51,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsScopublic RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)throws IOException {- if (vectorValues instanceof QuantizedByteVectorValues quantized+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantized&& quantized.getSlice() instanceof MemorySegmentAccessInput input) {return new RandomVectorScorerSupplierImpl(similarityFunction, quantized, input);}@@ -62,7 +62,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsScopublic RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)throws IOException {- if (vectorValues instanceof QuantizedByteVectorValues quantized+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantized&& quantized.getSlice() instanceof MemorySegmentAccessInput input) {if (Constants.NATIVE_DOT_PRODUCT_ENABLED) {return new NativeRandomVectorScorerImpl(similarityFunction, quantized, input, target);@@ -98,7 +98,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsScoRandomVectorScorerBase(VectorSimilarityFunction similarityFunction,- QuantizedByteVectorValues values,+ LegacyQuantizedByteVectorValues values,MemorySegmentAccessInput input) {super(values);@@ -208,7 +208,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsScoRandomVectorScorerImpl(VectorSimilarityFunction similarityFunction,- QuantizedByteVectorValues values,+ LegacyQuantizedByteVectorValues values,MemorySegmentAccessInput input,float[] target) {super(similarityFunction, values, input);@@ -255,7 +255,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsScoprivate static class NativeRandomVectorScorerImpl extends RandomVectorScorerImpl {NativeRandomVectorScorerImpl(VectorSimilarityFunction similarityFunction,- QuantizedByteVectorValues values,+ LegacyQuantizedByteVectorValues values,MemorySegmentAccessInput input,float[] target) {super(similarityFunction, values, input, target);@@ -294,7 +294,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsScoprivate record RandomVectorScorerSupplierImpl(VectorSimilarityFunction similarityFunction,- QuantizedByteVectorValues values,+ LegacyQuantizedByteVectorValues values,MemorySegmentAccessInput input)implements RandomVectorScorerSupplier {@@ -319,7 +319,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsScoUpdateableRandomVectorScorerImpl(VectorSimilarityFunction similarityFunction,- QuantizedByteVectorValues values,+ LegacyQuantizedByteVectorValues values,MemorySegmentAccessInput input) {super(similarityFunction, values, input);}@@ -373,7 +373,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsScoNativeUpdateableRandomVectorScorerImpl(VectorSimilarityFunction similarityFunction,- QuantizedByteVectorValues values,+ LegacyQuantizedByteVectorValues values,MemorySegmentAccessInput input) {super(similarityFunction, values, input);}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.javaindex 40a38497ba9..0ec1689c36e 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java@@ -29,7 +29,6 @@ import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.FilterCodec;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.KnnVectorsReader;-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.document.Document;@@ -50,6 +49,7 @@ import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.SameThreadExecutorService;import org.apache.lucene.util.VectorUtil;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;import org.junit.Before;public class TestLucene104HnswScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.javaindex 08a96b54bae..7825d92706e 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java@@ -28,7 +28,6 @@ import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.FilterCodec;import org.apache.lucene.codecs.KnnVectorsFormat;-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;import org.apache.lucene.document.Document;import org.apache.lucene.document.KnnFloatVectorField;@@ -52,6 +51,8 @@ import org.apache.lucene.store.IndexOutput;import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;import org.junit.Before;public class TestLucene104ScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.javaindex 9cc3bef3a16..7779f9f23fd 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java@@ -23,7 +23,6 @@ import java.util.Objects;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;@@ -39,6 +38,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.TestVectorUtil;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;import org.junit.Before;import org.junit.Test;@@ -75,7 +75,7 @@ public class TestFullPrecisionFloatVectorSimilarityValuesSource extends LuceneTeprivate KnnVectorsFormat getKnnFormat(int bits) {return new Lucene104HnswScalarQuantizedVectorsFormat(- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.fromNumBits(bits),+ ScalarEncoding.fromNumBits(bits),Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,1,diff --git a/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java b/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.javaindex cb0a0add706..510008a4b31 100644--- a/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java+++ b/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java@@ -24,7 +24,6 @@ import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.deQuaimport static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.packAsBinary;import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.unpackBinary;-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.util.VectorUtil;@@ -182,7 +181,8 @@ public class TestOptimizedScalarQuantizer extends LuceneTestCase {public void testUnpackBinary() {int dim = randomIntBetween(1, 4096);- ScalarEncoding encoding = ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE;+ QuantizedByteVectorValues.ScalarEncoding encoding =+ QuantizedByteVectorValues.ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE;byte[] scratch = new byte[encoding.getDiscreteDimensions(dim)];for (int i = 0; i < scratch.length; i++) {scratch[i] = randomBoolean() ? (byte) 1 : (byte) 0;@@ -196,7 +196,8 @@ public class TestOptimizedScalarQuantizer extends LuceneTestCase {public void testPackTransposeDibit() {int dim = randomIntBetween(1, 4096);- ScalarEncoding encoding = ScalarEncoding.DIBIT_QUERY_NIBBLE;+ QuantizedByteVectorValues.ScalarEncoding encoding =+ QuantizedByteVectorValues.ScalarEncoding.DIBIT_QUERY_NIBBLE;byte[] scratch = new byte[encoding.getDiscreteDimensions(dim)];for (int i = 0; i < scratch.length; i++) {scratch[i] = (byte) randomIntBetween(0, 3);diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.javaindex d0794bf9cc7..8e65a825407 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java@@ -34,7 +34,6 @@ import org.apache.lucene.codecs.PointsWriter;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;import org.apache.lucene.codecs.lucene90.Lucene90PointsReader;import org.apache.lucene.codecs.lucene90.Lucene90PointsWriter;@@ -62,6 +61,7 @@ import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.IORunnable;import org.apache.lucene.util.bkd.BKDConfig;import org.apache.lucene.util.bkd.BKDWriter;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;/*** Codec that assigns per-field random postings formats.@@ -275,21 +275,21 @@ public class RandomCodec extends AssertingCodec {concurrentKnnMerging ? ForkJoinPool.commonPool() : null,0),new Lucene104HnswScalarQuantizedVectorsFormat(- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT,+ QuantizedByteVectorValues.ScalarEncoding.SEVEN_BIT,TestUtil.nextInt(random, 5, 50),TestUtil.nextInt(random, 10, 50),concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,concurrentKnnMerging ? ForkJoinPool.commonPool() : null,0),new Lucene104HnswScalarQuantizedVectorsFormat(- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.UNSIGNED_BYTE,+ QuantizedByteVectorValues.ScalarEncoding.UNSIGNED_BYTE,TestUtil.nextInt(random, 5, 50),TestUtil.nextInt(random, 10, 50),concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,concurrentKnnMerging ? ForkJoinPool.commonPool() : null,0),new Lucene104HnswScalarQuantizedVectorsFormat(- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.PACKED_NIBBLE,+ QuantizedByteVectorValues.ScalarEncoding.PACKED_NIBBLE,TestUtil.nextInt(random, 5, 50),TestUtil.nextInt(random, 10, 50),concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,