Apache Lucene open-source search software
Extend Lucene104ScalarQuantization format to support quant asymmetry (#15271)
Benjamin Trent602bfbd9af0e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex aec32100577..51a45666dc3 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -156,6 +156,11 @@ New Features`Lucene104HnswScalarQuantizedVectorsFormat` replaces the now legacy `Lucene99HnswScalarQuantizedVectorsFormat`(Trevor McCulloch)+ * GITHUB#15271: Extend `Lucene104ScalarQuantizedVectorsFormat` and `Lucene104HnswScalarQuantizedVectorsFormat` to+ allow asymmetric quantization. The initially supported bits are single bit with 4 bit queries. This is a replacement+ for the now legacy `Lucene102HnswBinaryQuantizedVectorsFormat` and `Lucene102BinaryQuantizedVectorsFormat`.+ (Ben Trent)+Improvements---------------------* GITHUB#15148: Add support uint8 distance and allow 8 bit scalar quantization (Trevor McCulloch)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.javaindex 7d49183249e..e15e361759e 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java@@ -86,6 +86,7 @@ public class Lucene104HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat/*** Constructs a format using the given graph construction parameters and scalar quantization.*+ * @param encoding the quantization encoding used to encode the vectors* @param maxConn the maximum number of connections to a node in the HNSW graph* @param beamWidth the size of the queue maintained during graph construction.* @param numMergeWorkers number of workers (threads) that will be used when doing merge. Ifdiff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.javaindex a2072a29788..db2021a2a0e 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java@@ -64,10 +64,15 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {if (vectorValues instanceof QuantizedByteVectorValues qv) {checkDimensions(target.length, qv.dimension());OptimizedScalarQuantizer quantizer = qv.getQuantizer();- byte[] targetQuantized =- new byte- [OptimizedScalarQuantizer.discretize(- target.length, qv.getScalarEncoding().getDimensionsPerByte())];+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding scalarEncoding = qv.getScalarEncoding();+ byte[] scratch = new byte[scalarEncoding.getDiscreteDimensions(qv.dimension())];+ final byte[] targetQuantized;+ if (scalarEncoding.isAsymmetric() == false) {+ targetQuantized = scratch;+ } else {+ // This is asymmetric quantization, we will pack the vector+ targetQuantized = new byte[scalarEncoding.getQueryPackedLength(scratch.length)];+ }// We make a copy as the quantization process mutates the inputfloat[] copy = ArrayUtil.copyOfSubArray(target, 0, target.length);if (similarityFunction == COSINE) {@@ -76,7 +81,12 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {target = copy;var targetCorrectiveTerms =quantizer.scalarQuantize(- target, targetQuantized, qv.getScalarEncoding().getBits(), qv.getCentroid());+ target, scratch, scalarEncoding.getQueryBits(), qv.getCentroid());+ // for single bit query nibble, we need to transpose the nibbles for fast scoring comparisons+ if (scalarEncoding+ == Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE) {+ OptimizedScalarQuantizer.transposeHalfByte(scratch, targetQuantized);+ }return new RandomVectorScorer.AbstractRandomVectorScorer(qv) {@Overridepublic float score(int node) throws IOException {@@ -96,6 +106,14 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target);}+ RandomVectorScorerSupplier getRandomVectorScorerSupplier(+ VectorSimilarityFunction similarityFunction,+ QuantizedByteVectorValues scoringVectors,+ QuantizedByteVectorValues targetVectors) {+ return new AsymmetricQuantizedRandomVectorScorerSupplier(+ scoringVectors, targetVectors, similarityFunction);+ }+@Overridepublic String toString() {return "Lucene104ScalarQuantizedVectorScorer(nonQuantizedDelegate="@@ -103,6 +121,53 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {+ ")";}+ static class AsymmetricQuantizedRandomVectorScorerSupplier implements RandomVectorScorerSupplier {+ private final QuantizedByteVectorValues queryVectors;+ private final QuantizedByteVectorValues targetVectors;+ private final VectorSimilarityFunction similarityFunction;++ AsymmetricQuantizedRandomVectorScorerSupplier(+ QuantizedByteVectorValues queryVectors,+ QuantizedByteVectorValues targetVectors,+ VectorSimilarityFunction similarityFunction) {+ assert targetVectors.getScalarEncoding().isAsymmetric();+ this.queryVectors = queryVectors;+ this.targetVectors = targetVectors;+ this.similarityFunction = similarityFunction;+ }++ @Override+ public UpdateableRandomVectorScorer scorer() throws IOException {+ final QuantizedByteVectorValues targetVectors = this.targetVectors.copy();+ final QuantizedByteVectorValues queryVectors = this.queryVectors.copy();+ return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(targetVectors) {+ private OptimizedScalarQuantizer.QuantizationResult queryCorrections = null;+ private byte[] vector = null;++ @Override+ public void setScoringOrdinal(int node) throws IOException {+ vector = queryVectors.vectorValue(node);+ queryCorrections = queryVectors.getCorrectiveTerms(node);+ }++ @Override+ public float score(int node) throws IOException {+ if (vector == null || queryCorrections == null) {+ throw new IllegalStateException("setScoringOrdinal was not called");+ }++ return quantizedScore(vector, queryCorrections, targetVectors, node, similarityFunction);+ }+ };+ }++ @Override+ public RandomVectorScorerSupplier copy() throws IOException {+ return new AsymmetricQuantizedRandomVectorScorerSupplier(+ queryVectors.copy(), targetVectors.copy(), similarityFunction);+ }+ }+private static final class ScalarQuantizedVectorScorerSupplierimplements RandomVectorScorerSupplier {private final QuantizedByteVectorValues targetValues;@@ -111,6 +176,7 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {public ScalarQuantizedVectorScorerSupplier(QuantizedByteVectorValues values, VectorSimilarityFunction similarity) throws IOException {+ assert values.getScalarEncoding().isAsymmetric() == false;this.targetValues = values.copy();this.values = values;this.similarity = similarity;@@ -131,14 +197,17 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {public void setScoringOrdinal(int node) throws IOException {var rawTargetVector = targetValues.vectorValue(node);switch (values.getScalarEncoding()) {- case UNSIGNED_BYTE -> targetVector = rawTargetVector;- case SEVEN_BIT -> targetVector = rawTargetVector;+ case UNSIGNED_BYTE, SEVEN_BIT -> targetVector = rawTargetVector;case PACKED_NIBBLE -> {if (targetVector == null) {targetVector = new byte[OptimizedScalarQuantizer.discretize(values.dimension(), 2)];}OffHeapScalarQuantizedVectorValues.unpackNibbles(rawTargetVector, targetVector);}+ case SINGLE_BIT_QUERY_NIBBLE -> {+ throw new IllegalStateException(+ "SINGLE_BIT_QUERY_NIBBLE encoding is not supported for symmetric quantization");+ }}targetCorrectiveTerms = targetValues.getCorrectiveTerms(node);}@@ -177,16 +246,19 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {case UNSIGNED_BYTE -> VectorUtil.uint8DotProduct(quantizedQuery, quantizedDoc);case SEVEN_BIT -> VectorUtil.dotProduct(quantizedQuery, quantizedDoc);case PACKED_NIBBLE -> VectorUtil.int4DotProductSinglePacked(quantizedQuery, quantizedDoc);+ case SINGLE_BIT_QUERY_NIBBLE ->+ VectorUtil.int4BitDotProduct(quantizedQuery, quantizedDoc);};OptimizedScalarQuantizer.QuantizationResult indexCorrections =targetVectors.getCorrectiveTerms(targetOrd);+ float queryScale = SCALE_LUT[scalarEncoding.getQueryBits() - 1];float scale = SCALE_LUT[scalarEncoding.getBits() - 1];float x1 = indexCorrections.quantizedComponentSum();float ax = indexCorrections.lowerInterval();// Here we must scale according to the bitsfloat lx = (indexCorrections.upperInterval() - ax) * scale;float ay = queryCorrections.lowerInterval();- float ly = (queryCorrections.upperInterval() - ay) * scale;+ float ly = (queryCorrections.upperInterval() - ay) * queryScale;float y1 = queryCorrections.quantizedComponentSum();float score =ax * ay * targetVectors.dimension() + ay * lx * x1 + ax * ly * y1 + lx * ly * qcDist;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.javaindex 44e1ef092c1..8015751fa42 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java@@ -118,9 +118,9 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {*/public enum ScalarEncoding {/** Each dimension is quantized to 8 bits and treated as an unsigned value. */- UNSIGNED_BYTE(0, (byte) 8, 1),+ UNSIGNED_BYTE(0, (byte) 8, 8),/** Each dimension is quantized to 4 bits two values are packed into each output byte. */- PACKED_NIBBLE(1, (byte) 4, 2),+ PACKED_NIBBLE(1, (byte) 4, 4),/*** Each dimension is quantized to 7 bits and treated as a signed value.*@@ -128,7 +128,15 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {* This setting will produce an index the same size as {@link #UNSIGNED_BYTE} but will produce* less accurate vector comparisons.*/- SEVEN_BIT(2, (byte) 7, 1);+ SEVEN_BIT(2, (byte) 7, 8),+ /**+ * Each dimension is quantized to a single bit and packed into bytes. During query time, the+ * query vector is quantized to 4 bits per dimension.+ *+ * <p>This is the most space efficient encoding, and will produce an index 8x smaller than+ * {@link #UNSIGNED_BYTE}. However, this comes at the cost of accuracy.+ */+ SINGLE_BIT_QUERY_NIBBLE(3, (byte) 1, 1, (byte) 4, 4);public static ScalarEncoding fromNumBits(int bits) {for (ScalarEncoding encoding : values()) {@@ -142,13 +150,27 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {/** The number used to identify this encoding on the wire, rather than relying on ordinal. */private final int wireNumber;- private final byte bits;- private final int dimsPerByte;+ private final byte bits, queryBits;+ private final int bitsPerDim, queryBitsPerDim;- ScalarEncoding(int wireNumber, byte bits, int dimsPerByte) {+ ScalarEncoding(int wireNumber, byte bits, int bitsPerDim) {this.wireNumber = wireNumber;this.bits = bits;- this.dimsPerByte = dimsPerByte;+ this.queryBits = bits;+ this.bitsPerDim = bitsPerDim;+ this.queryBitsPerDim = bitsPerDim;+ }++ ScalarEncoding(int wireNumber, byte bits, int bitsPerDim, byte queryBits, int queryBitsPerDim) {+ this.wireNumber = wireNumber;+ this.bits = bits;+ this.queryBits = queryBits;+ this.bitsPerDim = bitsPerDim;+ this.queryBitsPerDim = queryBitsPerDim;+ }++ boolean isAsymmetric() {+ return bits != queryBits;}int getWireNumber() {@@ -160,14 +182,48 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {return bits;}+ public byte getQueryBits() {+ return queryBits;+ }++ /** Return the number of dimensions rounded up to fit into whole bytes. */+ public int getDiscreteDimensions(int dimensions) {+ if (queryBits == bits) {+ int totalBits = dimensions * bitsPerDim;+ return (totalBits + 7) / 8 * 8 / bitsPerDim;+ }+ int queryDiscretized = (dimensions * queryBitsPerDim + 7) / 8 * 8 / queryBitsPerDim;+ int docDiscretized = (dimensions * bitsPerDim + 7) / 8 * 8 / bitsPerDim;+ int maxDiscretized = Math.max(queryDiscretized, docDiscretized);+ assert maxDiscretized % (8.0 / queryBitsPerDim) == 0+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;+ assert maxDiscretized % (8.0 / bitsPerDim) == 0+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;+ return maxDiscretized;+ }+/** Return the number of dimensions that can be packed into a single byte. */- public int getDimensionsPerByte() {- return this.dimsPerByte;+ public int getDocBitsPerDim() {+ return this.bitsPerDim;+ }++ public int getQueryBitsPerDim() {+ return this.queryBitsPerDim;}/** Return the number of bytes required to store a packed vector of the given dimensions. */- public int getPackedLength(int dimensions) {- return (dimensions + this.dimsPerByte - 1) / this.dimsPerByte;+ public int getDocPackedLength(int dimensions) {+ int discretized = getDiscreteDimensions(dimensions);+ // how many bytes do we need to store the quantized vector?+ int totalBits = discretized * bitsPerDim;+ return (totalBits + 7) / 8;+ }++ public int getQueryPackedLength(int dimensions) {+ int discretized = getDiscreteDimensions(dimensions);+ // how many bytes do we need to store the quantized vector?+ int totalBits = discretized * queryBitsPerDim;+ return (totalBits + 7) / 8;}/** Returns the encoding for the given wire number, or empty if unknown. */@@ -186,7 +242,7 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {this(ScalarEncoding.UNSIGNED_BYTE);}- /** Creates a new instance with the chosen encoding. */+ /** Creates a new instance with the chosen quantization encoding. */public Lucene104ScalarQuantizedVectorsFormat(ScalarEncoding encoding) {super(NAME);this.encoding = encoding;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.javaindex 8012d6095c2..f98c0f630c8 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java@@ -141,7 +141,7 @@ class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReaderlong numQuantizedVectorBytes =Math.multiplyExact(- (fieldEntry.scalarEncoding.getPackedLength(dimension)+ (fieldEntry.scalarEncoding.getDocPackedLength(dimension)+ (Float.BYTES * 3)+ Integer.BYTES),(long) fieldEntry.size);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.javaindex 4238aed0360..69d23bc95df 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java@@ -21,6 +21,8 @@ import static org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsimport static org.apache.lucene.index.VectorSimilarityFunction.COSINE;import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance;+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.packAsBinary;+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.transposeHalfByte;import java.io.Closeable;import java.io.IOException;@@ -190,22 +192,21 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {FieldWriter fieldData, float[] clusterCenter, OptimizedScalarQuantizer scalarQuantizer)throws IOException {byte[] scratch =- new byte- [OptimizedScalarQuantizer.discretize(- fieldData.fieldInfo.getVectorDimension(), encoding.getDimensionsPerByte())];+ new byte[encoding.getDiscreteDimensions(fieldData.fieldInfo.getVectorDimension())];byte[] vector =switch (encoding) {- case UNSIGNED_BYTE -> scratch;- case SEVEN_BIT -> scratch;- case PACKED_NIBBLE ->- new byte[encoding.getPackedLength(fieldData.fieldInfo.getVectorDimension())];+ case UNSIGNED_BYTE, SEVEN_BIT -> scratch;+ case PACKED_NIBBLE, SINGLE_BIT_QUERY_NIBBLE ->+ new byte[encoding.getDocPackedLength(scratch.length)];};for (int i = 0; i < fieldData.getVectors().size(); i++) {float[] v = fieldData.getVectors().get(i);OptimizedScalarQuantizer.QuantizationResult corrections =scalarQuantizer.scalarQuantize(v, scratch, encoding.getBits(), clusterCenter);- if (encoding == ScalarEncoding.PACKED_NIBBLE) {- OffHeapScalarQuantizedVectorValues.packNibbles(scratch, vector);+ switch (encoding) {+ case PACKED_NIBBLE -> OffHeapScalarQuantizedVectorValues.packNibbles(scratch, vector);+ case SINGLE_BIT_QUERY_NIBBLE -> OptimizedScalarQuantizer.packAsBinary(scratch, vector);+ case UNSIGNED_BYTE, SEVEN_BIT -> {}}vectorData.writeBytes(vector, vector.length);vectorData.writeInt(Float.floatToIntBits(corrections.lowerInterval()));@@ -251,22 +252,21 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {OptimizedScalarQuantizer scalarQuantizer)throws IOException {byte[] scratch =- new byte- [OptimizedScalarQuantizer.discretize(- fieldData.fieldInfo.getVectorDimension(), encoding.getDimensionsPerByte())];+ new byte[encoding.getDiscreteDimensions(fieldData.fieldInfo.getVectorDimension())];byte[] vector =switch (encoding) {- case UNSIGNED_BYTE -> scratch;- case SEVEN_BIT -> scratch;- case PACKED_NIBBLE ->- new byte[encoding.getPackedLength(fieldData.fieldInfo.getVectorDimension())];+ case UNSIGNED_BYTE, SEVEN_BIT -> scratch;+ case PACKED_NIBBLE, SINGLE_BIT_QUERY_NIBBLE ->+ new byte[encoding.getDocPackedLength(scratch.length)];};for (int ordinal : ordMap) {float[] v = fieldData.getVectors().get(ordinal);OptimizedScalarQuantizer.QuantizationResult corrections =scalarQuantizer.scalarQuantize(v, scratch, encoding.getBits(), clusterCenter);- if (encoding == ScalarEncoding.PACKED_NIBBLE) {- OffHeapScalarQuantizedVectorValues.packNibbles(scratch, vector);+ switch (encoding) {+ case PACKED_NIBBLE -> OffHeapScalarQuantizedVectorValues.packNibbles(scratch, vector);+ case SINGLE_BIT_QUERY_NIBBLE -> OptimizedScalarQuantizer.packAsBinary(scratch, vector);+ case UNSIGNED_BYTE, SEVEN_BIT -> {}}vectorData.writeBytes(vector, vector.length);vectorData.writeInt(Float.floatToIntBits(corrections.lowerInterval()));@@ -385,6 +385,53 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {return docsWithField;}+ static DocsWithFieldSet writeBinarizedVectorAndQueryData(+ IndexOutput binarizedVectorData,+ ScalarEncoding encoding,+ IndexOutput binarizedQueryData,+ FloatVectorValues floatVectorValues,+ float[] centroid,+ OptimizedScalarQuantizer binaryQuantizer)+ throws IOException {+ if (encoding.isAsymmetric() == false) {+ throw new IllegalArgumentException("encoding and queryEncoding must be different");+ }+ DocsWithFieldSet docsWithField = new DocsWithFieldSet();+ int discretizedDims = encoding.getDiscreteDimensions(floatVectorValues.dimension());+ byte[][] quantizationScratch = new byte[2][];+ quantizationScratch[0] = new byte[discretizedDims];+ quantizationScratch[1] = new byte[discretizedDims];+ byte[] toIndex = new byte[encoding.getDocPackedLength(discretizedDims)];+ byte[] toQuery = new byte[encoding.getQueryPackedLength(discretizedDims)];+ KnnVectorValues.DocIndexIterator iterator = floatVectorValues.iterator();+ for (int docV = iterator.nextDoc(); docV != NO_MORE_DOCS; docV = iterator.nextDoc()) {+ // write index vector+ OptimizedScalarQuantizer.QuantizationResult[] r =+ binaryQuantizer.multiScalarQuantize(+ floatVectorValues.vectorValue(iterator.index()),+ quantizationScratch,+ new byte[] {encoding.getBits(), encoding.getQueryBits()},+ centroid);+ // pack and store document bit vector+ packAsBinary(quantizationScratch[0], toIndex);+ binarizedVectorData.writeBytes(toIndex, toIndex.length);+ binarizedVectorData.writeInt(Float.floatToIntBits(r[0].lowerInterval()));+ binarizedVectorData.writeInt(Float.floatToIntBits(r[0].upperInterval()));+ binarizedVectorData.writeInt(Float.floatToIntBits(r[0].additionalCorrection()));+ binarizedVectorData.writeInt(r[0].quantizedComponentSum());+ docsWithField.add(docV);++ // pack and store the 4bit query vector+ transposeHalfByte(quantizationScratch[1], toQuery);+ binarizedQueryData.writeBytes(toQuery, toQuery.length);+ binarizedQueryData.writeInt(Float.floatToIntBits(r[1].lowerInterval()));+ binarizedQueryData.writeInt(Float.floatToIntBits(r[1].upperInterval()));+ binarizedQueryData.writeInt(Float.floatToIntBits(r[1].additionalCorrection()));+ binarizedQueryData.writeInt(r[1].quantizedComponentSum());+ }+ return docsWithField;+ }+@Overridepublic CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(FieldInfo fieldInfo, MergeState mergeState) throws IOException {@@ -417,7 +464,9 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {throws IOException {long vectorDataOffset = vectorData.alignFilePointer(Float.BYTES);IndexOutput tempQuantizedVectorData = null;+ IndexOutput tempScoreQuantizedVectorData = null;IndexInput quantizedDataInput = null;+ IndexInput quantizedScoreDataInput = null;OptimizedScalarQuantizer quantizer =new OptimizedScalarQuantizer(fieldInfo.getVectorSimilarityFunction());try {@@ -425,15 +474,32 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {segmentWriteState.directory.createTempOutput(vectorData.getName(), "temp", segmentWriteState.context);final String tempQuantizedVectorName = tempQuantizedVectorData.getName();+ final String tempScoreQuantizedVectorName;+ if (encoding.isAsymmetric()) {+ tempScoreQuantizedVectorData =+ segmentWriteState.directory.createTempOutput(+ vectorData.getName() + "_score", "temp", segmentWriteState.context);+ tempScoreQuantizedVectorName = tempScoreQuantizedVectorData.getName();+ } else {+ tempScoreQuantizedVectorName = null;+ }FloatVectorValues floatVectorValues =MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState);if (fieldInfo.getVectorSimilarityFunction() == COSINE) {floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues);}DocsWithFieldSet docsWithField =- writeVectorData(- tempQuantizedVectorData,- new QuantizedFloatVectorValues(floatVectorValues, quantizer, encoding, centroid));+ encoding.isAsymmetric()+ ? writeBinarizedVectorAndQueryData(+ tempQuantizedVectorData,+ encoding,+ tempScoreQuantizedVectorData,+ floatVectorValues,+ centroid,+ quantizer)+ : writeVectorData(+ tempQuantizedVectorData,+ new QuantizedFloatVectorValues(floatVectorValues, quantizer, encoding, centroid));CodecUtil.writeFooter(tempQuantizedVectorData);IOUtils.close(tempQuantizedVectorData);quantizedDataInput =@@ -442,6 +508,13 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {quantizedDataInput, quantizedDataInput.length() - CodecUtil.footerLength());long vectorDataLength = vectorData.getFilePointer() - vectorDataOffset;CodecUtil.retrieveChecksum(quantizedDataInput);+ if (tempScoreQuantizedVectorData != null) {+ CodecUtil.writeFooter(tempScoreQuantizedVectorData);+ IOUtils.close(tempScoreQuantizedVectorData);+ quantizedScoreDataInput =+ segmentWriteState.directory.openInput(+ tempScoreQuantizedVectorName, segmentWriteState.context);+ }writeMeta(fieldInfo,segmentWriteState.segmentInfo.maxDoc(),@@ -452,8 +525,11 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {docsWithField);final IndexInput finalQuantizedDataInput = quantizedDataInput;+ final IndexInput finalQuantizedScoreDataInput = quantizedScoreDataInput;tempQuantizedVectorData = null;+ tempScoreQuantizedVectorData = null;quantizedDataInput = null;+ quantizedScoreDataInput = null;OffHeapScalarQuantizedVectorValues vectorValues =new OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues(@@ -466,23 +542,54 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {fieldInfo.getVectorSimilarityFunction(),vectorsScorer,finalQuantizedDataInput);+ OffHeapScalarQuantizedVectorValues scoreVectorValues = null;+ if (finalQuantizedScoreDataInput != null) {+ scoreVectorValues =+ new OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues(+ true,+ fieldInfo.getVectorDimension(),+ docsWithField.cardinality(),+ centroid,+ cDotC,+ quantizer,+ encoding,+ fieldInfo.getVectorSimilarityFunction(),+ vectorsScorer,+ finalQuantizedScoreDataInput);+ }RandomVectorScorerSupplier scorerSupplier =- vectorsScorer.getRandomVectorScorerSupplier(- fieldInfo.getVectorSimilarityFunction(), vectorValues);+ scoreVectorValues == null+ ? vectorsScorer.getRandomVectorScorerSupplier(+ fieldInfo.getVectorSimilarityFunction(), vectorValues)+ : vectorsScorer.getRandomVectorScorerSupplier(+ fieldInfo.getVectorSimilarityFunction(), scoreVectorValues, vectorValues);return new QuantizedCloseableRandomVectorScorerSupplier(scorerSupplier,vectorValues,() -> {- IOUtils.close(finalQuantizedDataInput);+ IOUtils.close(finalQuantizedDataInput, finalQuantizedScoreDataInput);+ if (tempScoreQuantizedVectorName != null) {+ IOUtils.deleteFilesIgnoringExceptions(+ segmentWriteState.directory, tempScoreQuantizedVectorName);+ }IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempQuantizedVectorName);});} catch (Throwable t) {- IOUtils.closeWhileSuppressingExceptions(t, tempQuantizedVectorData, quantizedDataInput);+ IOUtils.closeWhileSuppressingExceptions(+ t,+ tempQuantizedVectorData,+ tempScoreQuantizedVectorData,+ quantizedDataInput,+ quantizedScoreDataInput);if (tempQuantizedVectorData != null) {IOUtils.deleteFilesSuppressingExceptions(t, segmentWriteState.directory, tempQuantizedVectorData.getName());}+ if (tempScoreQuantizedVectorData != null) {+ IOUtils.deleteFilesSuppressingExceptions(+ t, segmentWriteState.directory, tempScoreQuantizedVectorData.getName());+ }throw t;}}@@ -687,15 +794,12 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {this.values = delegate;this.quantizer = quantizer;this.encoding = encoding;- this.quantized =- new byte- [OptimizedScalarQuantizer.discretize(- delegate.dimension(), encoding.getDimensionsPerByte())];+ this.quantized = new byte[encoding.getDiscreteDimensions(delegate.dimension())];this.packed =switch (encoding) {- case UNSIGNED_BYTE -> this.quantized;- case SEVEN_BIT -> this.quantized;- case PACKED_NIBBLE -> new byte[encoding.getPackedLength(delegate.dimension())];+ case UNSIGNED_BYTE, SEVEN_BIT -> this.quantized;+ case PACKED_NIBBLE, SINGLE_BIT_QUERY_NIBBLE ->+ new byte[encoding.getDocPackedLength(quantized.length)];};this.centroid = centroid;this.centroidDP = VectorUtil.dotProduct(centroid, centroid);@@ -734,7 +838,7 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {@Overridepublic ScalarEncoding getScalarEncoding() {- throw new UnsupportedOperationException();+ return encoding;}@Override@@ -766,8 +870,10 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {corrections =quantizer.scalarQuantize(values.vectorValue(ord), quantized, encoding.getBits(), centroid);- if (encoding == ScalarEncoding.PACKED_NIBBLE) {- OffHeapScalarQuantizedVectorValues.packNibbles(quantized, packed);+ switch (encoding) {+ case PACKED_NIBBLE -> OffHeapScalarQuantizedVectorValues.packNibbles(quantized, packed);+ case SINGLE_BIT_QUERY_NIBBLE -> OptimizedScalarQuantizer.packAsBinary(quantized, packed);+ case UNSIGNED_BYTE, SEVEN_BIT -> {}}}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.javaindex d2c678d8f8b..440e96d4041 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java@@ -53,6 +53,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVefinal ScalarEncoding encoding;final float[] centroid;final float centroidDp;+ final boolean isQuerySide;OffHeapScalarQuantizedVectorValues(int dimension,@@ -64,6 +65,32 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVeVectorSimilarityFunction similarityFunction,FlatVectorsScorer vectorsScorer,IndexInput slice) {+ this(+ false,+ dimension,+ size,+ centroid,+ centroidDp,+ quantizer,+ encoding,+ similarityFunction,+ vectorsScorer,+ slice);+ }++ OffHeapScalarQuantizedVectorValues(+ boolean isQuerySide,+ int dimension,+ int size,+ float[] centroid,+ float centroidDp,+ OptimizedScalarQuantizer quantizer,+ ScalarEncoding encoding,+ VectorSimilarityFunction similarityFunction,+ FlatVectorsScorer vectorsScorer,+ IndexInput slice) {+ assert isQuerySide == false || encoding.isAsymmetric();+ this.isQuerySide = isQuerySide;this.dimension = dimension;this.size = size;this.similarityFunction = similarityFunction;@@ -72,11 +99,15 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVethis.centroid = centroid;this.centroidDp = centroidDp;this.correctiveValues = new float[3];- this.byteSize = encoding.getPackedLength(dimension) + (Float.BYTES * 3) + Integer.BYTES;- this.byteBuffer = ByteBuffer.allocate(encoding.getPackedLength(dimension));+ this.encoding = encoding;+ int docPackedLength =+ isQuerySide+ ? encoding.getQueryPackedLength(dimension)+ : encoding.getDocPackedLength(dimension);+ this.byteSize = docPackedLength + (Float.BYTES * 3) + Integer.BYTES;+ this.byteBuffer = ByteBuffer.allocate(docPackedLength);this.vectorValue = byteBuffer.array();this.quantizer = quantizer;- this.encoding = encoding;}@Override@@ -114,7 +145,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVereturn new OptimizedScalarQuantizer.QuantizationResult(correctiveValues[0], correctiveValues[1], correctiveValues[2], quantizedComponentSum);}- slice.seek(((long) targetOrd * byteSize) + dimension);+ slice.seek(((long) targetOrd * byteSize) + vectorValue.length);slice.readFloats(correctiveValues, 0, 3);quantizedComponentSum = slice.readInt();return new OptimizedScalarQuantizer.QuantizationResult(@@ -229,9 +260,34 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVeslice);}+ DenseOffHeapVectorValues(+ boolean isQuerySide,+ int dimension,+ int size,+ float[] centroid,+ float centroidDp,+ OptimizedScalarQuantizer quantizer,+ ScalarEncoding encoding,+ VectorSimilarityFunction similarityFunction,+ FlatVectorsScorer vectorsScorer,+ IndexInput slice) {+ super(+ isQuerySide,+ dimension,+ size,+ centroid,+ centroidDp,+ quantizer,+ encoding,+ similarityFunction,+ vectorsScorer,+ slice);+ }+@Overridepublic OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues copy() throws IOException {return new OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues(+ isQuerySide,dimension,size,centroid,@@ -250,6 +306,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe@Overridepublic VectorScorer scorer(float[] target) throws IOException {+ assert isQuerySide == false;OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues copy = copy();DocIndexIterator iterator = copy.iterator();RandomVectorScorer scorer =@@ -304,6 +361,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVesimilarityFunction,vectorsScorer,slice);+ assert isQuerySide == false;this.configuration = configuration;this.dataIn = dataIn;this.ordToDoc = configuration.getDirectMonotonicReader(dataIn);@@ -312,6 +370,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe@Overridepublic SparseOffHeapVectorValues copy() throws IOException {+ assert isQuerySide == false;return new SparseOffHeapVectorValues(configuration,dimension,@@ -356,6 +415,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe@Overridepublic VectorScorer scorer(float[] target) throws IOException {+ assert isQuerySide == false;SparseOffHeapVectorValues copy = copy();DocIndexIterator iterator = copy.iterator();RandomVectorScorer scorer =@@ -389,6 +449,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVesimilarityFunction,vectorsScorer,null);+ assert isQuerySide == false;}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.javaindex 48d0c4e665f..02b9c074899 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java@@ -58,7 +58,7 @@ abstract class QuantizedByteVectorValues extends ByteVectorValues {public abstract OptimizedScalarQuantizer getQuantizer();/**- * @return the scalar encoding used to pack the vectors.+ * @return the scalar encoding used to pack the stored vectors.*/public abstract ScalarEncoding getScalarEncoding();diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.javaindex 6164b0062f0..10fc424c44e 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java@@ -30,6 +30,7 @@ import org.apache.lucene.codecs.FilterCodec;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.KnnVectorsReader;import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.document.Document;import org.apache.lucene.document.KnnFloatVectorField;@@ -47,14 +48,31 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.SameThreadExecutorService;+import org.junit.Before;public class TestLucene104HnswScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {- private static final KnnVectorsFormat FORMAT = new Lucene104HnswScalarQuantizedVectorsFormat();+ private KnnVectorsFormat format;+ private ScalarEncoding encoding;++ @Before+ @Override+ public void setUp() throws Exception {+ var encodingValues = ScalarEncoding.values();+ encoding = encodingValues[random().nextInt(encodingValues.length)];+ format =+ new Lucene104HnswScalarQuantizedVectorsFormat(+ encoding,+ Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,+ 1,+ null);+ super.setUp();+ }@Overrideprotected Codec getCodec() {- return TestUtil.alwaysKnnVectorsFormat(FORMAT);+ return TestUtil.alwaysKnnVectorsFormat(format);}public void testToString() {@@ -170,7 +188,7 @@ public class TestLucene104HnswScalarQuantizedVectorsFormat extends BaseKnnVectorassertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec"));assertEquals(1L, (long) offHeap.get("vex"));long corrections = Float.BYTES + Float.BYTES + Float.BYTES + Integer.BYTES;- long expected = fieldInfo.getVectorDimension() + corrections;+ long expected = encoding.getDocPackedLength(fieldInfo.getVectorDimension()) + corrections;assertEquals(expected, (long) offHeap.get("veq"));assertEquals(3, offHeap.size());}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.javaindex 29041b5b07f..150294df12c 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java@@ -166,9 +166,8 @@ public class TestLucene104ScalarQuantizedVectorsFormat extends BaseKnnVectorsForassertEquals(centroid.length, dims);OptimizedScalarQuantizer quantizer = new OptimizedScalarQuantizer(similarityFunction);- byte[] scratch =- new byte[OptimizedScalarQuantizer.discretize(dims, encoding.getDimensionsPerByte())];- byte[] expectedVector = new byte[encoding.getPackedLength(dims)];+ byte[] scratch = new byte[encoding.getDiscreteDimensions(dims)];+ byte[] expectedVector = new byte[encoding.getDocPackedLength(scratch.length)];if (similarityFunction == VectorSimilarityFunction.COSINE) {vectorValues =new Lucene104ScalarQuantizedVectorsWriter.NormalizedFloatVectorValues(vectorValues);@@ -183,10 +182,12 @@ public class TestLucene104ScalarQuantizedVectorsFormat extends BaseKnnVectorsForencoding.getBits(),centroid);switch (encoding) {- case UNSIGNED_BYTE -> System.arraycopy(scratch, 0, expectedVector, 0, dims);- case SEVEN_BIT -> System.arraycopy(scratch, 0, expectedVector, 0, dims);+ case UNSIGNED_BYTE, SEVEN_BIT ->+ System.arraycopy(scratch, 0, expectedVector, 0, dims);case PACKED_NIBBLE ->OffHeapScalarQuantizedVectorValues.packNibbles(scratch, expectedVector);+ case SINGLE_BIT_QUERY_NIBBLE ->+ OptimizedScalarQuantizer.packAsBinary(scratch, expectedVector);}assertArrayEquals(expectedVector, qvectorValues.vectorValue(docIndexIterator.index()));var actualCorrections = qvectorValues.getCorrectiveTerms(docIndexIterator.index());