Skip to content

Apache Lucene open-source search software

Extend Lucene104ScalarQuantization format to support quant asymmetry (#15271)

Benjamin Trent602bfbd9af0e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index aec32100577..51a45666dc3 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -156,6 +156,11 @@ New Features
`Lucene104HnswScalarQuantizedVectorsFormat` replaces the now legacy `Lucene99HnswScalarQuantizedVectorsFormat`
(Trevor McCulloch)
+ * GITHUB#15271: Extend `Lucene104ScalarQuantizedVectorsFormat` and `Lucene104HnswScalarQuantizedVectorsFormat` to
+ allow asymmetric quantization. The initially supported bits are single bit with 4 bit queries. This is a replacement
+ for the now legacy `Lucene102HnswBinaryQuantizedVectorsFormat` and `Lucene102BinaryQuantizedVectorsFormat`.
+ (Ben Trent)
+
Improvements
---------------------
* GITHUB#15148: Add support uint8 distance and allow 8 bit scalar quantization (Trevor McCulloch)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
index 7d49183249e..e15e361759e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
@@ -86,6 +86,7 @@ public class Lucene104HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat
/**
* Constructs a format using the given graph construction parameters and scalar quantization.
*
+ * @param encoding the quantization encoding used to encode the vectors
* @param maxConn the maximum number of connections to a node in the HNSW graph
* @param beamWidth the size of the queue maintained during graph construction.
* @param numMergeWorkers number of workers (threads) that will be used when doing merge. If
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
index a2072a29788..db2021a2a0e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
@@ -64,10 +64,15 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
if (vectorValues instanceof QuantizedByteVectorValues qv) {
checkDimensions(target.length, qv.dimension());
OptimizedScalarQuantizer quantizer = qv.getQuantizer();
- byte[] targetQuantized =
- new byte
- [OptimizedScalarQuantizer.discretize(
- target.length, qv.getScalarEncoding().getDimensionsPerByte())];
+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding scalarEncoding = qv.getScalarEncoding();
+ byte[] scratch = new byte[scalarEncoding.getDiscreteDimensions(qv.dimension())];
+ final byte[] targetQuantized;
+ if (scalarEncoding.isAsymmetric() == false) {
+ targetQuantized = scratch;
+ } else {
+ // This is asymmetric quantization, we will pack the vector
+ targetQuantized = new byte[scalarEncoding.getQueryPackedLength(scratch.length)];
+ }
// We make a copy as the quantization process mutates the input
float[] copy = ArrayUtil.copyOfSubArray(target, 0, target.length);
if (similarityFunction == COSINE) {
@@ -76,7 +81,12 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
target = copy;
var targetCorrectiveTerms =
quantizer.scalarQuantize(
- target, targetQuantized, qv.getScalarEncoding().getBits(), qv.getCentroid());
+ target, scratch, scalarEncoding.getQueryBits(), qv.getCentroid());
+ // for single bit query nibble, we need to transpose the nibbles for fast scoring comparisons
+ if (scalarEncoding
+ == Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE) {
+ OptimizedScalarQuantizer.transposeHalfByte(scratch, targetQuantized);
+ }
return new RandomVectorScorer.AbstractRandomVectorScorer(qv) {
@Override
public float score(int node) throws IOException {
@@ -96,6 +106,14 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target);
}
+ RandomVectorScorerSupplier getRandomVectorScorerSupplier(
+ VectorSimilarityFunction similarityFunction,
+ QuantizedByteVectorValues scoringVectors,
+ QuantizedByteVectorValues targetVectors) {
+ return new AsymmetricQuantizedRandomVectorScorerSupplier(
+ scoringVectors, targetVectors, similarityFunction);
+ }
+
@Override
public String toString() {
return "Lucene104ScalarQuantizedVectorScorer(nonQuantizedDelegate="
@@ -103,6 +121,53 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
+ ")";
}
+ static class AsymmetricQuantizedRandomVectorScorerSupplier implements RandomVectorScorerSupplier {
+ private final QuantizedByteVectorValues queryVectors;
+ private final QuantizedByteVectorValues targetVectors;
+ private final VectorSimilarityFunction similarityFunction;
+
+ AsymmetricQuantizedRandomVectorScorerSupplier(
+ QuantizedByteVectorValues queryVectors,
+ QuantizedByteVectorValues targetVectors,
+ VectorSimilarityFunction similarityFunction) {
+ assert targetVectors.getScalarEncoding().isAsymmetric();
+ this.queryVectors = queryVectors;
+ this.targetVectors = targetVectors;
+ this.similarityFunction = similarityFunction;
+ }
+
+ @Override
+ public UpdateableRandomVectorScorer scorer() throws IOException {
+ final QuantizedByteVectorValues targetVectors = this.targetVectors.copy();
+ final QuantizedByteVectorValues queryVectors = this.queryVectors.copy();
+ return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(targetVectors) {
+ private OptimizedScalarQuantizer.QuantizationResult queryCorrections = null;
+ private byte[] vector = null;
+
+ @Override
+ public void setScoringOrdinal(int node) throws IOException {
+ vector = queryVectors.vectorValue(node);
+ queryCorrections = queryVectors.getCorrectiveTerms(node);
+ }
+
+ @Override
+ public float score(int node) throws IOException {
+ if (vector == null || queryCorrections == null) {
+ throw new IllegalStateException("setScoringOrdinal was not called");
+ }
+
+ return quantizedScore(vector, queryCorrections, targetVectors, node, similarityFunction);
+ }
+ };
+ }
+
+ @Override
+ public RandomVectorScorerSupplier copy() throws IOException {
+ return new AsymmetricQuantizedRandomVectorScorerSupplier(
+ queryVectors.copy(), targetVectors.copy(), similarityFunction);
+ }
+ }
+
private static final class ScalarQuantizedVectorScorerSupplier
implements RandomVectorScorerSupplier {
private final QuantizedByteVectorValues targetValues;
@@ -111,6 +176,7 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
public ScalarQuantizedVectorScorerSupplier(
QuantizedByteVectorValues values, VectorSimilarityFunction similarity) throws IOException {
+ assert values.getScalarEncoding().isAsymmetric() == false;
this.targetValues = values.copy();
this.values = values;
this.similarity = similarity;
@@ -131,14 +197,17 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
public void setScoringOrdinal(int node) throws IOException {
var rawTargetVector = targetValues.vectorValue(node);
switch (values.getScalarEncoding()) {
- case UNSIGNED_BYTE -> targetVector = rawTargetVector;
- case SEVEN_BIT -> targetVector = rawTargetVector;
+ case UNSIGNED_BYTE, SEVEN_BIT -> targetVector = rawTargetVector;
case PACKED_NIBBLE -> {
if (targetVector == null) {
targetVector = new byte[OptimizedScalarQuantizer.discretize(values.dimension(), 2)];
}
OffHeapScalarQuantizedVectorValues.unpackNibbles(rawTargetVector, targetVector);
}
+ case SINGLE_BIT_QUERY_NIBBLE -> {
+ throw new IllegalStateException(
+ "SINGLE_BIT_QUERY_NIBBLE encoding is not supported for symmetric quantization");
+ }
}
targetCorrectiveTerms = targetValues.getCorrectiveTerms(node);
}
@@ -177,16 +246,19 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
case UNSIGNED_BYTE -> VectorUtil.uint8DotProduct(quantizedQuery, quantizedDoc);
case SEVEN_BIT -> VectorUtil.dotProduct(quantizedQuery, quantizedDoc);
case PACKED_NIBBLE -> VectorUtil.int4DotProductSinglePacked(quantizedQuery, quantizedDoc);
+ case SINGLE_BIT_QUERY_NIBBLE ->
+ VectorUtil.int4BitDotProduct(quantizedQuery, quantizedDoc);
};
OptimizedScalarQuantizer.QuantizationResult indexCorrections =
targetVectors.getCorrectiveTerms(targetOrd);
+ float queryScale = SCALE_LUT[scalarEncoding.getQueryBits() - 1];
float scale = SCALE_LUT[scalarEncoding.getBits() - 1];
float x1 = indexCorrections.quantizedComponentSum();
float ax = indexCorrections.lowerInterval();
// Here we must scale according to the bits
float lx = (indexCorrections.upperInterval() - ax) * scale;
float ay = queryCorrections.lowerInterval();
- float ly = (queryCorrections.upperInterval() - ay) * scale;
+ float ly = (queryCorrections.upperInterval() - ay) * queryScale;
float y1 = queryCorrections.quantizedComponentSum();
float score =
ax * ay * targetVectors.dimension() + ay * lx * x1 + ax * ly * y1 + lx * ly * qcDist;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
index 44e1ef092c1..8015751fa42 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
@@ -118,9 +118,9 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
*/
public enum ScalarEncoding {
/** Each dimension is quantized to 8 bits and treated as an unsigned value. */
- UNSIGNED_BYTE(0, (byte) 8, 1),
+ UNSIGNED_BYTE(0, (byte) 8, 8),
/** Each dimension is quantized to 4 bits two values are packed into each output byte. */
- PACKED_NIBBLE(1, (byte) 4, 2),
+ PACKED_NIBBLE(1, (byte) 4, 4),
/**
* Each dimension is quantized to 7 bits and treated as a signed value.
*
@@ -128,7 +128,15 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
* This setting will produce an index the same size as {@link #UNSIGNED_BYTE} but will produce
* less accurate vector comparisons.
*/
- SEVEN_BIT(2, (byte) 7, 1);
+ SEVEN_BIT(2, (byte) 7, 8),
+ /**
+ * Each dimension is quantized to a single bit and packed into bytes. During query time, the
+ * query vector is quantized to 4 bits per dimension.
+ *
+ * <p>This is the most space efficient encoding, and will produce an index 8x smaller than
+ * {@link #UNSIGNED_BYTE}. However, this comes at the cost of accuracy.
+ */
+ SINGLE_BIT_QUERY_NIBBLE(3, (byte) 1, 1, (byte) 4, 4);
public static ScalarEncoding fromNumBits(int bits) {
for (ScalarEncoding encoding : values()) {
@@ -142,13 +150,27 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
/** The number used to identify this encoding on the wire, rather than relying on ordinal. */
private final int wireNumber;
- private final byte bits;
- private final int dimsPerByte;
+ private final byte bits, queryBits;
+ private final int bitsPerDim, queryBitsPerDim;
- ScalarEncoding(int wireNumber, byte bits, int dimsPerByte) {
+ ScalarEncoding(int wireNumber, byte bits, int bitsPerDim) {
this.wireNumber = wireNumber;
this.bits = bits;
- this.dimsPerByte = dimsPerByte;
+ this.queryBits = bits;
+ this.bitsPerDim = bitsPerDim;
+ this.queryBitsPerDim = bitsPerDim;
+ }
+
+ ScalarEncoding(int wireNumber, byte bits, int bitsPerDim, byte queryBits, int queryBitsPerDim) {
+ this.wireNumber = wireNumber;
+ this.bits = bits;
+ this.queryBits = queryBits;
+ this.bitsPerDim = bitsPerDim;
+ this.queryBitsPerDim = queryBitsPerDim;
+ }
+
+ boolean isAsymmetric() {
+ return bits != queryBits;
}
int getWireNumber() {
@@ -160,14 +182,48 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
return bits;
}
+ public byte getQueryBits() {
+ return queryBits;
+ }
+
+ /** Return the number of dimensions rounded up to fit into whole bytes. */
+ public int getDiscreteDimensions(int dimensions) {
+ if (queryBits == bits) {
+ int totalBits = dimensions * bitsPerDim;
+ return (totalBits + 7) / 8 * 8 / bitsPerDim;
+ }
+ int queryDiscretized = (dimensions * queryBitsPerDim + 7) / 8 * 8 / queryBitsPerDim;
+ int docDiscretized = (dimensions * bitsPerDim + 7) / 8 * 8 / bitsPerDim;
+ int maxDiscretized = Math.max(queryDiscretized, docDiscretized);
+ assert maxDiscretized % (8.0 / queryBitsPerDim) == 0
+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;
+ assert maxDiscretized % (8.0 / bitsPerDim) == 0
+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;
+ return maxDiscretized;
+ }
+
/** Return the number of dimensions that can be packed into a single byte. */
- public int getDimensionsPerByte() {
- return this.dimsPerByte;
+ public int getDocBitsPerDim() {
+ return this.bitsPerDim;
+ }
+
+ public int getQueryBitsPerDim() {
+ return this.queryBitsPerDim;
}
/** Return the number of bytes required to store a packed vector of the given dimensions. */
- public int getPackedLength(int dimensions) {
- return (dimensions + this.dimsPerByte - 1) / this.dimsPerByte;
+ public int getDocPackedLength(int dimensions) {
+ int discretized = getDiscreteDimensions(dimensions);
+ // how many bytes do we need to store the quantized vector?
+ int totalBits = discretized * bitsPerDim;
+ return (totalBits + 7) / 8;
+ }
+
+ public int getQueryPackedLength(int dimensions) {
+ int discretized = getDiscreteDimensions(dimensions);
+ // how many bytes do we need to store the quantized vector?
+ int totalBits = discretized * queryBitsPerDim;
+ return (totalBits + 7) / 8;
}
/** Returns the encoding for the given wire number, or empty if unknown. */
@@ -186,7 +242,7 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
this(ScalarEncoding.UNSIGNED_BYTE);
}
- /** Creates a new instance with the chosen encoding. */
+ /** Creates a new instance with the chosen quantization encoding. */
public Lucene104ScalarQuantizedVectorsFormat(ScalarEncoding encoding) {
super(NAME);
this.encoding = encoding;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index 8012d6095c2..f98c0f630c8 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -141,7 +141,7 @@ class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
long numQuantizedVectorBytes =
Math.multiplyExact(
- (fieldEntry.scalarEncoding.getPackedLength(dimension)
+ (fieldEntry.scalarEncoding.getDocPackedLength(dimension)
+ (Float.BYTES * 3)
+ Integer.BYTES),
(long) fieldEntry.size);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
index 4238aed0360..69d23bc95df 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
@@ -21,6 +21,8 @@ import static org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectors
import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance;
+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.packAsBinary;
+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.transposeHalfByte;
import java.io.Closeable;
import java.io.IOException;
@@ -190,22 +192,21 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
FieldWriter fieldData, float[] clusterCenter, OptimizedScalarQuantizer scalarQuantizer)
throws IOException {
byte[] scratch =
- new byte
- [OptimizedScalarQuantizer.discretize(
- fieldData.fieldInfo.getVectorDimension(), encoding.getDimensionsPerByte())];
+ new byte[encoding.getDiscreteDimensions(fieldData.fieldInfo.getVectorDimension())];
byte[] vector =
switch (encoding) {
- case UNSIGNED_BYTE -> scratch;
- case SEVEN_BIT -> scratch;
- case PACKED_NIBBLE ->
- new byte[encoding.getPackedLength(fieldData.fieldInfo.getVectorDimension())];
+ case UNSIGNED_BYTE, SEVEN_BIT -> scratch;
+ case PACKED_NIBBLE, SINGLE_BIT_QUERY_NIBBLE ->
+ new byte[encoding.getDocPackedLength(scratch.length)];
};
for (int i = 0; i < fieldData.getVectors().size(); i++) {
float[] v = fieldData.getVectors().get(i);
OptimizedScalarQuantizer.QuantizationResult corrections =
scalarQuantizer.scalarQuantize(v, scratch, encoding.getBits(), clusterCenter);
- if (encoding == ScalarEncoding.PACKED_NIBBLE) {
- OffHeapScalarQuantizedVectorValues.packNibbles(scratch, vector);
+ switch (encoding) {
+ case PACKED_NIBBLE -> OffHeapScalarQuantizedVectorValues.packNibbles(scratch, vector);
+ case SINGLE_BIT_QUERY_NIBBLE -> OptimizedScalarQuantizer.packAsBinary(scratch, vector);
+ case UNSIGNED_BYTE, SEVEN_BIT -> {}
}
vectorData.writeBytes(vector, vector.length);
vectorData.writeInt(Float.floatToIntBits(corrections.lowerInterval()));
@@ -251,22 +252,21 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
OptimizedScalarQuantizer scalarQuantizer)
throws IOException {
byte[] scratch =
- new byte
- [OptimizedScalarQuantizer.discretize(
- fieldData.fieldInfo.getVectorDimension(), encoding.getDimensionsPerByte())];
+ new byte[encoding.getDiscreteDimensions(fieldData.fieldInfo.getVectorDimension())];
byte[] vector =
switch (encoding) {
- case UNSIGNED_BYTE -> scratch;
- case SEVEN_BIT -> scratch;
- case PACKED_NIBBLE ->
- new byte[encoding.getPackedLength(fieldData.fieldInfo.getVectorDimension())];
+ case UNSIGNED_BYTE, SEVEN_BIT -> scratch;
+ case PACKED_NIBBLE, SINGLE_BIT_QUERY_NIBBLE ->
+ new byte[encoding.getDocPackedLength(scratch.length)];
};
for (int ordinal : ordMap) {
float[] v = fieldData.getVectors().get(ordinal);
OptimizedScalarQuantizer.QuantizationResult corrections =
scalarQuantizer.scalarQuantize(v, scratch, encoding.getBits(), clusterCenter);
- if (encoding == ScalarEncoding.PACKED_NIBBLE) {
- OffHeapScalarQuantizedVectorValues.packNibbles(scratch, vector);
+ switch (encoding) {
+ case PACKED_NIBBLE -> OffHeapScalarQuantizedVectorValues.packNibbles(scratch, vector);
+ case SINGLE_BIT_QUERY_NIBBLE -> OptimizedScalarQuantizer.packAsBinary(scratch, vector);
+ case UNSIGNED_BYTE, SEVEN_BIT -> {}
}
vectorData.writeBytes(vector, vector.length);
vectorData.writeInt(Float.floatToIntBits(corrections.lowerInterval()));
@@ -385,6 +385,53 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
return docsWithField;
}
+ static DocsWithFieldSet writeBinarizedVectorAndQueryData(
+ IndexOutput binarizedVectorData,
+ ScalarEncoding encoding,
+ IndexOutput binarizedQueryData,
+ FloatVectorValues floatVectorValues,
+ float[] centroid,
+ OptimizedScalarQuantizer binaryQuantizer)
+ throws IOException {
+ if (encoding.isAsymmetric() == false) {
+ throw new IllegalArgumentException("encoding and queryEncoding must be different");
+ }
+ DocsWithFieldSet docsWithField = new DocsWithFieldSet();
+ int discretizedDims = encoding.getDiscreteDimensions(floatVectorValues.dimension());
+ byte[][] quantizationScratch = new byte[2][];
+ quantizationScratch[0] = new byte[discretizedDims];
+ quantizationScratch[1] = new byte[discretizedDims];
+ byte[] toIndex = new byte[encoding.getDocPackedLength(discretizedDims)];
+ byte[] toQuery = new byte[encoding.getQueryPackedLength(discretizedDims)];
+ KnnVectorValues.DocIndexIterator iterator = floatVectorValues.iterator();
+ for (int docV = iterator.nextDoc(); docV != NO_MORE_DOCS; docV = iterator.nextDoc()) {
+ // write index vector
+ OptimizedScalarQuantizer.QuantizationResult[] r =
+ binaryQuantizer.multiScalarQuantize(
+ floatVectorValues.vectorValue(iterator.index()),
+ quantizationScratch,
+ new byte[] {encoding.getBits(), encoding.getQueryBits()},
+ centroid);
+ // pack and store document bit vector
+ packAsBinary(quantizationScratch[0], toIndex);
+ binarizedVectorData.writeBytes(toIndex, toIndex.length);
+ binarizedVectorData.writeInt(Float.floatToIntBits(r[0].lowerInterval()));
+ binarizedVectorData.writeInt(Float.floatToIntBits(r[0].upperInterval()));
+ binarizedVectorData.writeInt(Float.floatToIntBits(r[0].additionalCorrection()));
+ binarizedVectorData.writeInt(r[0].quantizedComponentSum());
+ docsWithField.add(docV);
+
+ // pack and store the 4bit query vector
+ transposeHalfByte(quantizationScratch[1], toQuery);
+ binarizedQueryData.writeBytes(toQuery, toQuery.length);
+ binarizedQueryData.writeInt(Float.floatToIntBits(r[1].lowerInterval()));
+ binarizedQueryData.writeInt(Float.floatToIntBits(r[1].upperInterval()));
+ binarizedQueryData.writeInt(Float.floatToIntBits(r[1].additionalCorrection()));
+ binarizedQueryData.writeInt(r[1].quantizedComponentSum());
+ }
+ return docsWithField;
+ }
+
@Override
public CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(
FieldInfo fieldInfo, MergeState mergeState) throws IOException {
@@ -417,7 +464,9 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
throws IOException {
long vectorDataOffset = vectorData.alignFilePointer(Float.BYTES);
IndexOutput tempQuantizedVectorData = null;
+ IndexOutput tempScoreQuantizedVectorData = null;
IndexInput quantizedDataInput = null;
+ IndexInput quantizedScoreDataInput = null;
OptimizedScalarQuantizer quantizer =
new OptimizedScalarQuantizer(fieldInfo.getVectorSimilarityFunction());
try {
@@ -425,15 +474,32 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
segmentWriteState.directory.createTempOutput(
vectorData.getName(), "temp", segmentWriteState.context);
final String tempQuantizedVectorName = tempQuantizedVectorData.getName();
+ final String tempScoreQuantizedVectorName;
+ if (encoding.isAsymmetric()) {
+ tempScoreQuantizedVectorData =
+ segmentWriteState.directory.createTempOutput(
+ vectorData.getName() + "_score", "temp", segmentWriteState.context);
+ tempScoreQuantizedVectorName = tempScoreQuantizedVectorData.getName();
+ } else {
+ tempScoreQuantizedVectorName = null;
+ }
FloatVectorValues floatVectorValues =
MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState);
if (fieldInfo.getVectorSimilarityFunction() == COSINE) {
floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues);
}
DocsWithFieldSet docsWithField =
- writeVectorData(
- tempQuantizedVectorData,
- new QuantizedFloatVectorValues(floatVectorValues, quantizer, encoding, centroid));
+ encoding.isAsymmetric()
+ ? writeBinarizedVectorAndQueryData(
+ tempQuantizedVectorData,
+ encoding,
+ tempScoreQuantizedVectorData,
+ floatVectorValues,
+ centroid,
+ quantizer)
+ : writeVectorData(
+ tempQuantizedVectorData,
+ new QuantizedFloatVectorValues(floatVectorValues, quantizer, encoding, centroid));
CodecUtil.writeFooter(tempQuantizedVectorData);
IOUtils.close(tempQuantizedVectorData);
quantizedDataInput =
@@ -442,6 +508,13 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
quantizedDataInput, quantizedDataInput.length() - CodecUtil.footerLength());
long vectorDataLength = vectorData.getFilePointer() - vectorDataOffset;
CodecUtil.retrieveChecksum(quantizedDataInput);
+ if (tempScoreQuantizedVectorData != null) {
+ CodecUtil.writeFooter(tempScoreQuantizedVectorData);
+ IOUtils.close(tempScoreQuantizedVectorData);
+ quantizedScoreDataInput =
+ segmentWriteState.directory.openInput(
+ tempScoreQuantizedVectorName, segmentWriteState.context);
+ }
writeMeta(
fieldInfo,
segmentWriteState.segmentInfo.maxDoc(),
@@ -452,8 +525,11 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
docsWithField);
final IndexInput finalQuantizedDataInput = quantizedDataInput;
+ final IndexInput finalQuantizedScoreDataInput = quantizedScoreDataInput;
tempQuantizedVectorData = null;
+ tempScoreQuantizedVectorData = null;
quantizedDataInput = null;
+ quantizedScoreDataInput = null;
OffHeapScalarQuantizedVectorValues vectorValues =
new OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues(
@@ -466,23 +542,54 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
fieldInfo.getVectorSimilarityFunction(),
vectorsScorer,
finalQuantizedDataInput);
+ OffHeapScalarQuantizedVectorValues scoreVectorValues = null;
+ if (finalQuantizedScoreDataInput != null) {
+ scoreVectorValues =
+ new OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues(
+ true,
+ fieldInfo.getVectorDimension(),
+ docsWithField.cardinality(),
+ centroid,
+ cDotC,
+ quantizer,
+ encoding,
+ fieldInfo.getVectorSimilarityFunction(),
+ vectorsScorer,
+ finalQuantizedScoreDataInput);
+ }
RandomVectorScorerSupplier scorerSupplier =
- vectorsScorer.getRandomVectorScorerSupplier(
- fieldInfo.getVectorSimilarityFunction(), vectorValues);
+ scoreVectorValues == null
+ ? vectorsScorer.getRandomVectorScorerSupplier(
+ fieldInfo.getVectorSimilarityFunction(), vectorValues)
+ : vectorsScorer.getRandomVectorScorerSupplier(
+ fieldInfo.getVectorSimilarityFunction(), scoreVectorValues, vectorValues);
return new QuantizedCloseableRandomVectorScorerSupplier(
scorerSupplier,
vectorValues,
() -> {
- IOUtils.close(finalQuantizedDataInput);
+ IOUtils.close(finalQuantizedDataInput, finalQuantizedScoreDataInput);
+ if (tempScoreQuantizedVectorName != null) {
+ IOUtils.deleteFilesIgnoringExceptions(
+ segmentWriteState.directory, tempScoreQuantizedVectorName);
+ }
IOUtils.deleteFilesIgnoringExceptions(
segmentWriteState.directory, tempQuantizedVectorName);
});
} catch (Throwable t) {
- IOUtils.closeWhileSuppressingExceptions(t, tempQuantizedVectorData, quantizedDataInput);
+ IOUtils.closeWhileSuppressingExceptions(
+ t,
+ tempQuantizedVectorData,
+ tempScoreQuantizedVectorData,
+ quantizedDataInput,
+ quantizedScoreDataInput);
if (tempQuantizedVectorData != null) {
IOUtils.deleteFilesSuppressingExceptions(
t, segmentWriteState.directory, tempQuantizedVectorData.getName());
}
+ if (tempScoreQuantizedVectorData != null) {
+ IOUtils.deleteFilesSuppressingExceptions(
+ t, segmentWriteState.directory, tempScoreQuantizedVectorData.getName());
+ }
throw t;
}
}
@@ -687,15 +794,12 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
this.values = delegate;
this.quantizer = quantizer;
this.encoding = encoding;
- this.quantized =
- new byte
- [OptimizedScalarQuantizer.discretize(
- delegate.dimension(), encoding.getDimensionsPerByte())];
+ this.quantized = new byte[encoding.getDiscreteDimensions(delegate.dimension())];
this.packed =
switch (encoding) {
- case UNSIGNED_BYTE -> this.quantized;
- case SEVEN_BIT -> this.quantized;
- case PACKED_NIBBLE -> new byte[encoding.getPackedLength(delegate.dimension())];
+ case UNSIGNED_BYTE, SEVEN_BIT -> this.quantized;
+ case PACKED_NIBBLE, SINGLE_BIT_QUERY_NIBBLE ->
+ new byte[encoding.getDocPackedLength(quantized.length)];
};
this.centroid = centroid;
this.centroidDP = VectorUtil.dotProduct(centroid, centroid);
@@ -734,7 +838,7 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
@Override
public ScalarEncoding getScalarEncoding() {
- throw new UnsupportedOperationException();
+ return encoding;
}
@Override
@@ -766,8 +870,10 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
corrections =
quantizer.scalarQuantize(
values.vectorValue(ord), quantized, encoding.getBits(), centroid);
- if (encoding == ScalarEncoding.PACKED_NIBBLE) {
- OffHeapScalarQuantizedVectorValues.packNibbles(quantized, packed);
+ switch (encoding) {
+ case PACKED_NIBBLE -> OffHeapScalarQuantizedVectorValues.packNibbles(quantized, packed);
+ case SINGLE_BIT_QUERY_NIBBLE -> OptimizedScalarQuantizer.packAsBinary(quantized, packed);
+ case UNSIGNED_BYTE, SEVEN_BIT -> {}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
index d2c678d8f8b..440e96d4041 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
@@ -53,6 +53,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
final ScalarEncoding encoding;
final float[] centroid;
final float centroidDp;
+ final boolean isQuerySide;
OffHeapScalarQuantizedVectorValues(
int dimension,
@@ -64,6 +65,32 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
VectorSimilarityFunction similarityFunction,
FlatVectorsScorer vectorsScorer,
IndexInput slice) {
+ this(
+ false,
+ dimension,
+ size,
+ centroid,
+ centroidDp,
+ quantizer,
+ encoding,
+ similarityFunction,
+ vectorsScorer,
+ slice);
+ }
+
+ OffHeapScalarQuantizedVectorValues(
+ boolean isQuerySide,
+ int dimension,
+ int size,
+ float[] centroid,
+ float centroidDp,
+ OptimizedScalarQuantizer quantizer,
+ ScalarEncoding encoding,
+ VectorSimilarityFunction similarityFunction,
+ FlatVectorsScorer vectorsScorer,
+ IndexInput slice) {
+ assert isQuerySide == false || encoding.isAsymmetric();
+ this.isQuerySide = isQuerySide;
this.dimension = dimension;
this.size = size;
this.similarityFunction = similarityFunction;
@@ -72,11 +99,15 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
this.centroid = centroid;
this.centroidDp = centroidDp;
this.correctiveValues = new float[3];
- this.byteSize = encoding.getPackedLength(dimension) + (Float.BYTES * 3) + Integer.BYTES;
- this.byteBuffer = ByteBuffer.allocate(encoding.getPackedLength(dimension));
+ this.encoding = encoding;
+ int docPackedLength =
+ isQuerySide
+ ? encoding.getQueryPackedLength(dimension)
+ : encoding.getDocPackedLength(dimension);
+ this.byteSize = docPackedLength + (Float.BYTES * 3) + Integer.BYTES;
+ this.byteBuffer = ByteBuffer.allocate(docPackedLength);
this.vectorValue = byteBuffer.array();
this.quantizer = quantizer;
- this.encoding = encoding;
}
@Override
@@ -114,7 +145,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
return new OptimizedScalarQuantizer.QuantizationResult(
correctiveValues[0], correctiveValues[1], correctiveValues[2], quantizedComponentSum);
}
- slice.seek(((long) targetOrd * byteSize) + dimension);
+ slice.seek(((long) targetOrd * byteSize) + vectorValue.length);
slice.readFloats(correctiveValues, 0, 3);
quantizedComponentSum = slice.readInt();
return new OptimizedScalarQuantizer.QuantizationResult(
@@ -229,9 +260,34 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
slice);
}
+ DenseOffHeapVectorValues(
+ boolean isQuerySide,
+ int dimension,
+ int size,
+ float[] centroid,
+ float centroidDp,
+ OptimizedScalarQuantizer quantizer,
+ ScalarEncoding encoding,
+ VectorSimilarityFunction similarityFunction,
+ FlatVectorsScorer vectorsScorer,
+ IndexInput slice) {
+ super(
+ isQuerySide,
+ dimension,
+ size,
+ centroid,
+ centroidDp,
+ quantizer,
+ encoding,
+ similarityFunction,
+ vectorsScorer,
+ slice);
+ }
+
@Override
public OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues copy() throws IOException {
return new OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues(
+ isQuerySide,
dimension,
size,
centroid,
@@ -250,6 +306,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
@Override
public VectorScorer scorer(float[] target) throws IOException {
+ assert isQuerySide == false;
OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues copy = copy();
DocIndexIterator iterator = copy.iterator();
RandomVectorScorer scorer =
@@ -304,6 +361,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
similarityFunction,
vectorsScorer,
slice);
+ assert isQuerySide == false;
this.configuration = configuration;
this.dataIn = dataIn;
this.ordToDoc = configuration.getDirectMonotonicReader(dataIn);
@@ -312,6 +370,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
@Override
public SparseOffHeapVectorValues copy() throws IOException {
+ assert isQuerySide == false;
return new SparseOffHeapVectorValues(
configuration,
dimension,
@@ -356,6 +415,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
@Override
public VectorScorer scorer(float[] target) throws IOException {
+ assert isQuerySide == false;
SparseOffHeapVectorValues copy = copy();
DocIndexIterator iterator = copy.iterator();
RandomVectorScorer scorer =
@@ -389,6 +449,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
similarityFunction,
vectorsScorer,
null);
+ assert isQuerySide == false;
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java
index 48d0c4e665f..02b9c074899 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java
@@ -58,7 +58,7 @@ abstract class QuantizedByteVectorValues extends ByteVectorValues {
public abstract OptimizedScalarQuantizer getQuantizer();
/**
- * @return the scalar encoding used to pack the vectors.
+ * @return the scalar encoding used to pack the stored vectors.
*/
public abstract ScalarEncoding getScalarEncoding();
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java
index 6164b0062f0..10fc424c44e 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java
@@ -30,6 +30,7 @@ import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
@@ -47,14 +48,31 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.SameThreadExecutorService;
+import org.junit.Before;
public class TestLucene104HnswScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {
- private static final KnnVectorsFormat FORMAT = new Lucene104HnswScalarQuantizedVectorsFormat();
+ private KnnVectorsFormat format;
+ private ScalarEncoding encoding;
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ var encodingValues = ScalarEncoding.values();
+ encoding = encodingValues[random().nextInt(encodingValues.length)];
+ format =
+ new Lucene104HnswScalarQuantizedVectorsFormat(
+ encoding,
+ Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
+ 1,
+ null);
+ super.setUp();
+ }
@Override
protected Codec getCodec() {
- return TestUtil.alwaysKnnVectorsFormat(FORMAT);
+ return TestUtil.alwaysKnnVectorsFormat(format);
}
public void testToString() {
@@ -170,7 +188,7 @@ public class TestLucene104HnswScalarQuantizedVectorsFormat extends BaseKnnVector
assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec"));
assertEquals(1L, (long) offHeap.get("vex"));
long corrections = Float.BYTES + Float.BYTES + Float.BYTES + Integer.BYTES;
- long expected = fieldInfo.getVectorDimension() + corrections;
+ long expected = encoding.getDocPackedLength(fieldInfo.getVectorDimension()) + corrections;
assertEquals(expected, (long) offHeap.get("veq"));
assertEquals(3, offHeap.size());
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
index 29041b5b07f..150294df12c 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
@@ -166,9 +166,8 @@ public class TestLucene104ScalarQuantizedVectorsFormat extends BaseKnnVectorsFor
assertEquals(centroid.length, dims);
OptimizedScalarQuantizer quantizer = new OptimizedScalarQuantizer(similarityFunction);
- byte[] scratch =
- new byte[OptimizedScalarQuantizer.discretize(dims, encoding.getDimensionsPerByte())];
- byte[] expectedVector = new byte[encoding.getPackedLength(dims)];
+ byte[] scratch = new byte[encoding.getDiscreteDimensions(dims)];
+ byte[] expectedVector = new byte[encoding.getDocPackedLength(scratch.length)];
if (similarityFunction == VectorSimilarityFunction.COSINE) {
vectorValues =
new Lucene104ScalarQuantizedVectorsWriter.NormalizedFloatVectorValues(vectorValues);
@@ -183,10 +182,12 @@ public class TestLucene104ScalarQuantizedVectorsFormat extends BaseKnnVectorsFor
encoding.getBits(),
centroid);
switch (encoding) {
- case UNSIGNED_BYTE -> System.arraycopy(scratch, 0, expectedVector, 0, dims);
- case SEVEN_BIT -> System.arraycopy(scratch, 0, expectedVector, 0, dims);
+ case UNSIGNED_BYTE, SEVEN_BIT ->
+ System.arraycopy(scratch, 0, expectedVector, 0, dims);
case PACKED_NIBBLE ->
OffHeapScalarQuantizedVectorValues.packNibbles(scratch, expectedVector);
+ case SINGLE_BIT_QUERY_NIBBLE ->
+ OptimizedScalarQuantizer.packAsBinary(scratch, expectedVector);
}
assertArrayEquals(expectedVector, qvectorValues.vectorValue(docIndexIterator.index()));
var actualCorrections = qvectorValues.getCorrectiveTerms(docIndexIterator.index());