Apache Lucene open-source search software
Adds OffHeapQuantizedFloatVectorValues to access float vectors when only quantized vectors are present (#14792)
Pulkit Gupta647c05b1e279
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex c86d3bce076..c57456a8016 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -32,6 +32,8 @@ New Features* GITHUB#14178: Add a Faiss-based vector format in the sandbox module. (Kaival Parikh)+* GITHUB#14792: Introduced OffHeapQuantizedFloatVectorValues class to access float vectors when only quantized byte vectors are available in the index. (Pulkit Gupta)+Improvements---------------------diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.javaindex 59ff0a0beae..4d6526f43bd 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java@@ -189,6 +189,20 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReadepublic FloatVectorValues getFloatVectorValues(String field) throws IOException {final FieldEntry fieldEntry = getFieldEntry(field);final FloatVectorValues rawVectorValues = rawVectorsReader.getFloatVectorValues(field);+ if (rawVectorValues.size() == 0) {+ return OffHeapQuantizedFloatVectorValues.load(+ fieldEntry.ordToDoc,+ fieldEntry.dimension,+ fieldEntry.size,+ fieldEntry.scalarQuantizer,+ fieldEntry.similarityFunction,+ vectorScorer,+ fieldEntry.compress,+ fieldEntry.vectorDataOffset,+ fieldEntry.vectorDataLength,+ quantizedVectorData);+ }+OffHeapQuantizedByteVectorValues quantizedByteVectorValues =OffHeapQuantizedByteVectorValues.load(fieldEntry.ordToDoc,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedFloatVectorValues.javanew file mode 100644index 00000000000..71eba0b43bf--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedFloatVectorValues.java@@ -0,0 +1,376 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.codecs.lucene99;++import java.io.IOException;+import java.nio.ByteBuffer;+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;+import org.apache.lucene.codecs.lucene90.IndexedDISI;+import org.apache.lucene.codecs.lucene95.HasIndexSlice;+import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;+import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.VectorScorer;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.util.Bits;+import org.apache.lucene.util.hnsw.RandomVectorScorer;+import org.apache.lucene.util.packed.DirectMonotonicReader;+import org.apache.lucene.util.quantization.ScalarQuantizer;++/**+ * Reads quantized vector values from the index input and returns float vector values after+ * dequantizing them.+ *+ * <p>This class provides functionality to read byte vectors that have been quantized and stored in+ * the index, and then dequantize them back to float vectors with some precision loss. The+ * implementation is based on {@code OffHeapQuantizedByteVectorValues} with modifications to the+ * {@code vectorValue()} method to return float vectors after dequantizing the byte vectors.+ *+ * <p>Usage: This class is used for read-only indexes where full-precision float vectors have been+ * dropped from the index to save storage space. Full-precision vectors can be removed from an index+ * using a method as implemnted in {@code+ * TestLucene99ScalarQuantizedVectorsFormat.simulateEmptyRawVectors()}.+ *+ * @lucene.internal+ */+abstract class OffHeapQuantizedFloatVectorValues extends FloatVectorValues+ implements HasIndexSlice {++ protected final int dimension;+ protected final int size;+ protected final int numBytes;+ protected final ScalarQuantizer scalarQuantizer;+ protected final VectorSimilarityFunction similarityFunction;+ protected final FlatVectorsScorer vectorsScorer;+ protected final boolean compress;++ protected final IndexInput slice;+ protected final float[] floatValue;+ protected final byte[] binaryValue;+ protected final ByteBuffer byteBuffer;+ protected final int byteSize;+ protected int curOrd = -1;+ protected final float[] scoreCorrectionConstant = new float[1];++ OffHeapQuantizedFloatVectorValues(+ int dimension,+ int size,+ ScalarQuantizer scalarQuantizer,+ VectorSimilarityFunction similarityFunction,+ FlatVectorsScorer vectorsScorer,+ boolean compress,+ IndexInput slice) {+ this.dimension = dimension;+ this.size = size;+ this.slice = slice;+ this.scalarQuantizer = scalarQuantizer;+ this.compress = compress;+ if (scalarQuantizer.getBits() <= 4 && compress) {+ this.numBytes = (dimension + 1) >> 1;+ } else {+ this.numBytes = dimension;+ }+ this.byteSize = this.numBytes + Float.BYTES;+ byteBuffer = ByteBuffer.allocate(dimension);+ binaryValue = byteBuffer.array();+ floatValue = new float[dimension];+ this.similarityFunction = similarityFunction;+ this.vectorsScorer = vectorsScorer;+ }++ @Override+ public int dimension() {+ return dimension;+ }++ @Override+ public int size() {+ return size;+ }++ @Override+ public float[] vectorValue(int targetOrd) throws IOException {+ if (curOrd == targetOrd) {+ return floatValue;+ }+ slice.seek((long) targetOrd * byteSize);+ slice.readBytes(byteBuffer.array(), byteBuffer.arrayOffset(), numBytes);+ slice.readFloats(scoreCorrectionConstant, 0, 1);+ OffHeapQuantizedByteVectorValues.decompressBytes(binaryValue, numBytes);+ scalarQuantizer.deQuantize(binaryValue, floatValue);+ curOrd = targetOrd;+ return floatValue;+ }++ @Override+ public IndexInput getSlice() {+ return slice;+ }++ @Override+ public int getVectorByteLength() {+ return numBytes;+ }++ public static OffHeapQuantizedFloatVectorValues load(+ OrdToDocDISIReaderConfiguration configuration,+ int dimension,+ int size,+ ScalarQuantizer scalarQuantizer,+ VectorSimilarityFunction similarityFunction,+ FlatVectorsScorer vectorsScorer,+ boolean compress,+ long quantizedVectorDataOffset,+ long quantizedVectorDataLength,+ IndexInput vectorData)+ throws IOException {++ if (size == 0) {+ return new EmptyOffHeapVectorValues(dimension, similarityFunction, vectorsScorer);+ }+ IndexInput bytesSlice =+ vectorData.slice(+ "quantized-float-vector-data", quantizedVectorDataOffset, quantizedVectorDataLength);++ if (configuration.isDense()) {+ return new DenseOffHeapVectorValues(+ dimension,+ size,+ scalarQuantizer,+ compress,+ similarityFunction,+ vectorsScorer,+ bytesSlice);+ } else {+ return new SparseOffHeapVectorValues(+ configuration,+ dimension,+ size,+ scalarQuantizer,+ compress,+ vectorData,+ similarityFunction,+ vectorsScorer,+ bytesSlice);+ }+ }++ /**+ * Dense vector values that are stored off-heap. This is the most common case when every doc has a+ * vector.+ */+ private static class DenseOffHeapVectorValues extends OffHeapQuantizedFloatVectorValues {++ public DenseOffHeapVectorValues(+ int dimension,+ int size,+ ScalarQuantizer scalarQuantizer,+ boolean compress,+ VectorSimilarityFunction similarityFunction,+ FlatVectorsScorer vectorsScorer,+ IndexInput slice) {+ super(dimension, size, scalarQuantizer, similarityFunction, vectorsScorer, compress, slice);+ }++ @Override+ public DenseOffHeapVectorValues copy() throws IOException {+ return new DenseOffHeapVectorValues(+ dimension,+ size,+ scalarQuantizer,+ compress,+ similarityFunction,+ vectorsScorer,+ slice.clone());+ }++ @Override+ public Bits getAcceptOrds(Bits acceptDocs) {+ return acceptDocs;+ }++ @Override+ public DocIndexIterator iterator() {+ return createDenseIterator();+ }++ @Override+ public VectorScorer scorer(float[] target) throws IOException {+ DenseOffHeapVectorValues copy = copy();+ DocIndexIterator iterator = copy.iterator();+ RandomVectorScorer vectorScorer =+ vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target);+ return new VectorScorer() {+ @Override+ public float score() throws IOException {+ return vectorScorer.score(iterator.index());+ }++ @Override+ public DocIdSetIterator iterator() {+ return iterator;+ }+ };+ }+ }++ private static class SparseOffHeapVectorValues extends OffHeapQuantizedFloatVectorValues {+ private final DirectMonotonicReader ordToDoc;+ private final IndexedDISI disi;+ // dataIn was used to init a new IndexedDIS for #randomAccess()+ private final IndexInput dataIn;+ private final OrdToDocDISIReaderConfiguration configuration;++ public SparseOffHeapVectorValues(+ OrdToDocDISIReaderConfiguration configuration,+ int dimension,+ int size,+ ScalarQuantizer scalarQuantizer,+ boolean compress,+ IndexInput dataIn,+ VectorSimilarityFunction similarityFunction,+ FlatVectorsScorer vectorsScorer,+ IndexInput slice)+ throws IOException {+ super(dimension, size, scalarQuantizer, similarityFunction, vectorsScorer, compress, slice);+ this.configuration = configuration;+ this.dataIn = dataIn;+ this.ordToDoc = configuration.getDirectMonotonicReader(dataIn);+ this.disi = configuration.getIndexedDISI(dataIn);+ }++ @Override+ public DocIndexIterator iterator() {+ return IndexedDISI.asDocIndexIterator(disi);+ }++ @Override+ public SparseOffHeapVectorValues copy() throws IOException {+ return new SparseOffHeapVectorValues(+ configuration,+ dimension,+ size,+ scalarQuantizer,+ compress,+ dataIn,+ similarityFunction,+ vectorsScorer,+ slice.clone());+ }++ @Override+ public int ordToDoc(int ord) {+ return (int) ordToDoc.get(ord);+ }++ @Override+ public Bits getAcceptOrds(Bits acceptDocs) {+ if (acceptDocs == null) {+ return null;+ }+ return new Bits() {+ @Override+ public boolean get(int index) {+ return acceptDocs.get(ordToDoc(index));+ }++ @Override+ public int length() {+ return size;+ }+ };+ }++ @Override+ public VectorScorer scorer(float[] target) throws IOException {+ SparseOffHeapVectorValues copy = copy();+ DocIndexIterator iterator = copy.iterator();+ RandomVectorScorer vectorScorer =+ vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target);+ return new VectorScorer() {+ @Override+ public float score() throws IOException {+ return vectorScorer.score(iterator.index());+ }++ @Override+ public DocIdSetIterator iterator() {+ return iterator;+ }+ };+ }+ }++ private static class EmptyOffHeapVectorValues extends OffHeapQuantizedFloatVectorValues {++ public EmptyOffHeapVectorValues(+ int dimension,+ VectorSimilarityFunction similarityFunction,+ FlatVectorsScorer vectorsScorer) {+ super(+ dimension,+ 0,+ new ScalarQuantizer(-1, 1, (byte) 7),+ similarityFunction,+ vectorsScorer,+ false,+ null);+ }++ @Override+ public int dimension() {+ return super.dimension();+ }++ @Override+ public int size() {+ return 0;+ }++ @Override+ public DocIndexIterator iterator() {+ return createDenseIterator();+ }++ @Override+ public EmptyOffHeapVectorValues copy() {+ throw new UnsupportedOperationException();+ }++ @Override+ public float[] vectorValue(int targetOrd) {+ throw new UnsupportedOperationException();+ }++ @Override+ public int ordToDoc(int ord) {+ throw new UnsupportedOperationException();+ }++ @Override+ public Bits getAcceptOrds(Bits acceptDocs) {+ return null;+ }++ @Override+ public VectorScorer scorer(float[] target) {+ return null;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.javaindex d6e2c0986a2..04fa2c98b9f 100644--- a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java@@ -162,7 +162,7 @@ public class ScalarQuantizer {* @param src the source vector* @param dest the destination vector*/- void deQuantize(byte[] src, float[] dest) {+ public void deQuantize(byte[] src, float[] dest) {assert src.length == dest.length;for (int i = 0; i < src.length; i++) {dest[i] = (alpha * src[i]) + minQuantile;diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.javaindex cf1436f2109..989061b1a9a 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java@@ -17,6 +17,7 @@package org.apache.lucene.codecs.lucene99;import static java.lang.String.format;+import static org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;import static org.hamcrest.Matchers.is;import static org.hamcrest.Matchers.oneOf;@@ -25,14 +26,17 @@ import java.util.ArrayList;import java.util.List;import java.util.Locale;import org.apache.lucene.codecs.Codec;+import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.FilterCodec;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.KnnVectorsReader;+import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.KnnFloatVectorField;import org.apache.lucene.index.CodecReader;import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.FloatVectorValues;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;@@ -41,7 +45,11 @@ import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.NoMergePolicy;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;+import org.apache.lucene.tests.store.BaseDirectoryWrapper;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.VectorUtil;import org.apache.lucene.util.quantization.QuantizedByteVectorValues;@@ -73,6 +81,21 @@ public class TestLucene99ScalarQuantizedVectorsFormat extends BaseKnnVectorsFormreturn TestUtil.alwaysKnnVectorsFormat(format);}+ protected List<float[]> getRandomFloatVector(int numVectors, int dim, boolean normalize) {+ List<float[]> vectors = new ArrayList<>(numVectors);+ for (int i = 0; i < numVectors; i++) {+ float[] vec = randomVector(dim);+ if (normalize) {+ float[] copy = new float[vec.length];+ System.arraycopy(vec, 0, copy, 0, copy.length);+ VectorUtil.l2normalize(copy);+ vec = copy;+ }+ vectors.add(vec);+ }+ return vectors;+ }+public void testSearch() throws Exception {try (Directory dir = newDirectory();IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {@@ -193,6 +216,143 @@ public class TestLucene99ScalarQuantizedVectorsFormat extends BaseKnnVectorsForm}}+ public void testReadQuantizedVectorWithEmptyRawVectors() throws Exception {+ String vectorFieldName = "vec1";+ int numVectors = 1 + random().nextInt(50);+ int dim = random().nextInt(64) + 1;+ if (dim % 2 == 1) {+ dim++;+ }+ VectorSimilarityFunction similarityFunction = randomSimilarity();+ List<float[]> vectors =+ getRandomFloatVector(+ numVectors, dim, similarityFunction == VectorSimilarityFunction.COSINE);++ try (BaseDirectoryWrapper dir = newDirectory();+ IndexWriter w =+ new IndexWriter(+ dir,+ new IndexWriterConfig()+ .setMaxBufferedDocs(numVectors + 1)+ .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)+ .setMergePolicy(NoMergePolicy.INSTANCE)+ .setUseCompoundFile(false)+ .setCodec(getCodec()))) {+ dir.setCheckIndexOnClose(false);++ for (int i = 0; i < numVectors; i++) {+ Document doc = new Document();+ doc.add(new KnnFloatVectorField(vectorFieldName, vectors.get(i), similarityFunction));+ w.addDocument(doc);+ }+ w.commit();++ simulateEmptyRawVectors(dir);++ try (IndexReader reader = DirectoryReader.open(w)) {+ LeafReader r = getOnlyLeafReader(reader);+ if (r instanceof CodecReader codecReader) {+ KnnVectorsReader knnVectorsReader = codecReader.getVectorReader();+ if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) {+ knnVectorsReader = fieldsReader.getFieldReader(vectorFieldName);+ }+ if (knnVectorsReader instanceof Lucene99ScalarQuantizedVectorsReader quantizedReader) {+ FloatVectorValues floatVectorValues =+ quantizedReader.getFloatVectorValues(vectorFieldName);+ if (floatVectorValues instanceof OffHeapQuantizedFloatVectorValues) {+ KnnVectorValues.DocIndexIterator iter = floatVectorValues.iterator();+ for (int docId = iter.nextDoc(); docId != NO_MORE_DOCS; docId = iter.nextDoc()) {+ float[] dequantizedVector = floatVectorValues.vectorValue(iter.index());+ for (int i = 0; i < dim; i++) {+ assertEquals(dequantizedVector[i], vectors.get(docId)[i], 0.2f);+ }+ }+ } else {+ fail("floatVectorValues is not OffHeapQuantizedFloatVectorValues");+ }+ } else {+ System.out.println("Vector READER:: " + knnVectorsReader.toString());+ fail("reader is not Lucene99ScalarQuantizedVectorsReader");+ }+ } else {+ fail("reader is not CodecReader");+ }+ }+ }+ }++ /** Simulates empty raw vectors by modifying index files. */+ private void simulateEmptyRawVectors(Directory dir) throws Exception {+ final String[] indexFiles = dir.listAll();+ final String RAW_VECTOR_EXTENSION = "vec";+ final String VECTOR_META_EXTENSION = "vemf";++ for (String file : indexFiles) {+ if (file.endsWith("." + RAW_VECTOR_EXTENSION)) {+ replaceWithEmptyVectorFile(dir, file);+ } else if (file.endsWith("." + VECTOR_META_EXTENSION)) {+ updateVectorMetadataFile(dir, file);+ }+ }+ }++ /** Replaces a raw vector file with an empty one that has valid header/footer. */+ private void replaceWithEmptyVectorFile(Directory dir, String fileName) throws Exception {+ byte[] indexHeader;+ try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {+ indexHeader = CodecUtil.readIndexHeader(in);+ }+ dir.deleteFile(fileName);+ try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {+ // Write header+ out.writeBytes(indexHeader, 0, indexHeader.length);+ // Write footer (no content in between)+ CodecUtil.writeFooter(out);+ }+ }++ /** Updates vector metadata file to indicate zero vector length. */+ private void updateVectorMetadataFile(Directory dir, String fileName) throws Exception {+ // Read original metadata+ byte[] indexHeader;+ int fieldNumber, vectorEncoding, vectorSimilarityFunction, dimension;+ long vectorStartPos;++ try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {+ indexHeader = CodecUtil.readIndexHeader(in);+ fieldNumber = in.readInt();+ vectorEncoding = in.readInt();+ vectorSimilarityFunction = in.readInt();+ vectorStartPos = in.readVLong();+ in.readVLong(); // Skip original vector length+ dimension = in.readVInt();+ }++ // Create updated metadata file+ dir.deleteFile(fileName);+ try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {+ // Write header+ out.writeBytes(indexHeader, 0, indexHeader.length);++ // Write metadata with zero vector length+ out.writeInt(fieldNumber);+ out.writeInt(vectorEncoding);+ out.writeInt(vectorSimilarityFunction);+ out.writeVLong(vectorStartPos);+ out.writeVLong(0); // Set vector length to 0+ out.writeVInt(dimension);+ out.writeInt(0);++ // Write configuration+ OrdToDocDISIReaderConfiguration.writeStoredMeta(+ DIRECT_MONOTONIC_BLOCK_SHIFT, out, null, 0, 0, null);++ // Mark end of fields and write footer+ out.writeInt(-1);+ CodecUtil.writeFooter(out);+ }+ }+public void testToString() {FilterCodec customCodec =new FilterCodec("foo", Codec.getDefault()) {