Apache Lucene open-source search software
Fix quantized vector writer ram estimates (#13553)
Benjamin Trent9bfde5514ca8
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 16000083199..b75eaa73ebf 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -297,6 +297,9 @@ Bug Fixes* GITHUB#13463: Address bug in MultiLeafKnnCollector causing #minCompetitiveSimilarity to stay artificially low insome corner cases. (Greg Miller)+* GITHUB#13553: Correct RamUsageEstimate for scalar quantized knn vector formats so that raw vectors are correctly+ accounted for. (Ben Trent)+Other--------------------(No changes)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.javaindex e2171a3513f..949507848bf 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java@@ -171,10 +171,8 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {@Overridepublic long ramBytesUsed() {long total = SHALLOW_RAM_BYTES_USED;+ // The vector delegate will also account for this writer's KnnFieldVectorsWriter objectstotal += flatVectorWriter.ramBytesUsed();- for (FieldWriter<?> field : fields) {- total += field.ramBytesUsed();- }return total;}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.javaindex 98dfd891ebf..beb1af19ca1 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java@@ -299,9 +299,8 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite@Overridepublic long ramBytesUsed() {long total = SHALLOW_RAM_BYTES_USED;- for (FieldWriter field : fields) {- total += field.ramBytesUsed();- }+ // The vector delegate will also account for this writer's KnnFieldVectorsWriter objects+ total += rawVectorDelegate.ramBytesUsed();return total;}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.javaindex 7a1f8b232c9..a10d2642349 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java@@ -23,10 +23,15 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;import java.io.ByteArrayOutputStream;import java.io.IOException;import java.util.Arrays;+import java.util.Collections;+import java.util.HashMap;+import java.util.Map;import java.util.Set;import org.apache.lucene.codecs.Codec;+import org.apache.lucene.codecs.KnnFieldVectorsWriter;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.KnnVectorsReader;+import org.apache.lucene.codecs.KnnVectorsWriter;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.KnnByteVectorField;@@ -38,13 +43,19 @@ import org.apache.lucene.index.ByteVectorValues;import org.apache.lucene.index.CheckIndex;import org.apache.lucene.index.CodecReader;import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.DocValuesType;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.NoMergePolicy;+import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.index.StoredFields;import org.apache.lucene.index.Term;import org.apache.lucene.index.VectorEncoding;@@ -60,7 +71,10 @@ import org.apache.lucene.store.FSDirectory;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.Bits;import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.InfoStream;+import org.apache.lucene.util.StringHelper;import org.apache.lucene.util.VectorUtil;+import org.apache.lucene.util.Version;import org.junit.Before;/**@@ -216,6 +230,68 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe}}+ @SuppressWarnings("unchecked")+ public void testWriterRamEstimate() throws Exception {+ final FieldInfos fieldInfos = new FieldInfos(new FieldInfo[0]);+ final Directory dir = newDirectory();+ Codec codec = Codec.getDefault();+ final SegmentInfo si =+ new SegmentInfo(+ dir,+ Version.LATEST,+ Version.LATEST,+ "0",+ 10000,+ false,+ false,+ codec,+ Collections.emptyMap(),+ StringHelper.randomId(),+ new HashMap<>(),+ null);+ final SegmentWriteState state =+ new SegmentWriteState(+ InfoStream.getDefault(), dir, si, fieldInfos, null, newIOContext(random()));+ final KnnVectorsFormat format = codec.knnVectorsFormat();+ try (KnnVectorsWriter writer = format.fieldsWriter(state)) {+ final long ramBytesUsed = writer.ramBytesUsed();+ int dim = random().nextInt(64) + 1;+ if (dim % 2 == 1) {+ ++dim;+ }+ int numDocs = atLeast(100);+ KnnFieldVectorsWriter<float[]> fieldWriter =+ (KnnFieldVectorsWriter<float[]>)+ writer.addField(+ new FieldInfo(+ "fieldA",+ 0,+ false,+ false,+ false,+ IndexOptions.NONE,+ DocValuesType.NONE,+ false,+ -1,+ Map.of(),+ 0,+ 0,+ 0,+ dim,+ VectorEncoding.FLOAT32,+ VectorSimilarityFunction.DOT_PRODUCT,+ false,+ false));+ for (int i = 0; i < numDocs; i++) {+ fieldWriter.addValue(i, randomVector(dim));+ }+ final long ramBytesUsed2 = writer.ramBytesUsed();+ assertTrue(ramBytesUsed2 > ramBytesUsed);+ assertTrue(ramBytesUsed2 > (long) dim * numDocs * Float.BYTES);+ }+ dir.close();+ }+public void testIllegalSimilarityFunctionChangeTwoWriters() throws Exception {try (Directory dir = newDirectory()) {try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {