Apache Lucene open-source search software
Fix undercounting of RAM used by vectors buffered in in-memory segments (#15982)
Prithvi S735c6e026683
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 2bbfea5a2c6..34b393064cf 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -158,6 +158,13 @@ OptimizationsBug Fixes---------------------+* GITHUB#15901: Fix undercounting of RAM used by vectors buffered in in-memory segments.+ BufferingKnnVectorsWriter hardcoded Float.BYTES for all vector encodings, overcounting byte+ vectors by 4x. Scalar quantized vector writers (Lucene104, Lucene99, Lucene102) did not+ account for rawVectorDelegate RAM, making byte vector fields completely invisible in RAM+ reporting. Also added missing dimensionSums array accounting in quantized writers.+ (Prithvi S)+* GITHUB#14049: Randomize KNN codec params in RandomCodec. Fixes scalar quantization div-by-zerowhen all values are identical. (Mike Sokolov)diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.javaindex 4b873be3608..796bc5511c7 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java@@ -51,6 +51,7 @@ import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.VectorScorer;import org.apache.lucene.store.IndexOutput;import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.RamUsageEstimator;import org.apache.lucene.util.VectorUtil;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;@@ -453,9 +454,14 @@ class Lucene102BinaryQuantizedVectorsWriter extends FlatVectorsWriter {@Overridepublic long ramBytesUsed() {long total = SHALLOW_RAM_BYTES_USED;+ // The rawVectorDelegate tracks all vector data for both byte and float32 fields.+ // For byte vector fields (which bypass our FieldWriter), this is the only accounting.+ total += rawVectorDelegate.ramBytesUsed();for (FieldWriter field : fields) {- // the field tracks the delegate field usage- total += field.ramBytesUsed();+ // quantizationOverheadBytesUsed() intentionally excludes flatFieldVectorsWriter+ // because rawVectorDelegate.ramBytesUsed() already accounts for all flat vector+ // data at the writer level. Calling field.ramBytesUsed() here would double-count.+ total += field.quantizationOverheadBytesUsed();}return total;}@@ -530,11 +536,22 @@ class Lucene102BinaryQuantizedVectorsWriter extends FlatVectorsWriter {throw new UnsupportedOperationException();}+ /**+ * Returns the RAM usage of quantization-specific state only (magnitudes, dimensionSums, shallow+ * object overhead). The underlying flat vector data is tracked separately by the+ * rawVectorDelegate at the writer level to avoid double-counting.+ */+ long quantizationOverheadBytesUsed() {+ long size = SHALLOW_SIZE;+ size += magnitudes.ramBytesUsed();+ size += RamUsageEstimator.sizeOf(dimensionSums);+ return size;+ }+@Overridepublic long ramBytesUsed() {- long size = SHALLOW_SIZE;+ long size = quantizationOverheadBytesUsed();size += flatFieldVectorsWriter.ramBytesUsed();- size += magnitudes.ramBytesUsed();return size;}}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.javaindex 331b14ac18e..9d4b6f14107 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java@@ -282,9 +282,14 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite@Overridepublic long ramBytesUsed() {long total = SHALLOW_RAM_BYTES_USED;+ // The rawVectorDelegate tracks all vector data for both byte and float32 fields.+ // For byte vector fields (which bypass our FieldWriter), this is the only accounting.+ total += rawVectorDelegate.ramBytesUsed();for (FieldWriter field : fields) {- // the field tracks the delegate field usage- total += field.ramBytesUsed();+ // quantizationOverheadBytesUsed() intentionally excludes flatFieldVectorsWriter+ // because rawVectorDelegate.ramBytesUsed() already accounts for all flat vector+ // data at the writer level. Calling field.ramBytesUsed() here would double-count.+ total += field.quantizationOverheadBytesUsed();}return total;}@@ -727,9 +732,17 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWritereturn quantizer;}+ /**+ * Returns the RAM usage of quantization-specific state only. The underlying flat vector data is+ * tracked separately by the rawVectorDelegate at the writer level.+ */+ long quantizationOverheadBytesUsed() {+ return SHALLOW_SIZE;+ }+@Overridepublic long ramBytesUsed() {- long size = SHALLOW_SIZE;+ long size = quantizationOverheadBytesUsed();size += flatFieldVectorsWriter.ramBytesUsed();return size;}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.javaindex 32df27bad71..629afec1ec1 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java@@ -262,7 +262,7 @@ public abstract class BufferingKnnVectorsWriter extends KnnVectorsWriter {* (long)(RamUsageEstimator.NUM_BYTES_OBJECT_REF+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER)- + vectors.size() * (long) dim * Float.BYTES;+ + vectors.size() * (long) dim * fieldInfo.getVectorEncoding().byteSize;}}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.javaindex 5373ff85be3..af2374775be 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java@@ -49,6 +49,7 @@ import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.VectorScorer;import org.apache.lucene.store.IndexOutput;import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.RamUsageEstimator;import org.apache.lucene.util.VectorUtil;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;import org.apache.lucene.util.quantization.QuantizedByteVectorValues;@@ -508,9 +509,16 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {@Overridepublic long ramBytesUsed() {long total = SHALLOW_RAM_BYTES_USED;+ // The rawVectorDelegate tracks all vector data for both byte and float32 fields.+ // For byte vector fields (which bypass our FieldWriter), this is the only accounting.+ // For float32 fields, this covers the flat vector data; our FieldWriter adds the+ // quantization-specific overhead (magnitudes, dimensionSums) on top.+ total += rawVectorDelegate.ramBytesUsed();for (FieldWriter field : fields) {- // the field tracks the delegate field usage- total += field.ramBytesUsed();+ // quantizationOverheadBytesUsed() intentionally excludes flatFieldVectorsWriter+ // because rawVectorDelegate.ramBytesUsed() already accounts for all flat vector+ // data at the writer level. Calling field.ramBytesUsed() here would double-count.+ total += field.quantizationOverheadBytesUsed();}return total;}@@ -585,11 +593,22 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {throw new UnsupportedOperationException();}+ /**+ * Returns the RAM usage of quantization-specific state only (magnitudes, dimensionSums, shallow+ * object overhead). The underlying flat vector data is tracked separately by the+ * rawVectorDelegate at the writer level to avoid double-counting.+ */+ long quantizationOverheadBytesUsed() {+ long size = SHALLOW_SIZE;+ size += magnitudes.ramBytesUsed();+ size += RamUsageEstimator.sizeOf(dimensionSums);+ return size;+ }+@Overridepublic long ramBytesUsed() {- long size = SHALLOW_SIZE;+ long size = quantizationOverheadBytesUsed();size += flatFieldVectorsWriter.ramBytesUsed();- size += magnitudes.ramBytesUsed();return size;}}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.javaindex 8d338bc9d49..a7f3532d992 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java@@ -444,6 +444,75 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTedir.close();}+ @SuppressWarnings("unchecked")+ public void testWriterByteVectorRamEstimate() throws Exception {+ final FieldInfos fieldInfos = new FieldInfos(new FieldInfo[0]);+ final Directory dir = newDirectory();+ Codec codec = Codec.getDefault();+ final SegmentInfo si =+ new SegmentInfo(+ dir,+ Version.LATEST,+ Version.LATEST,+ "0",+ 10000,+ false,+ false,+ codec,+ Collections.emptyMap(),+ StringHelper.randomId(),+ new HashMap<>(),+ null);+ final SegmentWriteState state =+ new SegmentWriteState(+ InfoStream.getDefault(), dir, si, fieldInfos, null, newIOContext(random()));+ final KnnVectorsFormat format = codec.knnVectorsFormat();+ try (KnnVectorsWriter writer = format.fieldsWriter(state)) {+ int dim = random().nextInt(64) + 1;+ if (dim % 2 == 1) {+ ++dim;+ }+ int numDocs = atLeast(100);+ KnnFieldVectorsWriter<byte[]> fieldWriter =+ (KnnFieldVectorsWriter<byte[]>)+ writer.addField(+ new FieldInfo(+ "fieldA",+ 0,+ false,+ false,+ false,+ IndexOptions.NONE,+ DocValuesType.NONE,+ DocValuesSkipIndexType.NONE,+ -1,+ Map.of(),+ 0,+ 0,+ 0,+ dim,+ VectorEncoding.BYTE,+ VectorSimilarityFunction.DOT_PRODUCT,+ false,+ false));+ for (int i = 0; i < numDocs; i++) {+ fieldWriter.addValue(i, randomVector8(dim));+ }+ // Validate the field-level RAM accounting uses correct byte sizes.+ // The reported RAM must be at least the raw byte vector data.+ final long fieldRamBytesUsed = fieldWriter.ramBytesUsed();+ final long rawByteVectorData = (long) dim * numDocs * Byte.BYTES;+ assertTrue(+ "Expected field ramBytesUsed ("+ + fieldRamBytesUsed+ + ") >= raw byte vector data size ("+ + rawByteVectorData+ + ")",+ fieldRamBytesUsed >= rawByteVectorData);+ }+ dir.close();+ }+public void testIllegalSimilarityFunctionChangeTwoWriters() throws Exception {try (Directory dir = newDirectory()) {try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {