Skip to content

Apache Lucene open-source search software

Fix undercounting of RAM used by vectors buffered in in-memory segments (#15982)

Prithvi S735c6e026683
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2bbfea5a2c6..34b393064cf 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -158,6 +158,13 @@ Optimizations
Bug Fixes
---------------------
+* GITHUB#15901: Fix undercounting of RAM used by vectors buffered in in-memory segments.
+ BufferingKnnVectorsWriter hardcoded Float.BYTES for all vector encodings, overcounting byte
+ vectors by 4x. Scalar quantized vector writers (Lucene104, Lucene99, Lucene102) did not
+ account for rawVectorDelegate RAM, making byte vector fields completely invisible in RAM
+ reporting. Also added missing dimensionSums array accounting in quantized writers.
+ (Prithvi S)
+
* GITHUB#14049: Randomize KNN codec params in RandomCodec. Fixes scalar quantization div-by-zero
when all values are identical. (Mike Sokolov)
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
index 4b873be3608..796bc5511c7 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
@@ -51,6 +51,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
@@ -453,9 +454,14 @@ class Lucene102BinaryQuantizedVectorsWriter extends FlatVectorsWriter {
@Override
public long ramBytesUsed() {
long total = SHALLOW_RAM_BYTES_USED;
+ // The rawVectorDelegate tracks all vector data for both byte and float32 fields.
+ // For byte vector fields (which bypass our FieldWriter), this is the only accounting.
+ total += rawVectorDelegate.ramBytesUsed();
for (FieldWriter field : fields) {
- // the field tracks the delegate field usage
- total += field.ramBytesUsed();
+ // quantizationOverheadBytesUsed() intentionally excludes flatFieldVectorsWriter
+ // because rawVectorDelegate.ramBytesUsed() already accounts for all flat vector
+ // data at the writer level. Calling field.ramBytesUsed() here would double-count.
+ total += field.quantizationOverheadBytesUsed();
}
return total;
}
@@ -530,11 +536,22 @@ class Lucene102BinaryQuantizedVectorsWriter extends FlatVectorsWriter {
throw new UnsupportedOperationException();
}
+ /**
+ * Returns the RAM usage of quantization-specific state only (magnitudes, dimensionSums, shallow
+ * object overhead). The underlying flat vector data is tracked separately by the
+ * rawVectorDelegate at the writer level to avoid double-counting.
+ */
+ long quantizationOverheadBytesUsed() {
+ long size = SHALLOW_SIZE;
+ size += magnitudes.ramBytesUsed();
+ size += RamUsageEstimator.sizeOf(dimensionSums);
+ return size;
+ }
+
@Override
public long ramBytesUsed() {
- long size = SHALLOW_SIZE;
+ long size = quantizationOverheadBytesUsed();
size += flatFieldVectorsWriter.ramBytesUsed();
- size += magnitudes.ramBytesUsed();
return size;
}
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
index 331b14ac18e..9d4b6f14107 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
@@ -282,9 +282,14 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
@Override
public long ramBytesUsed() {
long total = SHALLOW_RAM_BYTES_USED;
+ // The rawVectorDelegate tracks all vector data for both byte and float32 fields.
+ // For byte vector fields (which bypass our FieldWriter), this is the only accounting.
+ total += rawVectorDelegate.ramBytesUsed();
for (FieldWriter field : fields) {
- // the field tracks the delegate field usage
- total += field.ramBytesUsed();
+ // quantizationOverheadBytesUsed() intentionally excludes flatFieldVectorsWriter
+ // because rawVectorDelegate.ramBytesUsed() already accounts for all flat vector
+ // data at the writer level. Calling field.ramBytesUsed() here would double-count.
+ total += field.quantizationOverheadBytesUsed();
}
return total;
}
@@ -727,9 +732,17 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
return quantizer;
}
+ /**
+ * Returns the RAM usage of quantization-specific state only. The underlying flat vector data is
+ * tracked separately by the rawVectorDelegate at the writer level.
+ */
+ long quantizationOverheadBytesUsed() {
+ return SHALLOW_SIZE;
+ }
+
@Override
public long ramBytesUsed() {
- long size = SHALLOW_SIZE;
+ long size = quantizationOverheadBytesUsed();
size += flatFieldVectorsWriter.ramBytesUsed();
return size;
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java
index 32df27bad71..629afec1ec1 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java
@@ -262,7 +262,7 @@ public abstract class BufferingKnnVectorsWriter extends KnnVectorsWriter {
* (long)
(RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER)
- + vectors.size() * (long) dim * Float.BYTES;
+ + vectors.size() * (long) dim * fieldInfo.getVectorEncoding().byteSize;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
index 5373ff85be3..af2374775be 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
@@ -49,6 +49,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
@@ -508,9 +509,16 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
@Override
public long ramBytesUsed() {
long total = SHALLOW_RAM_BYTES_USED;
+ // The rawVectorDelegate tracks all vector data for both byte and float32 fields.
+ // For byte vector fields (which bypass our FieldWriter), this is the only accounting.
+ // For float32 fields, this covers the flat vector data; our FieldWriter adds the
+ // quantization-specific overhead (magnitudes, dimensionSums) on top.
+ total += rawVectorDelegate.ramBytesUsed();
for (FieldWriter field : fields) {
- // the field tracks the delegate field usage
- total += field.ramBytesUsed();
+ // quantizationOverheadBytesUsed() intentionally excludes flatFieldVectorsWriter
+ // because rawVectorDelegate.ramBytesUsed() already accounts for all flat vector
+ // data at the writer level. Calling field.ramBytesUsed() here would double-count.
+ total += field.quantizationOverheadBytesUsed();
}
return total;
}
@@ -585,11 +593,22 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
throw new UnsupportedOperationException();
}
+ /**
+ * Returns the RAM usage of quantization-specific state only (magnitudes, dimensionSums, shallow
+ * object overhead). The underlying flat vector data is tracked separately by the
+ * rawVectorDelegate at the writer level to avoid double-counting.
+ */
+ long quantizationOverheadBytesUsed() {
+ long size = SHALLOW_SIZE;
+ size += magnitudes.ramBytesUsed();
+ size += RamUsageEstimator.sizeOf(dimensionSums);
+ return size;
+ }
+
@Override
public long ramBytesUsed() {
- long size = SHALLOW_SIZE;
+ long size = quantizationOverheadBytesUsed();
size += flatFieldVectorsWriter.ramBytesUsed();
- size += magnitudes.ramBytesUsed();
return size;
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
index 8d338bc9d49..a7f3532d992 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
@@ -444,6 +444,75 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
dir.close();
}
+ @SuppressWarnings("unchecked")
+ public void testWriterByteVectorRamEstimate() throws Exception {
+ final FieldInfos fieldInfos = new FieldInfos(new FieldInfo[0]);
+ final Directory dir = newDirectory();
+ Codec codec = Codec.getDefault();
+ final SegmentInfo si =
+ new SegmentInfo(
+ dir,
+ Version.LATEST,
+ Version.LATEST,
+ "0",
+ 10000,
+ false,
+ false,
+ codec,
+ Collections.emptyMap(),
+ StringHelper.randomId(),
+ new HashMap<>(),
+ null);
+ final SegmentWriteState state =
+ new SegmentWriteState(
+ InfoStream.getDefault(), dir, si, fieldInfos, null, newIOContext(random()));
+ final KnnVectorsFormat format = codec.knnVectorsFormat();
+ try (KnnVectorsWriter writer = format.fieldsWriter(state)) {
+ int dim = random().nextInt(64) + 1;
+ if (dim % 2 == 1) {
+ ++dim;
+ }
+ int numDocs = atLeast(100);
+ KnnFieldVectorsWriter<byte[]> fieldWriter =
+ (KnnFieldVectorsWriter<byte[]>)
+ writer.addField(
+ new FieldInfo(
+ "fieldA",
+ 0,
+ false,
+ false,
+ false,
+ IndexOptions.NONE,
+ DocValuesType.NONE,
+ DocValuesSkipIndexType.NONE,
+ -1,
+ Map.of(),
+ 0,
+ 0,
+ 0,
+ dim,
+ VectorEncoding.BYTE,
+ VectorSimilarityFunction.DOT_PRODUCT,
+ false,
+ false));
+ for (int i = 0; i < numDocs; i++) {
+ fieldWriter.addValue(i, randomVector8(dim));
+ }
+ // Validate the field-level RAM accounting uses correct byte sizes.
+ // The reported RAM must be at least the raw byte vector data.
+ final long fieldRamBytesUsed = fieldWriter.ramBytesUsed();
+ final long rawByteVectorData = (long) dim * numDocs * Byte.BYTES;
+ assertTrue(
+ "Expected field ramBytesUsed ("
+ + fieldRamBytesUsed
+ + ") >= raw byte vector data size ("
+ + rawByteVectorData
+ + ")",
+ fieldRamBytesUsed >= rawByteVectorData);
+ }
+ dir.close();
+ }
+
public void testIllegalSimilarityFunctionChangeTwoWriters() throws Exception {
try (Directory dir = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {