Skip to content

Apache Lucene open-source search software

Add KnnVectorsReader.getVectorCount to read vector counts from metadata (#16647)

Chris Hegarty874ba4d39afb
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 16f79b3dc3d..86797feadca 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -372,6 +372,8 @@ API Changes
* GITHUB#16385: Remove unused public methods in AllGroupHeadsCollector.
Update grouping package info to use CollectorManager instead of Collector. (Binlong Gao)
+* GITHUB#16647: Add KnnVectorsReader.getVectorCount to read vector counts from metadata (Chris Hegarty)
+
New Features
---------------------
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
index 59fd7d34beb..e7c6a869150 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
@@ -315,6 +315,17 @@ public class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader
return KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {
+ Objects.requireNonNull(fieldInfo);
+ FieldEntry fieldEntry = fields.get(fieldInfo.name);
+ if (fieldEntry == null) {
+ assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE;
+ return rawVectorsReader.getVectorCount(fieldInfo);
+ }
+ return fieldEntry.size();
+ }
+
float[] getCentroid(String field) {
FieldEntry fieldEntry = fields.get(field);
if (fieldEntry != null) {
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
index 008c0831d9e..28e269d483e 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
@@ -331,6 +331,11 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
return Map.ofEntries(raw, graph);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ return getFieldEntry(fieldInfo.name).size();
+ }
+
@Override
public void close() throws IOException {
IOUtils.close(vectorData, vectorIndex);
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
index 28570c8fd76..5ab743b9046 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
@@ -322,6 +322,11 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
return Map.ofEntries(raw, graph);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ return getFieldEntry(fieldInfo.name).size();
+ }
+
@Override
public void close() throws IOException {
IOUtils.close(vectorData, vectorIndex);
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java
index 24ab63067a9..86a02ec1a21 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java
@@ -287,6 +287,11 @@ public final class Lucene92HnswVectorsReader extends KnnVectorsReader {
return Map.ofEntries(raw, graph);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ return getFieldEntry(fieldInfo.name).size();
+ }
+
@Override
public void close() throws IOException {
IOUtils.close(vectorData, vectorIndex);
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
index 6cddb7df1c0..194ec6ab27e 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
@@ -335,6 +335,11 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
return Map.ofEntries(raw, graph);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ return getFieldEntryOrThrow(fieldInfo.name).size();
+ }
+
@Override
public void close() throws IOException {
IOUtils.close(vectorData, vectorIndex);
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java
index 3f524541161..bed0e5a1b2a 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java
@@ -386,6 +386,11 @@ public final class Lucene95HnswVectorsReader extends KnnVectorsReader implements
return Map.ofEntries(raw, graph);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ return getFieldEntryOrThrow(fieldInfo.name).size();
+ }
+
@Override
public void close() throws IOException {
IOUtils.close(vectorData, vectorIndex);
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
index d2cb4e2b38a..57b1f7fe130 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
@@ -23,6 +23,7 @@ import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVe
import java.io.IOException;
import java.util.Map;
+import java.util.Objects;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
@@ -337,6 +338,17 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
return KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {
+ Objects.requireNonNull(fieldInfo);
+ FieldEntry fieldEntry = fields.get(fieldInfo.number);
+ if (fieldEntry == null) {
+ assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE;
+ return rawVectorsReader.getVectorCount(fieldInfo);
+ }
+ return fieldEntry.size();
+ }
+
private FieldEntry readField(IndexInput input, int versionMeta, FieldInfo info)
throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
index 7c93ac8ffa6..2fc5a5ee9f2 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
@@ -345,6 +345,16 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
return Map.of(); // all in-heap
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ Objects.requireNonNull(fieldInfo);
+ FieldEntry fieldEntry = fieldEntries.get(fieldInfo.number);
+ if (fieldEntry == null) {
+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" not found");
+ }
+ return fieldEntry.size();
+ }
+
@Override
public void close() throws IOException {
dataIn.close();
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java
index 8d7205b66e0..c98d83b3e13 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java
@@ -168,6 +168,11 @@ public abstract class KnnVectorsFormat implements NamedSPILoader.NamedSPI {
return Map.of();
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public void close() {}
};
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java
index 768dd6efcd0..1fec137f17f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java
@@ -185,6 +185,36 @@ public abstract class KnnVectorsReader implements Closeable {
*/
public void finishMerge() throws IOException {}
+ /**
+ * Returns the number of indexed vectors for the given field in this segment.
+ *
+ * <p>This has the same meaning as {@code get*VectorValues(field).size()}: the number of vector
+ * ordinals stored for {@code fieldInfo} in this segment.
+ *
+ * <p>Standard vector formats read this from segment metadata without opening vector values. The
+ * default implementation opens vector values as a fallback; callers that need to avoid that I/O
+ * should use a reader that overrides this method.
+ *
+ * <p>Opening vector values may read index data from storage and prefetch into memory, especially
+ * for sparse fields. That cost is why this metadata-only path exists, analogous to how doc values
+ * expose counts through {@link org.apache.lucene.index.DocValuesSkipper#docCount()} rather than
+ * iterating values.
+ *
+ * @param fieldInfo the fieldInfo
+ * @return the number of indexed vectors for the field
+ * @throws IllegalArgumentException if the field is not vector-indexed in this segment
+ */
+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {
+ if (fieldInfo.getVectorDimension() <= 0) {
+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" not found");
+ }
+ return switch (fieldInfo.getVectorEncoding()) {
+ case FLOAT32 -> getFloatVectorValues(fieldInfo.name).size();
+ case BYTE -> getByteVectorValues(fieldInfo.name).size();
+ case FLOAT16 -> getFloat16VectorValues(fieldInfo.name).size();
+ };
+ }
+
/**
* Returns the desired size of off-heap memory for the given field. This size can be used to help
* determine the memory requirements for optimal search performance, which can be greatly affected
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index 854787772d3..3e6065ff663 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -404,6 +404,17 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
return KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {
+ Objects.requireNonNull(fieldInfo);
+ FieldEntry fieldEntry = fields.get(fieldInfo.name);
+ if (fieldEntry == null) {
+ assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE;
+ return rawVectorsReader.getVectorCount(fieldInfo);
+ }
+ return fieldEntry.size();
+ }
+
public float[] getCentroid(String field) {
FieldEntry fieldEntry = fields.get(field);
if (fieldEntry != null) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
index ad046ae64da..f29417e66c7 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
@@ -192,6 +192,11 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {
return Map.of(Lucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION, entry.vectorDataLength());
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ return getFieldEntryOrThrow(fieldInfo.name).size();
+ }
+
@Override
public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
CodecUtil.checksumEntireFile(vectorData, merge);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
index f196df3fe3e..8538b16edb7 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
@@ -437,6 +437,11 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader
return KnnVectorsReader.mergeOffHeapByteSizeMaps(flat, graph);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ return getFieldEntryOrThrow(fieldInfo.name).size();
+ }
+
@Override
public void close() throws IOException {
IOUtils.close(flatVectorsReader, vectorIndex);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java
index adc6c786982..f6ffdfca8a1 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java
@@ -362,6 +362,15 @@ public abstract class PerFieldKnnVectorsFormat extends KnnVectorsFormat {
return knnVectorsReader.getOffHeapByteSize(fieldInfo);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {
+ KnnVectorsReader knnVectorsReader = fields.get(fieldInfo.number);
+ if (knnVectorsReader == null) {
+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" not found");
+ }
+ return knnVectorsReader.getVectorCount(fieldInfo);
+ }
+
@Override
public void close() throws IOException {
List<KnnVectorsReader> readers = new ArrayList<>(fields.size());
diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
index b082d984996..bfcf6a1120c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
@@ -213,15 +213,34 @@ public final class SlowCodecReaderWrapper {
return vectorsReader.getOffHeapByteSize(fieldInfo);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {
+ SegmentReader segmentReader = unwrapSegmentReader(reader);
+ if (segmentReader != null) {
+ var vectorsReader = segmentReader.getVectorReader();
+ vectorsReader = vectorsReader.unwrapReaderForField(fieldInfo.name);
+ return vectorsReader.getVectorCount(fieldInfo);
+ }
+ return super.getVectorCount(fieldInfo);
+ }
+
static SegmentReader segmentReader(LeafReader reader) {
+ SegmentReader segmentReader = unwrapSegmentReader(reader);
+ if (segmentReader == null) {
+ throw new AssertionError("unexpected reader [" + reader + "]");
+ }
+ return segmentReader;
+ }
+
+ static SegmentReader unwrapSegmentReader(LeafReader reader) {
if (reader instanceof SegmentReader sr) {
return sr;
} else if (reader instanceof final FilterLeafReader fReader) {
- return segmentReader(FilterLeafReader.unwrap(fReader));
+ return unwrapSegmentReader(FilterLeafReader.unwrap(fReader));
} else if (reader instanceof final FilterCodecReader fReader) {
- return segmentReader(FilterCodecReader.unwrap(fReader));
+ return unwrapSegmentReader(FilterCodecReader.unwrap(fReader));
}
- throw new AssertionError("unexpected reader [" + reader + "]");
+ return null;
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java
index 6a01b1fa2f7..4fdeeb3a566 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java
@@ -849,6 +849,18 @@ final class SlowCompositeCodecReaderWrapper extends CodecReader {
return map;
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {
+ int count = 0;
+ for (int i = 0; i < readers.length; i++) {
+ FieldInfo subFieldInfo = codecReaders[i].getFieldInfos().fieldInfo(fieldInfo.name);
+ if (subFieldInfo != null && subFieldInfo.getVectorDimension() > 0) {
+ count += readers[i].getVectorCount(subFieldInfo);
+ }
+ }
+ return count;
+ }
+
class MergedFloatVectorValues extends FloatVectorValues {
final int dimension;
final int size;
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
index 659218353ee..a4750214b98 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
@@ -647,6 +647,11 @@ public final class SortingCodecReader extends FilterCodecReader {
return delegate.getOffHeapByteSize(fieldInfo);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {
+ return delegate.getVectorCount(fieldInfo);
+ }
+
@Override
public void close() throws IOException {
delegate.close();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestGetVectorCountCodecWrappers.java b/lucene/core/src/test/org/apache/lucene/index/TestGetVectorCountCodecWrappers.java
new file mode 100644
index 00000000000..85219d5e939
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestGetVectorCountCodecWrappers.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+/**
+ * Tests {@link org.apache.lucene.codecs.KnnVectorsReader#getVectorCount} on codec reader wrappers.
+ */
+public class TestGetVectorCountCodecWrappers extends LuceneTestCase {
+
+ public void testSlowCompositeCodecReaderWrapperGetVectorCount() throws Exception {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE);
+ try (IndexWriter w = new IndexWriter(dir, iwc)) {
+ Document doc = new Document();
+ doc.add(
+ new KnnFloatVectorField(
+ "f", new float[] {1, 0, 0, 0}, VectorSimilarityFunction.DOT_PRODUCT));
+ w.addDocument(doc);
+ w.commit();
+ doc = new Document();
+ doc.add(
+ new KnnFloatVectorField(
+ "f", new float[] {0, 1, 0, 0}, VectorSimilarityFunction.DOT_PRODUCT));
+ w.addDocument(doc);
+ w.commit();
+ }
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ assertEquals(2, reader.leaves().size());
+ List<CodecReader> codecReaders = new ArrayList<>(2);
+ int expected = 0;
+ for (LeafReaderContext ctx : reader.leaves()) {
+ codecReaders.add((CodecReader) ctx.reader());
+ FloatVectorValues values = ctx.reader().getFloatVectorValues("f");
+ expected += values.size();
+ }
+ CodecReader composite = SlowCompositeCodecReaderWrapper.wrap(codecReaders);
+ FieldInfo fieldInfo = composite.getFieldInfos().fieldInfo("f");
+ assertEquals(expected, composite.getVectorReader().getVectorCount(fieldInfo));
+ }
+ }
+ }
+}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java
index a54424abd09..87f3c51eadd 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java
@@ -369,6 +369,15 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
entry.fieldInfo.fieldOrdToGroupOrdSize() + entry.groupInfo.vectorDataSize());
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ FieldEntry entry = fields.get(fieldInfo.name);
+ if (entry == null) {
+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" not found");
+ }
+ return entry.fieldInfo().vectorCount();
+ }
+
// package-private for testing
record FieldEntry(ReadFieldInfo fieldInfo, GroupInfo groupInfo) {
private static final long SHALLOW_SIZE =
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
index 5551e11e15a..d0f1a4337d3 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
@@ -568,6 +568,15 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
entry.quantizedBlock().quantizedDataSize());
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) {
+ FieldEntry entry = fields.get(fieldInfo.name);
+ if (entry == null) {
+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" not found");
+ }
+ return entry.fieldInfo().vectorCount();
+ }
+
// package-private for testing
record FieldEntry(ReadFieldInfo fieldInfo, GroupInfo groupInfo, QuantizedBlock quantizedBlock) {
private static final long SHALLOW_SIZE =
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsReader.java
index 3d3e27db564..49386996fa7 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsReader.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsReader.java
@@ -203,6 +203,11 @@ final class FaissKnnVectorsReader extends KnnVectorsReader {
return rawVectorsReader.getOffHeapByteSize(fieldInfo);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {
+ return rawVectorsReader.getVectorCount(fieldInfo);
+ }
+
@Override
public void close() throws IOException {
if (closed == false) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java
index 6cc9746b1f6..6a55353ffc9 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java
@@ -263,6 +263,11 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {
return delegate.getOffHeapByteSize(fieldInfo);
}
+ @Override
+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {
+ return delegate.getVectorCount(fieldInfo);
+ }
+
@Override
public void close() throws IOException {
delegate.close();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
index 20e502887fe..4483cd52bc5 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
@@ -78,6 +78,8 @@ import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.index.SlowCodecReaderWrapper;
+import org.apache.lucene.index.SortingCodecReader;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.VectorEncoding;
@@ -1046,6 +1048,117 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
}
}
+ /**
+ * Verify {@link KnnVectorsReader#getVectorCount} matches opening vector values, without relying
+ * on the metadata-only path alone. Indexes both dense fields (vector on every doc) and sparse
+ * fields (vector on a small fraction of docs) in the same segment.
+ */
+ public void testVectorCount() throws Exception {
+ int numDocs = atLeast(200);
+ int numDenseFields = TestUtil.nextInt(random(), 1, 3);
+ int numSparseFields = TestUtil.nextInt(random(), 1, 3);
+ VectorSimilarityFunction[] denseSimilarityFunctions =
+ new VectorSimilarityFunction[numDenseFields];
+ VectorEncoding[] denseVectorEncodings = new VectorEncoding[numDenseFields];
+ int[] denseDims = new int[numDenseFields];
+ for (int i = 0; i < numDenseFields; i++) {
+ denseDims[i] = random().nextInt(20) + 1;
+ if (denseDims[i] % 2 != 0) {
+ denseDims[i]++;
+ }
+ denseSimilarityFunctions[i] = randomSimilarity();
+ denseVectorEncodings[i] = randomVectorEncoding();
+ }
+ VectorSimilarityFunction[] sparseSimilarityFunctions =
+ new VectorSimilarityFunction[numSparseFields];
+ VectorEncoding[] sparseVectorEncodings = new VectorEncoding[numSparseFields];
+ int[] sparseDims = new int[numSparseFields];
+ for (int i = 0; i < numSparseFields; i++) {
+ sparseDims[i] = random().nextInt(20) + 1;
+ if (sparseDims[i] % 2 != 0) {
+ sparseDims[i]++;
+ }
+ sparseSimilarityFunctions[i] = randomSimilarity();
+ sparseVectorEncodings[i] = randomVectorEncoding();
+ }
+ try (Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig())) {
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("sortkey", i));
+ for (int field = 0; field < numDenseFields; field++) {
+ addRandomVectorField(
+ doc,
+ "dense" + field,
+ denseVectorEncodings[field],
+ denseDims[field],
+ denseSimilarityFunctions[field]);
+ }
+ for (int field = 0; field < numSparseFields; field++) {
+ // ~1% of docs carry a vector, exercising the IndexedDISI sparse path
+ if (random().nextInt(100) == 17) {
+ addRandomVectorField(
+ doc,
+ "sparse" + field,
+ sparseVectorEncodings[field],
+ sparseDims[field],
+ sparseSimilarityFunctions[field]);
+ }
+ }
+ w.addDocument(doc);
+ }
+ if (random().nextBoolean()) {
+ w.forceMerge(1);
+ }
+ try (IndexReader reader = w.getReader()) {
+ Sort sort = new Sort(new SortField("sortkey", SortField.Type.INT));
+ for (LeafReaderContext ctx : reader.leaves()) {
+ CodecReader codecReader = (CodecReader) ctx.reader();
+ assertVectorCountMatchesVectorValuesSize(codecReader);
+ assertVectorCountMatchesVectorValuesSize(SlowCodecReaderWrapper.wrap(codecReader));
+ assertVectorCountMatchesVectorValuesSize(SortingCodecReader.wrap(codecReader, sort));
+ }
+ }
+ }
+ }
+
+ /** Non-vector fields must not be passed to {@link KnnVectorsReader#getVectorCount}. */
+ public void testGetVectorCountInvalidField() throws Exception {
+ try (Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig())) {
+ Document doc = new Document();
+ doc.add(new StringField("text", "value", Field.Store.NO));
+ addRandomVectorField(doc, "vector", randomVectorEncoding(), atLeast(2), randomSimilarity());
+ w.addDocument(doc);
+ try (IndexReader reader = w.getReader()) {
+ LeafReader leafReader = reader.leaves().get(0).reader();
+ FieldInfo textField = leafReader.getFieldInfos().fieldInfo("text");
+ KnnVectorsReader vectorsReader = ((CodecReader) leafReader).getVectorReader();
+ expectThrows(IllegalArgumentException.class, () -> vectorsReader.getVectorCount(textField));
+ }
+ }
+ }
+
+ private void addRandomVectorField(
+ Document doc,
+ String fieldName,
+ VectorEncoding encoding,
+ int dimension,
+ VectorSimilarityFunction similarityFunction) {
+ switch (encoding) {
+ case BYTE ->
+ doc.add(new KnnByteVectorField(fieldName, randomVector8(dimension), similarityFunction));
+ case FLOAT16 ->
+ doc.add(
+ new KnnFloat16VectorField(
+ fieldName, randomNormalizedFloat16Vector(dimension), similarityFunction));
+ case FLOAT32 ->
+ doc.add(
+ new KnnFloatVectorField(
+ fieldName, randomNormalizedVector(dimension), similarityFunction));
+ }
+ }
+
public void testFloatVectorScorerIteration() throws Exception {
IndexWriterConfig iwc = newIndexWriterConfig();
if (random().nextBoolean()) {
@@ -2499,10 +2612,50 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
protected static int getNumVectors(KnnVectorsReader reader, FieldInfo fieldInfo)
throws IOException {
+ return reader.getVectorCount(fieldInfo);
+ }
+
+ protected void assertVectorCountMatchesVectorValuesSize(LeafReader leafReader)
+ throws IOException {
+ if (leafReader instanceof CodecReader codecReader) {
+ assertVectorCountMatchesVectorValuesSize(codecReader);
+ }
+ }
+
+ protected void assertVectorCountMatchesVectorValuesSize(CodecReader codecReader)
+ throws IOException {
+ KnnVectorsReader vectorsReader = codecReader.getVectorReader();
+ for (FieldInfo fieldInfo : codecReader.getFieldInfos()) {
+ if (fieldInfo.getVectorDimension() <= 0) {
+ continue;
+ }
+ int expected = countVectorsFromValues(codecReader, fieldInfo);
+ assertEquals(
+ "wrapper vector count for field=" + fieldInfo.name,
+ expected,
+ vectorsReader.getVectorCount(fieldInfo));
+ assertEquals(
+ "format vector count for field=" + fieldInfo.name,
+ expected,
+ vectorsReader.unwrapReaderForField(fieldInfo.name).getVectorCount(fieldInfo));
+ }
+ }
+
+ private static int countVectorsFromValues(LeafReader leafReader, FieldInfo fieldInfo)
+ throws IOException {
return switch (fieldInfo.getVectorEncoding()) {
- case BYTE -> reader.getByteVectorValues(fieldInfo.getName()).size();
- case FLOAT32 -> reader.getFloatVectorValues(fieldInfo.getName()).size();
- case FLOAT16 -> reader.getFloat16VectorValues(fieldInfo.getName()).size();
+ case BYTE -> {
+ ByteVectorValues values = leafReader.getByteVectorValues(fieldInfo.name);
+ yield values != null ? values.size() : 0;
+ }
+ case FLOAT16 -> {
+ Float16VectorValues values = leafReader.getFloat16VectorValues(fieldInfo.name);
+ yield values != null ? values.size() : 0;
+ }
+ case FLOAT32 -> {
+ FloatVectorValues values = leafReader.getFloatVectorValues(fieldInfo.name);
+ yield values != null ? values.size() : 0;
+ }
};
}