Apache Lucene open-source search software
Add KnnVectorsReader.getVectorCount to read vector counts from metadata (#16647)
Chris Hegarty874ba4d39afb
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 16f79b3dc3d..86797feadca 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -372,6 +372,8 @@ API Changes* GITHUB#16385: Remove unused public methods in AllGroupHeadsCollector.Update grouping package info to use CollectorManager instead of Collector. (Binlong Gao)+* GITHUB#16647: Add KnnVectorsReader.getVectorCount to read vector counts from metadata (Chris Hegarty)+New Features---------------------diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.javaindex 59fd7d34beb..e7c6a869150 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java@@ -315,6 +315,17 @@ public class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReaderreturn KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {+ Objects.requireNonNull(fieldInfo);+ FieldEntry fieldEntry = fields.get(fieldInfo.name);+ if (fieldEntry == null) {+ assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE;+ return rawVectorsReader.getVectorCount(fieldInfo);+ }+ return fieldEntry.size();+ }+float[] getCentroid(String field) {FieldEntry fieldEntry = fields.get(field);if (fieldEntry != null) {diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.javaindex 008c0831d9e..28e269d483e 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java@@ -331,6 +331,11 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {return Map.ofEntries(raw, graph);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ return getFieldEntry(fieldInfo.name).size();+ }+@Overridepublic void close() throws IOException {IOUtils.close(vectorData, vectorIndex);diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.javaindex 28570c8fd76..5ab743b9046 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java@@ -322,6 +322,11 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {return Map.ofEntries(raw, graph);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ return getFieldEntry(fieldInfo.name).size();+ }+@Overridepublic void close() throws IOException {IOUtils.close(vectorData, vectorIndex);diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.javaindex 24ab63067a9..86a02ec1a21 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java@@ -287,6 +287,11 @@ public final class Lucene92HnswVectorsReader extends KnnVectorsReader {return Map.ofEntries(raw, graph);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ return getFieldEntry(fieldInfo.name).size();+ }+@Overridepublic void close() throws IOException {IOUtils.close(vectorData, vectorIndex);diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.javaindex 6cddb7df1c0..194ec6ab27e 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java@@ -335,6 +335,11 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {return Map.ofEntries(raw, graph);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ return getFieldEntryOrThrow(fieldInfo.name).size();+ }+@Overridepublic void close() throws IOException {IOUtils.close(vectorData, vectorIndex);diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.javaindex 3f524541161..bed0e5a1b2a 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java@@ -386,6 +386,11 @@ public final class Lucene95HnswVectorsReader extends KnnVectorsReader implementsreturn Map.ofEntries(raw, graph);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ return getFieldEntryOrThrow(fieldInfo.name).size();+ }+@Overridepublic void close() throws IOException {IOUtils.close(vectorData, vectorIndex);diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.javaindex d2cb4e2b38a..57b1f7fe130 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java@@ -23,6 +23,7 @@ import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVeimport java.io.IOException;import java.util.Map;+import java.util.Objects;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.KnnVectorsReader;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;@@ -337,6 +338,17 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReadereturn KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {+ Objects.requireNonNull(fieldInfo);+ FieldEntry fieldEntry = fields.get(fieldInfo.number);+ if (fieldEntry == null) {+ assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE;+ return rawVectorsReader.getVectorCount(fieldInfo);+ }+ return fieldEntry.size();+ }+private FieldEntry readField(IndexInput input, int versionMeta, FieldInfo info)throws IOException {VectorEncoding vectorEncoding = readVectorEncoding(input);diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.javaindex 7c93ac8ffa6..2fc5a5ee9f2 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java@@ -345,6 +345,16 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {return Map.of(); // all in-heap}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ Objects.requireNonNull(fieldInfo);+ FieldEntry fieldEntry = fieldEntries.get(fieldInfo.number);+ if (fieldEntry == null) {+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" not found");+ }+ return fieldEntry.size();+ }+@Overridepublic void close() throws IOException {dataIn.close();diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.javaindex 8d7205b66e0..c98d83b3e13 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java@@ -168,6 +168,11 @@ public abstract class KnnVectorsFormat implements NamedSPILoader.NamedSPI {return Map.of();}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ throw new UnsupportedOperationException();+ }+@Overridepublic void close() {}};diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.javaindex 768dd6efcd0..1fec137f17f 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java@@ -185,6 +185,36 @@ public abstract class KnnVectorsReader implements Closeable {*/public void finishMerge() throws IOException {}+ /**+ * Returns the number of indexed vectors for the given field in this segment.+ *+ * <p>This has the same meaning as {@code get*VectorValues(field).size()}: the number of vector+ * ordinals stored for {@code fieldInfo} in this segment.+ *+ * <p>Standard vector formats read this from segment metadata without opening vector values. The+ * default implementation opens vector values as a fallback; callers that need to avoid that I/O+ * should use a reader that overrides this method.+ *+ * <p>Opening vector values may read index data from storage and prefetch into memory, especially+ * for sparse fields. That cost is why this metadata-only path exists, analogous to how doc values+ * expose counts through {@link org.apache.lucene.index.DocValuesSkipper#docCount()} rather than+ * iterating values.+ *+ * @param fieldInfo the fieldInfo+ * @return the number of indexed vectors for the field+ * @throws IllegalArgumentException if the field is not vector-indexed in this segment+ */+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {+ if (fieldInfo.getVectorDimension() <= 0) {+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" not found");+ }+ return switch (fieldInfo.getVectorEncoding()) {+ case FLOAT32 -> getFloatVectorValues(fieldInfo.name).size();+ case BYTE -> getByteVectorValues(fieldInfo.name).size();+ case FLOAT16 -> getFloat16VectorValues(fieldInfo.name).size();+ };+ }+/*** Returns the desired size of off-heap memory for the given field. This size can be used to help* determine the memory requirements for optimal search performance, which can be greatly affecteddiff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.javaindex 854787772d3..3e6065ff663 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java@@ -404,6 +404,17 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReaderreturn KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {+ Objects.requireNonNull(fieldInfo);+ FieldEntry fieldEntry = fields.get(fieldInfo.name);+ if (fieldEntry == null) {+ assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE;+ return rawVectorsReader.getVectorCount(fieldInfo);+ }+ return fieldEntry.size();+ }+public float[] getCentroid(String field) {FieldEntry fieldEntry = fields.get(field);if (fieldEntry != null) {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.javaindex ad046ae64da..f29417e66c7 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java@@ -192,6 +192,11 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {return Map.of(Lucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION, entry.vectorDataLength());}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ return getFieldEntryOrThrow(fieldInfo.name).size();+ }+@Overridepublic void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {CodecUtil.checksumEntireFile(vectorData, merge);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.javaindex f196df3fe3e..8538b16edb7 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java@@ -437,6 +437,11 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReaderreturn KnnVectorsReader.mergeOffHeapByteSizeMaps(flat, graph);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ return getFieldEntryOrThrow(fieldInfo.name).size();+ }+@Overridepublic void close() throws IOException {IOUtils.close(flatVectorsReader, vectorIndex);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.javaindex adc6c786982..f6ffdfca8a1 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java@@ -362,6 +362,15 @@ public abstract class PerFieldKnnVectorsFormat extends KnnVectorsFormat {return knnVectorsReader.getOffHeapByteSize(fieldInfo);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {+ KnnVectorsReader knnVectorsReader = fields.get(fieldInfo.number);+ if (knnVectorsReader == null) {+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" not found");+ }+ return knnVectorsReader.getVectorCount(fieldInfo);+ }+@Overridepublic void close() throws IOException {List<KnnVectorsReader> readers = new ArrayList<>(fields.size());diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.javaindex b082d984996..bfcf6a1120c 100644--- a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java@@ -213,15 +213,34 @@ public final class SlowCodecReaderWrapper {return vectorsReader.getOffHeapByteSize(fieldInfo);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {+ SegmentReader segmentReader = unwrapSegmentReader(reader);+ if (segmentReader != null) {+ var vectorsReader = segmentReader.getVectorReader();+ vectorsReader = vectorsReader.unwrapReaderForField(fieldInfo.name);+ return vectorsReader.getVectorCount(fieldInfo);+ }+ return super.getVectorCount(fieldInfo);+ }+static SegmentReader segmentReader(LeafReader reader) {+ SegmentReader segmentReader = unwrapSegmentReader(reader);+ if (segmentReader == null) {+ throw new AssertionError("unexpected reader [" + reader + "]");+ }+ return segmentReader;+ }++ static SegmentReader unwrapSegmentReader(LeafReader reader) {if (reader instanceof SegmentReader sr) {return sr;} else if (reader instanceof final FilterLeafReader fReader) {- return segmentReader(FilterLeafReader.unwrap(fReader));+ return unwrapSegmentReader(FilterLeafReader.unwrap(fReader));} else if (reader instanceof final FilterCodecReader fReader) {- return segmentReader(FilterCodecReader.unwrap(fReader));+ return unwrapSegmentReader(FilterCodecReader.unwrap(fReader));}- throw new AssertionError("unexpected reader [" + reader + "]");+ return null;}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.javaindex 6a01b1fa2f7..4fdeeb3a566 100644--- a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java@@ -849,6 +849,18 @@ final class SlowCompositeCodecReaderWrapper extends CodecReader {return map;}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {+ int count = 0;+ for (int i = 0; i < readers.length; i++) {+ FieldInfo subFieldInfo = codecReaders[i].getFieldInfos().fieldInfo(fieldInfo.name);+ if (subFieldInfo != null && subFieldInfo.getVectorDimension() > 0) {+ count += readers[i].getVectorCount(subFieldInfo);+ }+ }+ return count;+ }+class MergedFloatVectorValues extends FloatVectorValues {final int dimension;final int size;diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.javaindex 659218353ee..a4750214b98 100644--- a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java+++ b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java@@ -647,6 +647,11 @@ public final class SortingCodecReader extends FilterCodecReader {return delegate.getOffHeapByteSize(fieldInfo);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {+ return delegate.getVectorCount(fieldInfo);+ }+@Overridepublic void close() throws IOException {delegate.close();diff --git a/lucene/core/src/test/org/apache/lucene/index/TestGetVectorCountCodecWrappers.java b/lucene/core/src/test/org/apache/lucene/index/TestGetVectorCountCodecWrappers.javanew file mode 100644index 00000000000..85219d5e939--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/index/TestGetVectorCountCodecWrappers.java@@ -0,0 +1,63 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.index;++import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.util.LuceneTestCase;++/**+ * Tests {@link org.apache.lucene.codecs.KnnVectorsReader#getVectorCount} on codec reader wrappers.+ */+public class TestGetVectorCountCodecWrappers extends LuceneTestCase {++ public void testSlowCompositeCodecReaderWrapperGetVectorCount() throws Exception {+ try (Directory dir = newDirectory()) {+ IndexWriterConfig iwc = new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE);+ try (IndexWriter w = new IndexWriter(dir, iwc)) {+ Document doc = new Document();+ doc.add(+ new KnnFloatVectorField(+ "f", new float[] {1, 0, 0, 0}, VectorSimilarityFunction.DOT_PRODUCT));+ w.addDocument(doc);+ w.commit();+ doc = new Document();+ doc.add(+ new KnnFloatVectorField(+ "f", new float[] {0, 1, 0, 0}, VectorSimilarityFunction.DOT_PRODUCT));+ w.addDocument(doc);+ w.commit();+ }+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ assertEquals(2, reader.leaves().size());+ List<CodecReader> codecReaders = new ArrayList<>(2);+ int expected = 0;+ for (LeafReaderContext ctx : reader.leaves()) {+ codecReaders.add((CodecReader) ctx.reader());+ FloatVectorValues values = ctx.reader().getFloatVectorValues("f");+ expected += values.size();+ }+ CodecReader composite = SlowCompositeCodecReaderWrapper.wrap(codecReaders);+ FieldInfo fieldInfo = composite.getFieldInfos().fieldInfo("f");+ assertEquals(expected, composite.getVectorReader().getVectorCount(fieldInfo));+ }+ }+ }+}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.javaindex a54424abd09..87f3c51eadd 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java@@ -369,6 +369,15 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {entry.fieldInfo.fieldOrdToGroupOrdSize() + entry.groupInfo.vectorDataSize());}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ FieldEntry entry = fields.get(fieldInfo.name);+ if (entry == null) {+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" not found");+ }+ return entry.fieldInfo().vectorCount();+ }+// package-private for testingrecord FieldEntry(ReadFieldInfo fieldInfo, GroupInfo groupInfo) {private static final long SHALLOW_SIZE =diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.javaindex 5551e11e15a..d0f1a4337d3 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java@@ -568,6 +568,15 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReaderentry.quantizedBlock().quantizedDataSize());}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) {+ FieldEntry entry = fields.get(fieldInfo.name);+ if (entry == null) {+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" not found");+ }+ return entry.fieldInfo().vectorCount();+ }+// package-private for testingrecord FieldEntry(ReadFieldInfo fieldInfo, GroupInfo groupInfo, QuantizedBlock quantizedBlock) {private static final long SHALLOW_SIZE =diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsReader.javaindex 3d3e27db564..49386996fa7 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsReader.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsReader.java@@ -203,6 +203,11 @@ final class FaissKnnVectorsReader extends KnnVectorsReader {return rawVectorsReader.getOffHeapByteSize(fieldInfo);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {+ return rawVectorsReader.getVectorCount(fieldInfo);+ }+@Overridepublic void close() throws IOException {if (closed == false) {diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.javaindex 6cc9746b1f6..6a55353ffc9 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java@@ -263,6 +263,11 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {return delegate.getOffHeapByteSize(fieldInfo);}+ @Override+ public int getVectorCount(FieldInfo fieldInfo) throws IOException {+ return delegate.getVectorCount(fieldInfo);+ }+@Overridepublic void close() throws IOException {delegate.close();diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.javaindex 20e502887fe..4483cd52bc5 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java@@ -78,6 +78,8 @@ import org.apache.lucene.index.NoMergePolicy;import org.apache.lucene.index.SegmentInfo;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.index.SerialMergeScheduler;+import org.apache.lucene.index.SlowCodecReaderWrapper;+import org.apache.lucene.index.SortingCodecReader;import org.apache.lucene.index.StoredFields;import org.apache.lucene.index.Term;import org.apache.lucene.index.VectorEncoding;@@ -1046,6 +1048,117 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe}}+ /**+ * Verify {@link KnnVectorsReader#getVectorCount} matches opening vector values, without relying+ * on the metadata-only path alone. Indexes both dense fields (vector on every doc) and sparse+ * fields (vector on a small fraction of docs) in the same segment.+ */+ public void testVectorCount() throws Exception {+ int numDocs = atLeast(200);+ int numDenseFields = TestUtil.nextInt(random(), 1, 3);+ int numSparseFields = TestUtil.nextInt(random(), 1, 3);+ VectorSimilarityFunction[] denseSimilarityFunctions =+ new VectorSimilarityFunction[numDenseFields];+ VectorEncoding[] denseVectorEncodings = new VectorEncoding[numDenseFields];+ int[] denseDims = new int[numDenseFields];+ for (int i = 0; i < numDenseFields; i++) {+ denseDims[i] = random().nextInt(20) + 1;+ if (denseDims[i] % 2 != 0) {+ denseDims[i]++;+ }+ denseSimilarityFunctions[i] = randomSimilarity();+ denseVectorEncodings[i] = randomVectorEncoding();+ }+ VectorSimilarityFunction[] sparseSimilarityFunctions =+ new VectorSimilarityFunction[numSparseFields];+ VectorEncoding[] sparseVectorEncodings = new VectorEncoding[numSparseFields];+ int[] sparseDims = new int[numSparseFields];+ for (int i = 0; i < numSparseFields; i++) {+ sparseDims[i] = random().nextInt(20) + 1;+ if (sparseDims[i] % 2 != 0) {+ sparseDims[i]++;+ }+ sparseSimilarityFunctions[i] = randomSimilarity();+ sparseVectorEncodings[i] = randomVectorEncoding();+ }+ try (Directory dir = newDirectory();+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig())) {+ for (int i = 0; i < numDocs; i++) {+ Document doc = new Document();+ doc.add(new NumericDocValuesField("sortkey", i));+ for (int field = 0; field < numDenseFields; field++) {+ addRandomVectorField(+ doc,+ "dense" + field,+ denseVectorEncodings[field],+ denseDims[field],+ denseSimilarityFunctions[field]);+ }+ for (int field = 0; field < numSparseFields; field++) {+ // ~1% of docs carry a vector, exercising the IndexedDISI sparse path+ if (random().nextInt(100) == 17) {+ addRandomVectorField(+ doc,+ "sparse" + field,+ sparseVectorEncodings[field],+ sparseDims[field],+ sparseSimilarityFunctions[field]);+ }+ }+ w.addDocument(doc);+ }+ if (random().nextBoolean()) {+ w.forceMerge(1);+ }+ try (IndexReader reader = w.getReader()) {+ Sort sort = new Sort(new SortField("sortkey", SortField.Type.INT));+ for (LeafReaderContext ctx : reader.leaves()) {+ CodecReader codecReader = (CodecReader) ctx.reader();+ assertVectorCountMatchesVectorValuesSize(codecReader);+ assertVectorCountMatchesVectorValuesSize(SlowCodecReaderWrapper.wrap(codecReader));+ assertVectorCountMatchesVectorValuesSize(SortingCodecReader.wrap(codecReader, sort));+ }+ }+ }+ }++ /** Non-vector fields must not be passed to {@link KnnVectorsReader#getVectorCount}. */+ public void testGetVectorCountInvalidField() throws Exception {+ try (Directory dir = newDirectory();+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig())) {+ Document doc = new Document();+ doc.add(new StringField("text", "value", Field.Store.NO));+ addRandomVectorField(doc, "vector", randomVectorEncoding(), atLeast(2), randomSimilarity());+ w.addDocument(doc);+ try (IndexReader reader = w.getReader()) {+ LeafReader leafReader = reader.leaves().get(0).reader();+ FieldInfo textField = leafReader.getFieldInfos().fieldInfo("text");+ KnnVectorsReader vectorsReader = ((CodecReader) leafReader).getVectorReader();+ expectThrows(IllegalArgumentException.class, () -> vectorsReader.getVectorCount(textField));+ }+ }+ }++ private void addRandomVectorField(+ Document doc,+ String fieldName,+ VectorEncoding encoding,+ int dimension,+ VectorSimilarityFunction similarityFunction) {+ switch (encoding) {+ case BYTE ->+ doc.add(new KnnByteVectorField(fieldName, randomVector8(dimension), similarityFunction));+ case FLOAT16 ->+ doc.add(+ new KnnFloat16VectorField(+ fieldName, randomNormalizedFloat16Vector(dimension), similarityFunction));+ case FLOAT32 ->+ doc.add(+ new KnnFloatVectorField(+ fieldName, randomNormalizedVector(dimension), similarityFunction));+ }+ }+public void testFloatVectorScorerIteration() throws Exception {IndexWriterConfig iwc = newIndexWriterConfig();if (random().nextBoolean()) {@@ -2499,10 +2612,50 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTeprotected static int getNumVectors(KnnVectorsReader reader, FieldInfo fieldInfo)throws IOException {+ return reader.getVectorCount(fieldInfo);+ }++ protected void assertVectorCountMatchesVectorValuesSize(LeafReader leafReader)+ throws IOException {+ if (leafReader instanceof CodecReader codecReader) {+ assertVectorCountMatchesVectorValuesSize(codecReader);+ }+ }++ protected void assertVectorCountMatchesVectorValuesSize(CodecReader codecReader)+ throws IOException {+ KnnVectorsReader vectorsReader = codecReader.getVectorReader();+ for (FieldInfo fieldInfo : codecReader.getFieldInfos()) {+ if (fieldInfo.getVectorDimension() <= 0) {+ continue;+ }+ int expected = countVectorsFromValues(codecReader, fieldInfo);+ assertEquals(+ "wrapper vector count for field=" + fieldInfo.name,+ expected,+ vectorsReader.getVectorCount(fieldInfo));+ assertEquals(+ "format vector count for field=" + fieldInfo.name,+ expected,+ vectorsReader.unwrapReaderForField(fieldInfo.name).getVectorCount(fieldInfo));+ }+ }++ private static int countVectorsFromValues(LeafReader leafReader, FieldInfo fieldInfo)+ throws IOException {return switch (fieldInfo.getVectorEncoding()) {- case BYTE -> reader.getByteVectorValues(fieldInfo.getName()).size();- case FLOAT32 -> reader.getFloatVectorValues(fieldInfo.getName()).size();- case FLOAT16 -> reader.getFloat16VectorValues(fieldInfo.getName()).size();+ case BYTE -> {+ ByteVectorValues values = leafReader.getByteVectorValues(fieldInfo.name);+ yield values != null ? values.size() : 0;+ }+ case FLOAT16 -> {+ Float16VectorValues values = leafReader.getFloat16VectorValues(fieldInfo.name);+ yield values != null ? values.size() : 0;+ }+ case FLOAT32 -> {+ FloatVectorValues values = leafReader.getFloatVectorValues(fieldInfo.name);+ yield values != null ? values.size() : 0;+ }};}