Apache Lucene open-source search software
GITHUB#11911: improve checkindex to be more thorough for vectors (#11916)
Benjamin Trent3a506ec87a01
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 5a286300867..aee9d66a641 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -187,6 +187,10 @@ Build======================== Lucene 9.4.2 =======================+Improvements+---------------------+* GITHUB#11916: improve checkindex to be more thorough for vectors. (Ben Trent)+Bug Fixes---------------------* GITHUB#11905: Fix integer overflow when seeking the vector index for connections in a single segment.diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.javaindex 78b5e57e1a5..29c0b4aaea5 100644--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java@@ -55,11 +55,7 @@ import org.apache.lucene.document.DocumentStoredFieldVisitor;import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;import org.apache.lucene.index.PointValues.IntersectVisitor;import org.apache.lucene.index.PointValues.Relation;-import org.apache.lucene.search.DocIdSetIterator;-import org.apache.lucene.search.FieldExistsQuery;-import org.apache.lucene.search.LeafFieldComparator;-import org.apache.lucene.search.Sort;-import org.apache.lucene.search.SortField;+import org.apache.lucene.search.*;import org.apache.lucene.store.AlreadyClosedException;import org.apache.lucene.store.Directory;import org.apache.lucene.store.FSDirectory;@@ -2593,8 +2589,33 @@ public final class CheckIndex implements Closeable {status.totalKnnVectorFields++;int docCount = 0;+ final Bits bits = reader.getLiveDocs();+ int everyNdoc = Math.max(values.size() / 64, 1);while (values.nextDoc() != NO_MORE_DOCS) {- int valueLength = values.vectorValue().length;+ float[] vectorValue = values.vectorValue();+ // search the first maxNumSearches vectors to exercise the graph+ if (values.docID() % everyNdoc == 0) {+ TopDocs docs =+ reader+ .getVectorReader()+ .search(fieldInfo.name, vectorValue, 10, bits, Integer.MAX_VALUE);+ if (docs.scoreDocs.length == 0) {+ throw new CheckIndexException(+ "Field \"" + fieldInfo.name + "\" failed to search k nearest neighbors");+ }+ if (bits != null) {+ for (ScoreDoc doc : docs.scoreDocs) {+ if (bits.get(doc.doc) == false) {+ throw new CheckIndexException(+ "Searching Field \""+ + fieldInfo.name+ + "\" matched deleted doc="+ + doc.doc);+ }+ }+ }+ }+ int valueLength = vectorValue.length;if (valueLength != dimension) {throw new CheckIndexException("Field \""