Apache Lucene open-source search software
Add some basic HNSW graph checks to CheckIndex (#13984)
Ben Chaplin11eb2c86979e
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.javaindex d957af01d0a..b3a5e4dc5d1 100644--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java@@ -26,8 +26,11 @@ import java.io.PrintStream;import java.nio.file.Path;import java.nio.file.Paths;import java.text.NumberFormat;+import java.util.ArrayDeque;import java.util.ArrayList;import java.util.Arrays;+import java.util.Collections;+import java.util.Deque;import java.util.HashMap;import java.util.Iterator;import java.util.List;@@ -52,12 +55,14 @@ import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.StoredFieldsReader;import org.apache.lucene.codecs.TermVectorsReader;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;+import org.apache.lucene.codecs.hnsw.HnswGraphProvider;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.DocumentStoredFieldVisitor;import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;import org.apache.lucene.index.PointValues.IntersectVisitor;import org.apache.lucene.index.PointValues.Relation;+import org.apache.lucene.internal.hppc.IntIntHashMap;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.FieldExistsQuery;import org.apache.lucene.search.KnnCollector;@@ -74,6 +79,7 @@ import org.apache.lucene.store.IOContext;import org.apache.lucene.store.Lock;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.ArrayUtil.ByteArrayComparator;+import org.apache.lucene.util.BitSet;import org.apache.lucene.util.Bits;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.BytesRefBuilder;@@ -91,6 +97,7 @@ import org.apache.lucene.util.automaton.Automaton;import org.apache.lucene.util.automaton.ByteRunAutomaton;import org.apache.lucene.util.automaton.CompiledAutomaton;import org.apache.lucene.util.automaton.Operations;+import org.apache.lucene.util.hnsw.HnswGraph;/*** Basic tool and API to check the health of an index and write a new segments file that removes@@ -249,6 +256,9 @@ public final class CheckIndex implements Closeable {/** Status of vectors */public VectorValuesStatus vectorValuesStatus;+ /** Status of HNSW graph */+ public HnswGraphsStatus hnswGraphsStatus;+/** Status of soft deletes */public SoftDeletesStatus softDeletesStatus;@@ -406,6 +416,32 @@ public final class CheckIndex implements Closeable {public Throwable error;}+ /** Status from testing a single HNSW graph */+ public static final class HnswGraphStatus {++ HnswGraphStatus() {}++ /** Number of nodes at each level */+ public List<Integer> numNodesAtLevel;++ /** Connectedness at each level represented as a fraction */+ public List<String> connectednessAtLevel;+ }++ /** Status from testing all HNSW graphs */+ public static final class HnswGraphsStatus {++ HnswGraphsStatus() {+ this.hnswGraphsStatusByField = new HashMap<>();+ }++ /** Status of the HNSW graph keyed with field name */+ public Map<String, HnswGraphStatus> hnswGraphsStatusByField;++ /** Exception thrown during term index test (null on success) */+ public Throwable error;+ }+/** Status from testing index sort */public static final class IndexSortStatus {IndexSortStatus() {}@@ -1085,6 +1121,9 @@ public final class CheckIndex implements Closeable {// Test FloatVectorValues and ByteVectorValuessegInfoStat.vectorValuesStatus = testVectors(reader, infoStream, failFast);+ // Test HNSW graph+ segInfoStat.hnswGraphsStatus = testHnswGraphs(reader, infoStream, failFast);+// Test Index Sortif (indexSort != null) {segInfoStat.indexSortStatus = testSort(reader, indexSort, infoStream, failFast);@@ -2746,6 +2785,196 @@ public final class CheckIndex implements Closeable {return status;}+ /** Test the HNSW graph. */+ public static Status.HnswGraphsStatus testHnswGraphs(+ CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {+ if (infoStream != null) {+ infoStream.print(" test: hnsw graphs.........");+ }+ long startNS = System.nanoTime();+ Status.HnswGraphsStatus status = new Status.HnswGraphsStatus();+ KnnVectorsReader vectorsReader = reader.getVectorReader();+ FieldInfos fieldInfos = reader.getFieldInfos();++ try {+ if (fieldInfos.hasVectorValues()) {+ for (FieldInfo fieldInfo : fieldInfos) {+ if (fieldInfo.hasVectorValues()) {+ KnnVectorsReader fieldReader = getFieldReaderForName(vectorsReader, fieldInfo.name);+ if (fieldReader instanceof HnswGraphProvider graphProvider) {+ HnswGraph hnswGraph = graphProvider.getGraph(fieldInfo.name);+ testHnswGraph(hnswGraph, fieldInfo.name, status);+ }+ }+ }+ }+ msg(+ infoStream,+ String.format(+ Locale.ROOT,+ "OK [%d fields] [took %.3f sec]",+ status.hnswGraphsStatusByField.size(),+ nsToSec(System.nanoTime() - startNS)));+ printHnswInfo(infoStream, status.hnswGraphsStatusByField);+ } catch (Exception e) {+ if (failFast) {+ throw IOUtils.rethrowAlways(e);+ }+ msg(infoStream, "ERROR: " + e);+ status.error = e;+ if (infoStream != null) {+ e.printStackTrace(infoStream);+ }+ }++ return status;+ }++ private static KnnVectorsReader getFieldReaderForName(+ KnnVectorsReader vectorsReader, String fieldName) {+ if (vectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) {+ return fieldsReader.getFieldReader(fieldName);+ } else {+ return vectorsReader;+ }+ }++ private static void printHnswInfo(+ PrintStream infoStream, Map<String, CheckIndex.Status.HnswGraphStatus> fieldsStatus) {+ for (Map.Entry<String, CheckIndex.Status.HnswGraphStatus> entry : fieldsStatus.entrySet()) {+ String fieldName = entry.getKey();+ CheckIndex.Status.HnswGraphStatus status = entry.getValue();+ msg(infoStream, " hnsw field name: " + fieldName);++ int numLevels = Math.min(status.numNodesAtLevel.size(), status.connectednessAtLevel.size());+ for (int level = numLevels - 1; level >= 0; level--) {+ int numNodes = status.numNodesAtLevel.get(level);+ String connectedness = status.connectednessAtLevel.get(level);+ msg(+ infoStream,+ String.format(+ Locale.ROOT,+ " level %d: %d nodes, %s connected",+ level,+ numNodes,+ connectedness));+ }+ }+ }++ private static void testHnswGraph(+ HnswGraph hnswGraph, String fieldName, Status.HnswGraphsStatus status)+ throws IOException, CheckIndexException {+ if (hnswGraph != null) {+ status.hnswGraphsStatusByField.put(fieldName, new Status.HnswGraphStatus());+ final int numLevels = hnswGraph.numLevels();+ status.hnswGraphsStatusByField.get(fieldName).numNodesAtLevel =+ new ArrayList<>(Collections.nCopies(numLevels, null));+ status.hnswGraphsStatusByField.get(fieldName).connectednessAtLevel =+ new ArrayList<>(Collections.nCopies(numLevels, null));+ // Perform checks on each level of the HNSW graph+ for (int level = numLevels - 1; level >= 0; level--) {+ // Collect BitSet of all nodes on this level+ BitSet nodesOnThisLevel = new FixedBitSet(hnswGraph.size());+ HnswGraph.NodesIterator nodesIterator = hnswGraph.getNodesOnLevel(level);+ while (nodesIterator.hasNext()) {+ nodesOnThisLevel.set(nodesIterator.nextInt());+ }++ nodesIterator = hnswGraph.getNodesOnLevel(level);+ // Perform checks on each node on the level+ while (nodesIterator.hasNext()) {+ int node = nodesIterator.nextInt();+ if (node < 0 || node > hnswGraph.size() - 1) {+ throw new CheckIndexException(+ "Field \""+ + fieldName+ + "\" has node: "+ + node+ + " not in the expected range [0, "+ + (hnswGraph.size() - 1)+ + "]");+ }++ // Perform checks on the node's neighbors+ hnswGraph.seek(level, node);+ int nbr, lastNeighbor = -1, firstNeighbor = -1;+ while ((nbr = hnswGraph.nextNeighbor()) != NO_MORE_DOCS) {+ if (!nodesOnThisLevel.get(nbr)) {+ throw new CheckIndexException(+ "Field \""+ + fieldName+ + "\" has node: "+ + node+ + " with a neighbor "+ + nbr+ + " which is not on its level ("+ + level+ + ")");+ }+ if (firstNeighbor == -1) {+ firstNeighbor = nbr;+ }+ if (nbr < lastNeighbor) {+ throw new CheckIndexException(+ "Field \""+ + fieldName+ + "\" has neighbors out of order for node "+ + node+ + ": "+ + nbr+ + "<"+ + lastNeighbor+ + " 1st="+ + firstNeighbor);+ } else if (nbr == lastNeighbor) {+ throw new CheckIndexException(+ "Field \""+ + fieldName+ + "\" has repeated neighbors of node "+ + node+ + " with value "+ + nbr);+ }+ lastNeighbor = nbr;+ }+ }+ int numNodesOnLayer = nodesIterator.size();+ status.hnswGraphsStatusByField.get(fieldName).numNodesAtLevel.set(level, numNodesOnLayer);++ // Evaluate connectedness at this level by measuring the number of nodes reachable from the+ // entry point+ IntIntHashMap connectedNodes = getConnectedNodesOnLevel(hnswGraph, numNodesOnLayer, level);+ status+ .hnswGraphsStatusByField+ .get(fieldName)+ .connectednessAtLevel+ .set(level, connectedNodes.size() + "/" + numNodesOnLayer);+ }+ }+ }++ private static IntIntHashMap getConnectedNodesOnLevel(+ HnswGraph hnswGraph, int numNodesOnLayer, int level) throws IOException {+ IntIntHashMap connectedNodes = new IntIntHashMap(numNodesOnLayer);+ int entryPoint = hnswGraph.entryNode();+ Deque<Integer> stack = new ArrayDeque<>();+ stack.push(entryPoint);+ while (!stack.isEmpty()) {+ int node = stack.pop();+ if (connectedNodes.containsKey(node)) {+ continue;+ }+ connectedNodes.put(node, 1);+ hnswGraph.seek(level, node);+ int friendOrd;+ while ((friendOrd = hnswGraph.nextNeighbor()) != NO_MORE_DOCS) {+ stack.push(friendOrd);+ }+ }+ return connectedNodes;+ }+private static boolean vectorsReaderSupportsSearch(CodecReader codecReader, String fieldName) {KnnVectorsReader vectorsReader = codecReader.getVectorReader();if (vectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader perFieldReader) {