Skip to content

Apache Lucene open-source search software

Add some basic HNSW graph checks to CheckIndex (#13984)

Ben Chaplin11eb2c86979e
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index d957af01d0a..b3a5e4dc5d1 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -26,8 +26,11 @@ import java.io.PrintStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.NumberFormat;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.Deque;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -52,12 +55,14 @@ import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
+import org.apache.lucene.codecs.hnsw.HnswGraphProvider;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.internal.hppc.IntIntHashMap;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldExistsQuery;
import org.apache.lucene.search.KnnCollector;
@@ -74,6 +79,7 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.Lock;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.ArrayUtil.ByteArrayComparator;
+import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -91,6 +97,7 @@ import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.ByteRunAutomaton;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.util.hnsw.HnswGraph;
/**
* Basic tool and API to check the health of an index and write a new segments file that removes
@@ -249,6 +256,9 @@ public final class CheckIndex implements Closeable {
/** Status of vectors */
public VectorValuesStatus vectorValuesStatus;
+ /** Status of HNSW graph */
+ public HnswGraphsStatus hnswGraphsStatus;
+
/** Status of soft deletes */
public SoftDeletesStatus softDeletesStatus;
@@ -406,6 +416,32 @@ public final class CheckIndex implements Closeable {
public Throwable error;
}
+ /** Status from testing a single HNSW graph */
+ public static final class HnswGraphStatus {
+
+ HnswGraphStatus() {}
+
+ /** Number of nodes at each level */
+ public List<Integer> numNodesAtLevel;
+
+ /** Connectedness at each level represented as a fraction */
+ public List<String> connectednessAtLevel;
+ }
+
+ /** Status from testing all HNSW graphs */
+ public static final class HnswGraphsStatus {
+
+ HnswGraphsStatus() {
+ this.hnswGraphsStatusByField = new HashMap<>();
+ }
+
+ /** Status of the HNSW graph keyed with field name */
+ public Map<String, HnswGraphStatus> hnswGraphsStatusByField;
+
+ /** Exception thrown during term index test (null on success) */
+ public Throwable error;
+ }
+
/** Status from testing index sort */
public static final class IndexSortStatus {
IndexSortStatus() {}
@@ -1085,6 +1121,9 @@ public final class CheckIndex implements Closeable {
// Test FloatVectorValues and ByteVectorValues
segInfoStat.vectorValuesStatus = testVectors(reader, infoStream, failFast);
+ // Test HNSW graph
+ segInfoStat.hnswGraphsStatus = testHnswGraphs(reader, infoStream, failFast);
+
// Test Index Sort
if (indexSort != null) {
segInfoStat.indexSortStatus = testSort(reader, indexSort, infoStream, failFast);
@@ -2746,6 +2785,196 @@ public final class CheckIndex implements Closeable {
return status;
}
+ /** Test the HNSW graph. */
+ public static Status.HnswGraphsStatus testHnswGraphs(
+ CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {
+ if (infoStream != null) {
+ infoStream.print(" test: hnsw graphs.........");
+ }
+ long startNS = System.nanoTime();
+ Status.HnswGraphsStatus status = new Status.HnswGraphsStatus();
+ KnnVectorsReader vectorsReader = reader.getVectorReader();
+ FieldInfos fieldInfos = reader.getFieldInfos();
+
+ try {
+ if (fieldInfos.hasVectorValues()) {
+ for (FieldInfo fieldInfo : fieldInfos) {
+ if (fieldInfo.hasVectorValues()) {
+ KnnVectorsReader fieldReader = getFieldReaderForName(vectorsReader, fieldInfo.name);
+ if (fieldReader instanceof HnswGraphProvider graphProvider) {
+ HnswGraph hnswGraph = graphProvider.getGraph(fieldInfo.name);
+ testHnswGraph(hnswGraph, fieldInfo.name, status);
+ }
+ }
+ }
+ }
+ msg(
+ infoStream,
+ String.format(
+ Locale.ROOT,
+ "OK [%d fields] [took %.3f sec]",
+ status.hnswGraphsStatusByField.size(),
+ nsToSec(System.nanoTime() - startNS)));
+ printHnswInfo(infoStream, status.hnswGraphsStatusByField);
+ } catch (Exception e) {
+ if (failFast) {
+ throw IOUtils.rethrowAlways(e);
+ }
+ msg(infoStream, "ERROR: " + e);
+ status.error = e;
+ if (infoStream != null) {
+ e.printStackTrace(infoStream);
+ }
+ }
+
+ return status;
+ }
+
+ private static KnnVectorsReader getFieldReaderForName(
+ KnnVectorsReader vectorsReader, String fieldName) {
+ if (vectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) {
+ return fieldsReader.getFieldReader(fieldName);
+ } else {
+ return vectorsReader;
+ }
+ }
+
+ private static void printHnswInfo(
+ PrintStream infoStream, Map<String, CheckIndex.Status.HnswGraphStatus> fieldsStatus) {
+ for (Map.Entry<String, CheckIndex.Status.HnswGraphStatus> entry : fieldsStatus.entrySet()) {
+ String fieldName = entry.getKey();
+ CheckIndex.Status.HnswGraphStatus status = entry.getValue();
+ msg(infoStream, " hnsw field name: " + fieldName);
+
+ int numLevels = Math.min(status.numNodesAtLevel.size(), status.connectednessAtLevel.size());
+ for (int level = numLevels - 1; level >= 0; level--) {
+ int numNodes = status.numNodesAtLevel.get(level);
+ String connectedness = status.connectednessAtLevel.get(level);
+ msg(
+ infoStream,
+ String.format(
+ Locale.ROOT,
+ " level %d: %d nodes, %s connected",
+ level,
+ numNodes,
+ connectedness));
+ }
+ }
+ }
+
+ private static void testHnswGraph(
+ HnswGraph hnswGraph, String fieldName, Status.HnswGraphsStatus status)
+ throws IOException, CheckIndexException {
+ if (hnswGraph != null) {
+ status.hnswGraphsStatusByField.put(fieldName, new Status.HnswGraphStatus());
+ final int numLevels = hnswGraph.numLevels();
+ status.hnswGraphsStatusByField.get(fieldName).numNodesAtLevel =
+ new ArrayList<>(Collections.nCopies(numLevels, null));
+ status.hnswGraphsStatusByField.get(fieldName).connectednessAtLevel =
+ new ArrayList<>(Collections.nCopies(numLevels, null));
+ // Perform checks on each level of the HNSW graph
+ for (int level = numLevels - 1; level >= 0; level--) {
+ // Collect BitSet of all nodes on this level
+ BitSet nodesOnThisLevel = new FixedBitSet(hnswGraph.size());
+ HnswGraph.NodesIterator nodesIterator = hnswGraph.getNodesOnLevel(level);
+ while (nodesIterator.hasNext()) {
+ nodesOnThisLevel.set(nodesIterator.nextInt());
+ }
+
+ nodesIterator = hnswGraph.getNodesOnLevel(level);
+ // Perform checks on each node on the level
+ while (nodesIterator.hasNext()) {
+ int node = nodesIterator.nextInt();
+ if (node < 0 || node > hnswGraph.size() - 1) {
+ throw new CheckIndexException(
+ "Field \""
+ + fieldName
+ + "\" has node: "
+ + node
+ + " not in the expected range [0, "
+ + (hnswGraph.size() - 1)
+ + "]");
+ }
+
+ // Perform checks on the node's neighbors
+ hnswGraph.seek(level, node);
+ int nbr, lastNeighbor = -1, firstNeighbor = -1;
+ while ((nbr = hnswGraph.nextNeighbor()) != NO_MORE_DOCS) {
+ if (!nodesOnThisLevel.get(nbr)) {
+ throw new CheckIndexException(
+ "Field \""
+ + fieldName
+ + "\" has node: "
+ + node
+ + " with a neighbor "
+ + nbr
+ + " which is not on its level ("
+ + level
+ + ")");
+ }
+ if (firstNeighbor == -1) {
+ firstNeighbor = nbr;
+ }
+ if (nbr < lastNeighbor) {
+ throw new CheckIndexException(
+ "Field \""
+ + fieldName
+ + "\" has neighbors out of order for node "
+ + node
+ + ": "
+ + nbr
+ + "<"
+ + lastNeighbor
+ + " 1st="
+ + firstNeighbor);
+ } else if (nbr == lastNeighbor) {
+ throw new CheckIndexException(
+ "Field \""
+ + fieldName
+ + "\" has repeated neighbors of node "
+ + node
+ + " with value "
+ + nbr);
+ }
+ lastNeighbor = nbr;
+ }
+ }
+ int numNodesOnLayer = nodesIterator.size();
+ status.hnswGraphsStatusByField.get(fieldName).numNodesAtLevel.set(level, numNodesOnLayer);
+
+ // Evaluate connectedness at this level by measuring the number of nodes reachable from the
+ // entry point
+ IntIntHashMap connectedNodes = getConnectedNodesOnLevel(hnswGraph, numNodesOnLayer, level);
+ status
+ .hnswGraphsStatusByField
+ .get(fieldName)
+ .connectednessAtLevel
+ .set(level, connectedNodes.size() + "/" + numNodesOnLayer);
+ }
+ }
+ }
+
+ private static IntIntHashMap getConnectedNodesOnLevel(
+ HnswGraph hnswGraph, int numNodesOnLayer, int level) throws IOException {
+ IntIntHashMap connectedNodes = new IntIntHashMap(numNodesOnLayer);
+ int entryPoint = hnswGraph.entryNode();
+ Deque<Integer> stack = new ArrayDeque<>();
+ stack.push(entryPoint);
+ while (!stack.isEmpty()) {
+ int node = stack.pop();
+ if (connectedNodes.containsKey(node)) {
+ continue;
+ }
+ connectedNodes.put(node, 1);
+ hnswGraph.seek(level, node);
+ int friendOrd;
+ while ((friendOrd = hnswGraph.nextNeighbor()) != NO_MORE_DOCS) {
+ stack.push(friendOrd);
+ }
+ }
+ return connectedNodes;
+ }
+
private static boolean vectorsReaderSupportsSearch(CodecReader codecReader, String fieldName) {
KnnVectorsReader vectorsReader = codecReader.getVectorReader();
if (vectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader perFieldReader) {