Apache Lucene open-source search software
Bulk score hnsw diversity check (#15607)
Benjamin Trentc646d438e321
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex bb4e1ce8f34..5e9ac863126 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -339,6 +339,9 @@ Optimizations* GITHUB#15632: Use a coarser-grained competitive iterator with lower construction costs fornumeric sorts against fields with DocValuesSkippers. (Alan Woodward)+* GITHUB#15607: Utilize bulk scoring for diversity checking when building HNSW vector indices. This results+ in some performance improvements during indexing and segment merges. (Ben Trent)+Bug Fixes---------------------* GITHUB#14161: PointInSetQuery's constructor now throws IllegalArgumentExceptiondiff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.javaindex 2e55e3e0664..c5fba5c88d4 100644--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java@@ -64,9 +64,13 @@ public class HnswGraphBuilder implements HnswBuilder {@SuppressWarnings("NonFinalStaticField")public static long randSeed = DEFAULT_RAND_SEED;+ private static final int MAX_BULK_SCORE_NODES = 8;+protected final int M; // max number of connections on upper layersprivate final double ml;+ private final int[] bulkScoreNodes; // for bulk scoring+ private final float[] bulkScores; // for bulk scoringprivate final SplittableRandom random;protected final UpdateableRandomVectorScorer scorer;protected final HnswGraphSearcher graphSearcher;@@ -156,6 +160,10 @@ public class HnswGraphBuilder implements HnswBuilder {this.hnsw = hnsw;this.hnswLock = hnswLock;this.graphSearcher = graphSearcher;+ // pick a number that keeps us from scoring TOO much for diversity checking+ // but enough to take advantage of bulk scoring+ this.bulkScoreNodes = new int[MAX_BULK_SCORE_NODES];+ this.bulkScores = new float[MAX_BULK_SCORE_NODES];entryCandidates = new GraphBuilderKnnCollector(1);beamCandidates = new GraphBuilderKnnCollector(beamWidth);beamCandidates0 = new GraphBuilderKnnCollector(Math.min(beamWidth / 2, M * 3));@@ -470,9 +478,11 @@ public class HnswGraphBuilder implements HnswBuilder {*/private boolean diversityCheck(float score, NeighborArray neighbors, RandomVectorScorer scorer)throws IOException {- for (int i = 0; i < neighbors.size(); i++) {- float neighborSimilarity = scorer.score(neighbors.nodes()[i]);- if (neighborSimilarity >= score) {+ final int bulkScoreChunk = Math.min((neighbors.size() + 1) / 2, bulkScoreNodes.length);+ for (int scored = 0; scored < neighbors.size(); scored += bulkScoreChunk) {+ int chunkSize = Math.min(bulkScoreChunk, neighbors.size() - scored);+ System.arraycopy(neighbors.nodes(), scored, bulkScoreNodes, 0, chunkSize);+ if (scorer.bulkScore(bulkScoreNodes, bulkScores, chunkSize) >= score) {return false;}}