Apache Lucene open-source search software
Compare live vector counts when picking the HNSW merge base graph (#16739)
Lantao Jin0ff1bf1534bc
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 4ad3b46d420..3e84d045980 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -261,6 +261,11 @@ OptimizationsBug Fixes---------------------+* GITHUB#16739: IncrementalHnswGraphMerger now picks the base graph for a merge by comparing live+ vector counts. It previously compared a candidate's live count against the current base's total+ node count, so a graph with deletions could be kept as the base over a larger deletion-free graph,+ depending on the order in which readers were added. (Lantao Jin)+* GITHUB#16657: Fix hashCode/equals contract violation in LRUQueryCache's QueryCacheKey and account acached query's RAM once per distinct query. (Sagar Upadhyaya)diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/IncrementalHnswGraphMerger.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/IncrementalHnswGraphMerger.javaindex 32016e74676..0dfd72dcd0a 100644--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/IncrementalHnswGraphMerger.java+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/IncrementalHnswGraphMerger.java@@ -64,9 +64,16 @@ public class IncrementalHnswGraphMerger implements HnswGraphMerger {*/static final int DELETE_PCT_THRESHOLD = 40;- /** Represents a vector reader that contains graph info. */+ /**+ * Represents a vector reader that contains graph info.+ *+ * @param reader the vectors reader that provides the graph+ * @param initDocMap maps the reader's doc IDs to doc IDs in the merged segment+ * @param graphSize total number of nodes in the graph, including deleted ones+ * @param liveVectorCount number of nodes in the graph whose documents are live+ */protected record GraphReader(- KnnVectorsReader reader, MergeState.DocMap initDocMap, int graphSize) {}+ KnnVectorsReader reader, MergeState.DocMap initDocMap, int graphSize, int liveVectorCount) {}/*** @param fieldInfo FieldInfo for the field being merged@@ -121,12 +128,13 @@ public class IncrementalHnswGraphMerger implements HnswGraphMerger {int candidateVectorCount = countLiveVectors(liveDocs, knnVectorValues);int graphSize = graph.size();- GraphReader graphReader = new GraphReader(reader, docMap, graphSize);+ GraphReader graphReader = new GraphReader(reader, docMap, graphSize, candidateVectorCount);int deletePct = ((graphSize - candidateVectorCount) * 100) / graphSize;if (deletePct <= DELETE_PCT_THRESHOLD- && (largestGraphReader == null || candidateVectorCount > largestGraphReader.graphSize)) {+ && (largestGraphReader == null+ || candidateVectorCount > largestGraphReader.liveVectorCount)) {largestGraphReader = graphReader;}diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/TestIncrementalHnswGraphMergerBaseSelection.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/TestIncrementalHnswGraphMergerBaseSelection.javanew file mode 100644index 00000000000..da83b1dc855--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/TestIncrementalHnswGraphMergerBaseSelection.java@@ -0,0 +1,198 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.util.hnsw;++import java.io.IOException;+import java.util.ArrayList;+import java.util.Collections;+import java.util.List;+import org.apache.lucene.codecs.KnnVectorsReader;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.index.CodecReader;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.NoMergePolicy;+import org.apache.lucene.index.Term;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;++/**+ * Tests which source graph {@link IncrementalHnswGraphMerger} picks as the base graph when some+ * candidates carry deletions.+ */+public class TestIncrementalHnswGraphMergerBaseSelection extends LuceneTestCase {++ private static final int DIM = 16;+ private static final int M = 16;+ private static final int BEAM_WIDTH = 100;+ private static final String VECTOR_FIELD = "v";+ private static final String ID_FIELD = "id";+ private static final int DELETE_PCT_THRESHOLD = IncrementalHnswGraphMerger.DELETE_PCT_THRESHOLD;++ // A deletes half of what the threshold allows, so it stays eligible as a base+ private static final int A_DOCS = 1000;+ private static final int A_DELETES = A_DOCS * DELETE_PCT_THRESHOLD / 200;+ // B has no deletions and falls strictly between A's live count and A's total node count+ private static final int B_DOCS = A_DOCS - A_DELETES / 2;++ /**+ * Segment A carries deletions under the threshold; segment B has none. B holds more live vectors+ * than A but fewer nodes, so it must be chosen as the base regardless of the order in which the+ * readers are added.+ */+ public void testBaseGraphChosenByLiveCount() throws IOException {+ assertTrue(A_DELETES > 0);+ assertTrue(A_DOCS - A_DELETES < B_DOCS && B_DOCS < A_DOCS);+ try (Directory dir = newDirectory()) {+ buildIndex(dir, new int[] {A_DOCS, B_DOCS}, new int[] {A_DELETES, 0});+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ List<CodecReader> segments = segments(reader);+ assertEquals(2, segments.size());+ CodecReader a = segments.get(0);+ CodecReader b = segments.get(1);+ assertEquals(A_DOCS, a.maxDoc());+ assertEquals(A_DOCS - A_DELETES, a.numDocs());+ assertEquals(B_DOCS, b.numDocs());++ for (boolean aFirst : new boolean[] {true, false}) {+ for (boolean concurrent : new boolean[] {false, true}) {+ IncrementalHnswGraphMerger merger =+ addReaders(aFirst ? List.of(a, b) : List.of(b, a), concurrent);+ assertNotNull(merger.largestGraphReader);+ assertSame(+ "aFirst=" + aFirst + " concurrent=" + concurrent,+ b.getVectorReader(),+ merger.largestGraphReader.reader());+ }+ }+ }+ }+ }++ /**+ * Segments of random sizes and deletion counts, added in random order. The base must be a graph+ * within the deletion threshold that has the most live vectors, or none if no graph is within the+ * threshold.+ */+ public void testBaseGraphHasMostLiveVectorsAmongEligible() throws IOException {+ int numSegments = TestUtil.nextInt(random(), 2, 6);+ int[] docs = new int[numSegments];+ int[] deletes = new int[numSegments];+ for (int s = 0; s < numSegments; s++) {+ docs[s] = TestUtil.nextInt(random(), 10, 200);+ // half of the segments stay within the threshold; at least one doc stays live, otherwise the+ // segment is dropped+ int maxDeletes = random().nextBoolean() ? docs[s] * DELETE_PCT_THRESHOLD / 100 : docs[s] - 1;+ deletes[s] = TestUtil.nextInt(random(), 0, maxDeletes);+ }+ try (Directory dir = newDirectory()) {+ buildIndex(dir, docs, deletes);+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ List<CodecReader> segments = new ArrayList<>(segments(reader));+ assertEquals(numSegments, segments.size());+ int expectedLive = -1;+ for (CodecReader segment : segments) {+ int total = segment.maxDoc();+ int live = segment.numDocs();+ if ((total - live) * 100 / total <= DELETE_PCT_THRESHOLD) {+ expectedLive = Math.max(expectedLive, live);+ }+ }++ for (int iter = 0; iter < 5; iter++) {+ Collections.shuffle(segments, random());+ for (boolean concurrent : new boolean[] {false, true}) {+ IncrementalHnswGraphMerger merger = addReaders(segments, concurrent);+ if (expectedLive == -1) {+ assertNull(merger.largestGraphReader);+ } else {+ assertNotNull(merger.largestGraphReader);+ CodecReader base = owner(segments, merger.largestGraphReader.reader());+ assertEquals("concurrent=" + concurrent, expectedLive, base.numDocs());+ }+ }+ }+ }+ }+ }++ private static IncrementalHnswGraphMerger addReaders(+ List<CodecReader> segments, boolean concurrent) throws IOException {+ FieldInfo fieldInfo = segments.get(0).getFieldInfos().fieldInfo(VECTOR_FIELD);+ IncrementalHnswGraphMerger merger =+ concurrent+ ? new ConcurrentHnswMerger(fieldInfo, null, M, BEAM_WIDTH, null, 2)+ : new IncrementalHnswGraphMerger(fieldInfo, null, M, BEAM_WIDTH);+ for (CodecReader segment : segments) {+ merger.addReader(segment.getVectorReader(), doc -> doc, segment.getLiveDocs());+ }+ return merger;+ }++ private static CodecReader owner(List<CodecReader> segments, KnnVectorsReader vectorReader) {+ for (CodecReader segment : segments) {+ if (segment.getVectorReader() == vectorReader) {+ return segment;+ }+ }+ throw new AssertionError("base graph reader does not belong to any segment");+ }++ private static List<CodecReader> segments(DirectoryReader reader) {+ List<CodecReader> segments = new ArrayList<>();+ for (LeafReaderContext ctx : reader.leaves()) {+ segments.add((CodecReader) ctx.reader());+ }+ return segments;+ }++ /** Flushes one segment per entry of {@code docs}, then deletes the first docs of each segment. */+ private void buildIndex(Directory dir, int[] docs, int[] deletes) throws IOException {+ IndexWriterConfig cfg = new IndexWriterConfig();+ cfg.setCodec(TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswVectorsFormat(M, BEAM_WIDTH, 0)));+ cfg.setMergePolicy(NoMergePolicy.INSTANCE);+ try (IndexWriter w = new IndexWriter(dir, cfg)) {+ for (int s = 0; s < docs.length; s++) {+ for (int i = 0; i < docs[s]; i++) {+ Document doc = new Document();+ doc.add(new StringField(ID_FIELD, s + "_" + i, Field.Store.NO));+ float[] v = new float[DIM];+ for (int j = 0; j < DIM; j++) {+ v[j] = random().nextFloat();+ }+ doc.add(new KnnFloatVectorField(VECTOR_FIELD, v));+ w.addDocument(doc);+ }+ w.flush();+ }+ for (int s = 0; s < docs.length; s++) {+ for (int i = 0; i < deletes[s]; i++) {+ w.deleteDocuments(new Term(ID_FIELD, s + "_" + i));+ }+ }+ w.commit();+ }+ }+}