Skip to content

Apache Lucene open-source search software

LUCENE-10040: Add test for vector search with skewed deletions (#527)

Julie Tibshirani5d39bca87a44
diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/TestHnswGraph.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/TestHnswGraph.java
index 16a3e60b01d..69b478ca249 100644
--- a/lucene/core/src/test/org/apache/lucene/util/hnsw/TestHnswGraph.java
+++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/TestHnswGraph.java
@@ -196,6 +196,41 @@ public class TestHnswGraph extends LuceneTestCase {
assertTrue("sum(result docs)=" + sum, sum < 75);
}
+ public void testSearchWithSkewedAcceptOrds() throws IOException {
+ int nDoc = 1000;
+ CircularVectorValues vectors = new CircularVectorValues(nDoc);
+ HnswGraphBuilder builder =
+ new HnswGraphBuilder(
+ vectors, VectorSimilarityFunction.DOT_PRODUCT, 16, 100, random().nextInt());
+ HnswGraph hnsw = builder.build(vectors);
+
+ // Skip over half of the documents that are closest to the query vector
+ FixedBitSet acceptOrds = new FixedBitSet(nDoc);
+ for (int i = 500; i < nDoc; i++) {
+ acceptOrds.set(i);
+ }
+ NeighborQueue nn =
+ HnswGraph.search(
+ new float[] {1, 0},
+ 10,
+ 10,
+ vectors.randomAccess(),
+ VectorSimilarityFunction.DOT_PRODUCT,
+ hnsw,
+ acceptOrds,
+ new SplittableRandom(random().nextLong()));
+ int[] nodes = nn.nodes();
+ assertTrue("Number of found results is not equal to [10].", nodes.length == 10);
+ int sum = 0;
+ for (int node : nodes) {
+ assertTrue("the results include a deleted document: " + node, acceptOrds.get(node));
+ sum += node;
+ }
+ // We still expect to get reasonable recall. The lowest non-skipped docIds
+ // are closest to the query vector: sum(500,509) = 5045
+ assertTrue("sum(result docs)=" + sum, sum < 5100);
+ }
+
public void testBoundsCheckerMax() {
BoundsChecker max = BoundsChecker.create(false);
float f = random().nextFloat() - 0.5f;