Skip to content

Apache Lucene open-source search software

Build the AcceptDocs bit set from the raw filter iterator (#16588)

John Maurice1a65ceae18ea
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index e5cd853bcfa..bf53046f9b2 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -497,6 +497,9 @@ Optimizations
* GITHUB#16631: Vectorize the uint4 nibble unpack used by Lucene104ScalarQuantizedVectorsFormat
via VectorUtilSupport, vectors are now unpacked with the Panama Vector API. (Jakub Slowinski)
+* GITHUB#16588: AcceptDocs builds its bit set from the raw filter iterator so that
+ DocIdSetIterator#intoBitSet is used on segments with deletions. (John Maurice)
+
Bug Fixes
---------------------
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/FilteredKnnVectorQueryBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/FilteredKnnVectorQueryBenchmark.java
new file mode 100644
index 00000000000..7cad02a4132
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/FilteredKnnVectorQueryBenchmark.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.KnnFloatVectorQuery;
+import org.apache.lucene.search.LRUQueryCache;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.ScorerSupplier;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.util.IOUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/**
+ * Benchmarks end-to-end latency of {@link KnnFloatVectorQuery} whose pre-filter is a cached term
+ * query, over a single segment.
+ *
+ * <p>The filter is warmed into {@link LRUQueryCache} before measurement, so every measured query
+ * reuses the cached bit set, which is how a hot pre-filter reaches the kNN query in practice. The
+ * corpus shape decides whether {@link org.apache.lucene.search.AcceptDocs} can load that filter in
+ * bulk.
+ *
+ * <p>Parameters explore:
+ *
+ * <ul>
+ * <li>{@code numDocs} — total number of documents in the segment.
+ * <li>{@code dim} — vector dimension.
+ * <li>{@code k} — number of results requested.
+ * <li>{@code filterSelectivity} — fraction of docs matched by the pre-filter.
+ * <li>{@code vectorlessFraction} — fraction of docs indexed without a vector.
+ * <li>{@code deletedFraction} — fraction of docs deleted after indexing, which keeps the
+ * segment's liveDocs non-null.
+ * </ul>
+ *
+ * Run with:
+ *
+ * <pre>
+ * ./gradlew -p lucene/benchmark-jmh assemble
+ * java -jar lucene/benchmark-jmh/build/benchmarks/lucene-benchmark-jmh-*.jar FilteredKnnVectorQueryBenchmark
+ * </pre>
+ */
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 5, time = 2)
+@Measurement(iterations = 5, time = 5)
+@Fork(
+ value = 1,
+ jvmArgsAppend = {"-Xmx4g", "-Xms4g", "-XX:+AlwaysPreTouch"})
+public class FilteredKnnVectorQueryBenchmark {
+
+ private static final String FIELD = "vec";
+ private static final String FILTER_FIELD = "tag";
+ private static final String ID_FIELD = "id";
+ private static final String YES = "yes";
+ private static final int NUM_QUERY_VECTORS = 256;
+
+ @Param({"200000"})
+ public int numDocs;
+
+ @Param({"128"})
+ public int dim;
+
+ @Param({"100"})
+ public int k;
+
+ @Param({"0.95", "0.05"})
+ public double filterSelectivity;
+
+ @Param({"0", "0.1"})
+ public double vectorlessFraction;
+
+ @Param({"0", "0.05"})
+ public double deletedFraction;
+
+ private Path tmpDir;
+ private Directory dir;
+ private IndexReader reader;
+ private IndexSearcher searcher;
+ private LRUQueryCache queryCache;
+ private Query filter;
+ private float[][] queryVectors;
+ private int queryIdx;
+
+ @Setup(Level.Trial)
+ public void setup() throws IOException {
+ tmpDir = Files.createTempDirectory("FilteredKnnVectorQueryBenchmark");
+ dir = MMapDirectory.open(tmpDir);
+
+ Random randomForDoc = new Random(42);
+ try (IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig())) {
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ doc.add(new StringField(ID_FIELD, Integer.toString(i), Field.Store.NO));
+ if (randomForDoc.nextDouble() < filterSelectivity) {
+ doc.add(new StringField(FILTER_FIELD, YES, Field.Store.NO));
+ }
+ if (randomForDoc.nextDouble() >= vectorlessFraction) {
+ doc.add(
+ new KnnFloatVectorField(
+ FIELD,
+ randomUnitVector(dim, randomForDoc),
+ VectorSimilarityFunction.DOT_PRODUCT));
+ }
+ indexWriter.addDocument(doc);
+ }
+ indexWriter.forceMerge(1);
+ if (deletedFraction > 0) {
+ for (int i = 0; i < numDocs; i++) {
+ if (randomForDoc.nextDouble() < deletedFraction) {
+ indexWriter.deleteDocuments(new Term(ID_FIELD, Integer.toString(i)));
+ }
+ }
+ indexWriter.commit();
+ }
+ }
+
+ reader = DirectoryReader.open(dir);
+ if (reader.leaves().size() != 1) {
+ throw new AssertionError("expected one segment, got " + reader.leaves().size());
+ }
+ filter = new TermQuery(new Term(FILTER_FIELD, YES));
+ queryCache = new LRUQueryCache(256, 64L * 1024 * 1024, context -> context.reader() != null, 1f);
+ searcher = new IndexSearcher(reader);
+ searcher.setQueryCache(queryCache);
+ searcher.setQueryCachingPolicy(cacheOnly(filter));
+
+ // Pull a scorer through the cache once: IndexSearcher#count would answer a term query from
+ // index statistics without ever creating the scorer that populates the cache.
+ Weight filterWeight =
+ searcher.createWeight(searcher.rewrite(filter), ScoreMode.COMPLETE_NO_SCORES, 1f);
+ ScorerSupplier scorerSupplier = filterWeight.scorerSupplier(reader.leaves().get(0));
+ if (scorerSupplier == null) {
+ throw new AssertionError("filter matches nothing");
+ }
+ scorerSupplier.get(Long.MAX_VALUE).iterator().nextDoc();
+ if (queryCache.getCacheSize() == 0) {
+ throw new AssertionError("filter query was not cached");
+ }
+
+ Random randomForQuery = new Random(123);
+ queryVectors = new float[NUM_QUERY_VECTORS][];
+ for (int i = 0; i < NUM_QUERY_VECTORS; i++) {
+ queryVectors[i] = randomUnitVector(dim, randomForQuery);
+ }
+ }
+
+ @TearDown(Level.Trial)
+ public void teardown() throws IOException {
+ IOUtils.close(queryCache, reader, dir);
+ IOUtils.rm(tmpDir);
+ }
+
+ @Benchmark
+ public TopDocs search() throws IOException {
+ float[] query = queryVectors[queryIdx++ & (NUM_QUERY_VECTORS - 1)];
+ return searcher.search(new KnnFloatVectorQuery(FIELD, query, k, filter), k);
+ }
+
+ private static QueryCachingPolicy cacheOnly(Query queryToCache) {
+ return new QueryCachingPolicy() {
+ @Override
+ public void onUse(Query query) {}
+
+ @Override
+ public boolean shouldCache(Query query) {
+ return queryToCache.equals(query);
+ }
+ };
+ }
+
+ private static float[] randomUnitVector(int dim, Random rnd) {
+ float[] v = new float[dim];
+ for (int i = 0; i < dim; i++) v[i] = rnd.nextFloat() * 2 - 1;
+ return l2Normalization(v);
+ }
+
+ private static float[] l2Normalization(float[] v) {
+ float norm = 0;
+ for (float x : v) norm += x * x;
+ norm = (float) Math.sqrt(norm);
+ for (int i = 0; i < v.length; i++) v[i] /= norm;
+ return v;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/AcceptDocs.java b/lucene/core/src/java/org/apache/lucene/search/AcceptDocs.java
index 8771f63438e..d73f1e837dd 100644
--- a/lucene/core/src/java/org/apache/lucene/search/AcceptDocs.java
+++ b/lucene/core/src/java/org/apache/lucene/search/AcceptDocs.java
@@ -176,7 +176,10 @@ public abstract class AcceptDocs {
private void createBitSetAcceptDocsIfNecessary() throws IOException {
if (acceptBitSet == null) {
- acceptBitSet = Objects.requireNonNull(createBitSet(iterator(), liveDocs, maxDoc));
+ // Pass the raw iterator: #createBitSet applies liveDocs itself, and filtering upfront
+ // would hide DocIdSetIterator#intoBitSet behind a wrapper that has no bulk implementation.
+ DocIdSetIterator iterator = Objects.requireNonNull(iteratorSupplier.get());
+ acceptBitSet = Objects.requireNonNull(createBitSet(iterator, liveDocs, maxDoc));
cardinality = acceptBitSet.cardinality();
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAcceptDocs.java b/lucene/core/src/test/org/apache/lucene/search/TestAcceptDocs.java
index b8a8e7ba51b..20a3ec5d643 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestAcceptDocs.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAcceptDocs.java
@@ -19,6 +19,9 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.BitSet;
+import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
@@ -91,4 +94,179 @@ public class TestAcceptDocs extends LuceneTestCase {
assertEquals(-1, iterator.docID());
}
}
+
+ /**
+ * A {@link BitSetIterator} that counts calls to {@link #nextDoc()}. It extends {@link
+ * BitSetIterator} rather than wrapping one so that the bulk {@link BitSetIterator#intoBitSet}
+ * implementation remains visible to callers.
+ */
+ private static class CountingBitSetIterator extends BitSetIterator {
+
+ int nextDocCalls;
+
+ CountingBitSetIterator(BitSet bits, long cost) {
+ super(bits, cost);
+ }
+
+ @Override
+ public int nextDoc() {
+ nextDocCalls++;
+ return super.nextDoc();
+ }
+ }
+
+ /**
+ * Dense accept docs must be loaded through {@link DocIdSetIterator#intoBitSet}, which {@link
+ * BitSetIterator} implements by copying whole words, and only then be masked with live docs via
+ * {@link Bits#applyMask}. Filtering the iterator upfront would fall back to one {@link
+ * DocIdSetIterator#nextDoc()} call per matching doc. Both approaches return the same bits, so
+ * this can only be observed by counting calls on the source iterator.
+ */
+ public void testDenseIteratorIsConsumedInBulkWhenSegmentHasDeletions() throws IOException {
+ int iters = atLeast(20);
+ for (int iter = 0; iter < iters; ++iter) {
+ int maxDoc = TestUtil.nextInt(random(), 1024, 1 << 16);
+
+ // Match every `step`th doc, so that the cost is always >= maxDoc >> 7 and AcceptDocs loads
+ // matches into a FixedBitSet.
+ FixedBitSet filter = new FixedBitSet(maxDoc);
+ int step = TestUtil.nextInt(random(), 1, 16);
+ for (int doc = random().nextInt(step); doc < maxDoc; doc += step) {
+ filter.set(doc);
+ }
+ int filterCardinality = filter.cardinality();
+ assertTrue(filterCardinality > (maxDoc >> 7));
+
+ // A segment that has deletions, ie. a non-null liveDocs: none, one, or many deleted docs.
+ FixedBitSet liveDocs = new FixedBitSet(maxDoc);
+ liveDocs.set(0, maxDoc);
+ int deleteCount =
+ switch (random().nextInt(3)) {
+ case 0 -> 0;
+ case 1 -> 1;
+ default -> TestUtil.nextInt(random(), 2, maxDoc);
+ };
+ for (int i = 0; i < deleteCount; ++i) {
+ liveDocs.clear(random().nextInt(maxDoc));
+ }
+
+ CountingBitSetIterator iterator = new CountingBitSetIterator(filter, filterCardinality);
+ int[] iteratorsPulled = new int[1];
+ AcceptDocs acceptDocs =
+ AcceptDocs.fromIteratorSupplier(
+ () -> {
+ iteratorsPulled[0]++;
+ return iterator;
+ },
+ liveDocs.asReadOnlyBits(),
+ maxDoc);
+
+ Bits acceptBits = acceptDocs.bits();
+
+ // The bit set must be built from a single bulk pass over the source iterator.
+ assertEquals(1, iteratorsPulled[0]);
+ assertTrue(
+ "iterator was consumed one doc at a time: "
+ + iterator.nextDocCalls
+ + " nextDoc() calls for "
+ + filterCardinality
+ + " matching docs",
+ iterator.nextDocCalls <= 2);
+
+ // Loading in bulk must not change the result.
+ FixedBitSet expected = filter.clone();
+ expected.and(liveDocs);
+ assertEquals(expected, acceptBits);
+ assertEquals(expected.cardinality(), acceptDocs.cost());
+ }
+ }
+
+ /**
+ * The bits exposed by {@link AcceptDocs#bits()} are always the matches of the iterator
+ * intersected with live docs, on both sides of the {@code maxDoc >> 7} dense/sparse boundary and
+ * whatever the number of deleted docs.
+ */
+ public void testRandomBitsAreMatchesIntersectedWithLiveDocs() throws IOException {
+ int iters = atLeast(100);
+ for (int iter = 0; iter < iters; ++iter) {
+ int maxDoc = TestUtil.nextInt(random(), 1, 5000);
+ int threshold = maxDoc >> 7; // AcceptDocs' dense/sparse boundary
+
+ int targetCardinality =
+ switch (random().nextInt(6)) {
+ case 0 -> 0;
+ case 1 -> Math.max(0, threshold - 1);
+ case 2 -> threshold;
+ case 3 -> threshold + 1;
+ case 4 -> maxDoc;
+ default -> TestUtil.nextInt(random(), 0, maxDoc);
+ };
+ targetCardinality = Math.min(targetCardinality, maxDoc);
+
+ FixedBitSet filter = new FixedBitSet(maxDoc);
+ for (int cardinality = 0; cardinality < targetCardinality; ) {
+ if (filter.getAndSet(random().nextInt(maxDoc)) == false) {
+ cardinality++;
+ }
+ }
+ long cost = filter.cardinality();
+
+ // None, one or many deleted docs -- and sometimes a segment with no deletions at all.
+ FixedBitSet liveDocs = new FixedBitSet(maxDoc);
+ liveDocs.set(0, maxDoc);
+ Bits liveDocsBits;
+ if (random().nextInt(5) == 0) {
+ liveDocsBits = null; // no deletions
+ } else {
+ int deleteCount =
+ switch (random().nextInt(3)) {
+ case 0 -> 0;
+ case 1 -> 1;
+ default -> TestUtil.nextInt(random(), 1, maxDoc);
+ };
+ for (int i = 0; i < deleteCount; ++i) {
+ liveDocs.clear(random().nextInt(maxDoc));
+ }
+ liveDocsBits = liveDocs.asReadOnlyBits();
+ }
+
+ FixedBitSet expected = filter.clone();
+ expected.and(liveDocs);
+
+ // Half of the time, hide the fact that the source is a BitSetIterator behind a
+ // FilterDocIdSetIterator, which uses the default impl of intoBitSet, so that the generic
+ // path gets the same coverage as the optimized one.
+ boolean opaque = random().nextBoolean();
+ AcceptDocs acceptDocs =
+ AcceptDocs.fromIteratorSupplier(
+ () -> {
+ DocIdSetIterator iterator = new BitSetIterator(filter, cost);
+ return opaque ? new FilterDocIdSetIterator(iterator) : iterator;
+ },
+ liveDocsBits,
+ maxDoc);
+
+ Bits acceptBits = acceptDocs.bits();
+ assertEquals(maxDoc, acceptBits.length());
+ for (int doc = 0; doc < maxDoc; ++doc) {
+ assertEquals(
+ "doc=" + doc + " maxDoc=" + maxDoc + " cardinality=" + cost,
+ expected.get(doc),
+ acceptBits.get(doc));
+ }
+ assertEquals(expected.cardinality(), acceptDocs.cost());
+ if (acceptBits instanceof FixedBitSet fixedBitSet) {
+ assertEquals(expected, fixedBitSet);
+ }
+
+ // The iterator must agree with the bits.
+ DocIdSetIterator iterator = acceptDocs.iterator();
+ for (int doc = expected.nextSetBit(0);
+ doc != DocIdSetIterator.NO_MORE_DOCS;
+ doc = doc + 1 >= maxDoc ? DocIdSetIterator.NO_MORE_DOCS : expected.nextSetBit(doc + 1)) {
+ assertEquals(doc, iterator.nextDoc());
+ }
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
+ }
+ }
}