Apache Lucene open-source search software
Build the AcceptDocs bit set from the raw filter iterator (#16588)
John Maurice1a65ceae18ea
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex e5cd853bcfa..bf53046f9b2 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -497,6 +497,9 @@ Optimizations* GITHUB#16631: Vectorize the uint4 nibble unpack used by Lucene104ScalarQuantizedVectorsFormatvia VectorUtilSupport, vectors are now unpacked with the Panama Vector API. (Jakub Slowinski)+* GITHUB#16588: AcceptDocs builds its bit set from the raw filter iterator so that+ DocIdSetIterator#intoBitSet is used on segments with deletions. (John Maurice)+Bug Fixes---------------------diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/FilteredKnnVectorQueryBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/FilteredKnnVectorQueryBenchmark.javanew file mode 100644index 00000000000..7cad02a4132--- /dev/null+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/FilteredKnnVectorQueryBenchmark.java@@ -0,0 +1,232 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.benchmark.jmh;++import java.io.IOException;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.Random;+import java.util.concurrent.TimeUnit;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.Term;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.KnnFloatVectorQuery;+import org.apache.lucene.search.LRUQueryCache;+import org.apache.lucene.search.Query;+import org.apache.lucene.search.QueryCachingPolicy;+import org.apache.lucene.search.ScoreMode;+import org.apache.lucene.search.ScorerSupplier;+import org.apache.lucene.search.TermQuery;+import org.apache.lucene.search.TopDocs;+import org.apache.lucene.search.Weight;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.MMapDirectory;+import org.apache.lucene.util.IOUtils;+import org.openjdk.jmh.annotations.Benchmark;+import org.openjdk.jmh.annotations.BenchmarkMode;+import org.openjdk.jmh.annotations.Fork;+import org.openjdk.jmh.annotations.Level;+import org.openjdk.jmh.annotations.Measurement;+import org.openjdk.jmh.annotations.Mode;+import org.openjdk.jmh.annotations.OutputTimeUnit;+import org.openjdk.jmh.annotations.Param;+import org.openjdk.jmh.annotations.Scope;+import org.openjdk.jmh.annotations.Setup;+import org.openjdk.jmh.annotations.State;+import org.openjdk.jmh.annotations.TearDown;+import org.openjdk.jmh.annotations.Warmup;++/**+ * Benchmarks end-to-end latency of {@link KnnFloatVectorQuery} whose pre-filter is a cached term+ * query, over a single segment.+ *+ * <p>The filter is warmed into {@link LRUQueryCache} before measurement, so every measured query+ * reuses the cached bit set, which is how a hot pre-filter reaches the kNN query in practice. The+ * corpus shape decides whether {@link org.apache.lucene.search.AcceptDocs} can load that filter in+ * bulk.+ *+ * <p>Parameters explore:+ *+ * <ul>+ * <li>{@code numDocs} — total number of documents in the segment.+ * <li>{@code dim} — vector dimension.+ * <li>{@code k} — number of results requested.+ * <li>{@code filterSelectivity} — fraction of docs matched by the pre-filter.+ * <li>{@code vectorlessFraction} — fraction of docs indexed without a vector.+ * <li>{@code deletedFraction} — fraction of docs deleted after indexing, which keeps the+ * segment's liveDocs non-null.+ * </ul>+ *+ * Run with:+ *+ * <pre>+ * ./gradlew -p lucene/benchmark-jmh assemble+ * java -jar lucene/benchmark-jmh/build/benchmarks/lucene-benchmark-jmh-*.jar FilteredKnnVectorQueryBenchmark+ * </pre>+ */+@BenchmarkMode(Mode.SampleTime)+@OutputTimeUnit(TimeUnit.MILLISECONDS)+@State(Scope.Benchmark)+@Warmup(iterations = 5, time = 2)+@Measurement(iterations = 5, time = 5)+@Fork(+ value = 1,+ jvmArgsAppend = {"-Xmx4g", "-Xms4g", "-XX:+AlwaysPreTouch"})+public class FilteredKnnVectorQueryBenchmark {++ private static final String FIELD = "vec";+ private static final String FILTER_FIELD = "tag";+ private static final String ID_FIELD = "id";+ private static final String YES = "yes";+ private static final int NUM_QUERY_VECTORS = 256;++ @Param({"200000"})+ public int numDocs;++ @Param({"128"})+ public int dim;++ @Param({"100"})+ public int k;++ @Param({"0.95", "0.05"})+ public double filterSelectivity;++ @Param({"0", "0.1"})+ public double vectorlessFraction;++ @Param({"0", "0.05"})+ public double deletedFraction;++ private Path tmpDir;+ private Directory dir;+ private IndexReader reader;+ private IndexSearcher searcher;+ private LRUQueryCache queryCache;+ private Query filter;+ private float[][] queryVectors;+ private int queryIdx;++ @Setup(Level.Trial)+ public void setup() throws IOException {+ tmpDir = Files.createTempDirectory("FilteredKnnVectorQueryBenchmark");+ dir = MMapDirectory.open(tmpDir);++ Random randomForDoc = new Random(42);+ try (IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig())) {+ for (int i = 0; i < numDocs; i++) {+ Document doc = new Document();+ doc.add(new StringField(ID_FIELD, Integer.toString(i), Field.Store.NO));+ if (randomForDoc.nextDouble() < filterSelectivity) {+ doc.add(new StringField(FILTER_FIELD, YES, Field.Store.NO));+ }+ if (randomForDoc.nextDouble() >= vectorlessFraction) {+ doc.add(+ new KnnFloatVectorField(+ FIELD,+ randomUnitVector(dim, randomForDoc),+ VectorSimilarityFunction.DOT_PRODUCT));+ }+ indexWriter.addDocument(doc);+ }+ indexWriter.forceMerge(1);+ if (deletedFraction > 0) {+ for (int i = 0; i < numDocs; i++) {+ if (randomForDoc.nextDouble() < deletedFraction) {+ indexWriter.deleteDocuments(new Term(ID_FIELD, Integer.toString(i)));+ }+ }+ indexWriter.commit();+ }+ }++ reader = DirectoryReader.open(dir);+ if (reader.leaves().size() != 1) {+ throw new AssertionError("expected one segment, got " + reader.leaves().size());+ }+ filter = new TermQuery(new Term(FILTER_FIELD, YES));+ queryCache = new LRUQueryCache(256, 64L * 1024 * 1024, context -> context.reader() != null, 1f);+ searcher = new IndexSearcher(reader);+ searcher.setQueryCache(queryCache);+ searcher.setQueryCachingPolicy(cacheOnly(filter));++ // Pull a scorer through the cache once: IndexSearcher#count would answer a term query from+ // index statistics without ever creating the scorer that populates the cache.+ Weight filterWeight =+ searcher.createWeight(searcher.rewrite(filter), ScoreMode.COMPLETE_NO_SCORES, 1f);+ ScorerSupplier scorerSupplier = filterWeight.scorerSupplier(reader.leaves().get(0));+ if (scorerSupplier == null) {+ throw new AssertionError("filter matches nothing");+ }+ scorerSupplier.get(Long.MAX_VALUE).iterator().nextDoc();+ if (queryCache.getCacheSize() == 0) {+ throw new AssertionError("filter query was not cached");+ }++ Random randomForQuery = new Random(123);+ queryVectors = new float[NUM_QUERY_VECTORS][];+ for (int i = 0; i < NUM_QUERY_VECTORS; i++) {+ queryVectors[i] = randomUnitVector(dim, randomForQuery);+ }+ }++ @TearDown(Level.Trial)+ public void teardown() throws IOException {+ IOUtils.close(queryCache, reader, dir);+ IOUtils.rm(tmpDir);+ }++ @Benchmark+ public TopDocs search() throws IOException {+ float[] query = queryVectors[queryIdx++ & (NUM_QUERY_VECTORS - 1)];+ return searcher.search(new KnnFloatVectorQuery(FIELD, query, k, filter), k);+ }++ private static QueryCachingPolicy cacheOnly(Query queryToCache) {+ return new QueryCachingPolicy() {+ @Override+ public void onUse(Query query) {}++ @Override+ public boolean shouldCache(Query query) {+ return queryToCache.equals(query);+ }+ };+ }++ private static float[] randomUnitVector(int dim, Random rnd) {+ float[] v = new float[dim];+ for (int i = 0; i < dim; i++) v[i] = rnd.nextFloat() * 2 - 1;+ return l2Normalization(v);+ }++ private static float[] l2Normalization(float[] v) {+ float norm = 0;+ for (float x : v) norm += x * x;+ norm = (float) Math.sqrt(norm);+ for (int i = 0; i < v.length; i++) v[i] /= norm;+ return v;+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/search/AcceptDocs.java b/lucene/core/src/java/org/apache/lucene/search/AcceptDocs.javaindex 8771f63438e..d73f1e837dd 100644--- a/lucene/core/src/java/org/apache/lucene/search/AcceptDocs.java+++ b/lucene/core/src/java/org/apache/lucene/search/AcceptDocs.java@@ -176,7 +176,10 @@ public abstract class AcceptDocs {private void createBitSetAcceptDocsIfNecessary() throws IOException {if (acceptBitSet == null) {- acceptBitSet = Objects.requireNonNull(createBitSet(iterator(), liveDocs, maxDoc));+ // Pass the raw iterator: #createBitSet applies liveDocs itself, and filtering upfront+ // would hide DocIdSetIterator#intoBitSet behind a wrapper that has no bulk implementation.+ DocIdSetIterator iterator = Objects.requireNonNull(iteratorSupplier.get());+ acceptBitSet = Objects.requireNonNull(createBitSet(iterator, liveDocs, maxDoc));cardinality = acceptBitSet.cardinality();}}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAcceptDocs.java b/lucene/core/src/test/org/apache/lucene/search/TestAcceptDocs.javaindex b8a8e7ba51b..20a3ec5d643 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestAcceptDocs.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestAcceptDocs.java@@ -19,6 +19,9 @@ package org.apache.lucene.search;import java.io.IOException;import java.util.Arrays;import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.BitSet;+import org.apache.lucene.util.BitSetIterator;import org.apache.lucene.util.Bits;import org.apache.lucene.util.FixedBitSet;@@ -91,4 +94,179 @@ public class TestAcceptDocs extends LuceneTestCase {assertEquals(-1, iterator.docID());}}++ /**+ * A {@link BitSetIterator} that counts calls to {@link #nextDoc()}. It extends {@link+ * BitSetIterator} rather than wrapping one so that the bulk {@link BitSetIterator#intoBitSet}+ * implementation remains visible to callers.+ */+ private static class CountingBitSetIterator extends BitSetIterator {++ int nextDocCalls;++ CountingBitSetIterator(BitSet bits, long cost) {+ super(bits, cost);+ }++ @Override+ public int nextDoc() {+ nextDocCalls++;+ return super.nextDoc();+ }+ }++ /**+ * Dense accept docs must be loaded through {@link DocIdSetIterator#intoBitSet}, which {@link+ * BitSetIterator} implements by copying whole words, and only then be masked with live docs via+ * {@link Bits#applyMask}. Filtering the iterator upfront would fall back to one {@link+ * DocIdSetIterator#nextDoc()} call per matching doc. Both approaches return the same bits, so+ * this can only be observed by counting calls on the source iterator.+ */+ public void testDenseIteratorIsConsumedInBulkWhenSegmentHasDeletions() throws IOException {+ int iters = atLeast(20);+ for (int iter = 0; iter < iters; ++iter) {+ int maxDoc = TestUtil.nextInt(random(), 1024, 1 << 16);++ // Match every `step`th doc, so that the cost is always >= maxDoc >> 7 and AcceptDocs loads+ // matches into a FixedBitSet.+ FixedBitSet filter = new FixedBitSet(maxDoc);+ int step = TestUtil.nextInt(random(), 1, 16);+ for (int doc = random().nextInt(step); doc < maxDoc; doc += step) {+ filter.set(doc);+ }+ int filterCardinality = filter.cardinality();+ assertTrue(filterCardinality > (maxDoc >> 7));++ // A segment that has deletions, ie. a non-null liveDocs: none, one, or many deleted docs.+ FixedBitSet liveDocs = new FixedBitSet(maxDoc);+ liveDocs.set(0, maxDoc);+ int deleteCount =+ switch (random().nextInt(3)) {+ case 0 -> 0;+ case 1 -> 1;+ default -> TestUtil.nextInt(random(), 2, maxDoc);+ };+ for (int i = 0; i < deleteCount; ++i) {+ liveDocs.clear(random().nextInt(maxDoc));+ }++ CountingBitSetIterator iterator = new CountingBitSetIterator(filter, filterCardinality);+ int[] iteratorsPulled = new int[1];+ AcceptDocs acceptDocs =+ AcceptDocs.fromIteratorSupplier(+ () -> {+ iteratorsPulled[0]++;+ return iterator;+ },+ liveDocs.asReadOnlyBits(),+ maxDoc);++ Bits acceptBits = acceptDocs.bits();++ // The bit set must be built from a single bulk pass over the source iterator.+ assertEquals(1, iteratorsPulled[0]);+ assertTrue(+ "iterator was consumed one doc at a time: "+ + iterator.nextDocCalls+ + " nextDoc() calls for "+ + filterCardinality+ + " matching docs",+ iterator.nextDocCalls <= 2);++ // Loading in bulk must not change the result.+ FixedBitSet expected = filter.clone();+ expected.and(liveDocs);+ assertEquals(expected, acceptBits);+ assertEquals(expected.cardinality(), acceptDocs.cost());+ }+ }++ /**+ * The bits exposed by {@link AcceptDocs#bits()} are always the matches of the iterator+ * intersected with live docs, on both sides of the {@code maxDoc >> 7} dense/sparse boundary and+ * whatever the number of deleted docs.+ */+ public void testRandomBitsAreMatchesIntersectedWithLiveDocs() throws IOException {+ int iters = atLeast(100);+ for (int iter = 0; iter < iters; ++iter) {+ int maxDoc = TestUtil.nextInt(random(), 1, 5000);+ int threshold = maxDoc >> 7; // AcceptDocs' dense/sparse boundary++ int targetCardinality =+ switch (random().nextInt(6)) {+ case 0 -> 0;+ case 1 -> Math.max(0, threshold - 1);+ case 2 -> threshold;+ case 3 -> threshold + 1;+ case 4 -> maxDoc;+ default -> TestUtil.nextInt(random(), 0, maxDoc);+ };+ targetCardinality = Math.min(targetCardinality, maxDoc);++ FixedBitSet filter = new FixedBitSet(maxDoc);+ for (int cardinality = 0; cardinality < targetCardinality; ) {+ if (filter.getAndSet(random().nextInt(maxDoc)) == false) {+ cardinality++;+ }+ }+ long cost = filter.cardinality();++ // None, one or many deleted docs -- and sometimes a segment with no deletions at all.+ FixedBitSet liveDocs = new FixedBitSet(maxDoc);+ liveDocs.set(0, maxDoc);+ Bits liveDocsBits;+ if (random().nextInt(5) == 0) {+ liveDocsBits = null; // no deletions+ } else {+ int deleteCount =+ switch (random().nextInt(3)) {+ case 0 -> 0;+ case 1 -> 1;+ default -> TestUtil.nextInt(random(), 1, maxDoc);+ };+ for (int i = 0; i < deleteCount; ++i) {+ liveDocs.clear(random().nextInt(maxDoc));+ }+ liveDocsBits = liveDocs.asReadOnlyBits();+ }++ FixedBitSet expected = filter.clone();+ expected.and(liveDocs);++ // Half of the time, hide the fact that the source is a BitSetIterator behind a+ // FilterDocIdSetIterator, which uses the default impl of intoBitSet, so that the generic+ // path gets the same coverage as the optimized one.+ boolean opaque = random().nextBoolean();+ AcceptDocs acceptDocs =+ AcceptDocs.fromIteratorSupplier(+ () -> {+ DocIdSetIterator iterator = new BitSetIterator(filter, cost);+ return opaque ? new FilterDocIdSetIterator(iterator) : iterator;+ },+ liveDocsBits,+ maxDoc);++ Bits acceptBits = acceptDocs.bits();+ assertEquals(maxDoc, acceptBits.length());+ for (int doc = 0; doc < maxDoc; ++doc) {+ assertEquals(+ "doc=" + doc + " maxDoc=" + maxDoc + " cardinality=" + cost,+ expected.get(doc),+ acceptBits.get(doc));+ }+ assertEquals(expected.cardinality(), acceptDocs.cost());+ if (acceptBits instanceof FixedBitSet fixedBitSet) {+ assertEquals(expected, fixedBitSet);+ }++ // The iterator must agree with the bits.+ DocIdSetIterator iterator = acceptDocs.iterator();+ for (int doc = expected.nextSetBit(0);+ doc != DocIdSetIterator.NO_MORE_DOCS;+ doc = doc + 1 >= maxDoc ? DocIdSetIterator.NO_MORE_DOCS : expected.nextSetBit(doc + 1)) {+ assertEquals(doc, iterator.nextDoc());+ }+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());+ }+ }}