Skip to content

Apache Lucene open-source search software

LUCENE-10233: Use AND NOT for inverse intersector (#499)

gf21215f09bb3fab0e
diff --git a/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
index b1950e5b366..4adf7452913 100644
--- a/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
@@ -166,7 +166,7 @@ final class LatLonPointDistanceQuery extends Query {
// by computing the set of documents that do NOT match the range
final FixedBitSet result = new FixedBitSet(reader.maxDoc());
result.set(0, reader.maxDoc());
- int[] cost = new int[] {reader.maxDoc()};
+ long[] cost = new long[] {reader.maxDoc()};
values.intersect(getInverseIntersectVisitor(result, cost));
final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);
return new ConstantScoreScorer(weight, score(), scoreMode, iterator);
@@ -286,7 +286,7 @@ final class LatLonPointDistanceQuery extends Query {
}
/** Create a visitor that clears documents that do NOT match the range. */
- private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, int[] cost) {
+ private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, long[] cost) {
return new IntersectVisitor() {
@Override
@@ -295,6 +295,12 @@ final class LatLonPointDistanceQuery extends Query {
cost[0]--;
}
+ @Override
+ public void visit(DocIdSetIterator iterator) throws IOException {
+ result.andNot(iterator);
+ cost[0] = Math.max(0, cost[0] - iterator.cost());
+ }
+
@Override
public void visit(int docID, byte[] packedValue) {
if (matches(packedValue) == false) {
diff --git a/lucene/core/src/java/org/apache/lucene/document/SpatialQuery.java b/lucene/core/src/java/org/apache/lucene/document/SpatialQuery.java
index 19a5d83bcb8..c6170ecad62 100644
--- a/lucene/core/src/java/org/apache/lucene/document/SpatialQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/document/SpatialQuery.java
@@ -468,6 +468,12 @@ abstract class SpatialQuery extends Query {
cost[0]++;
}
+ @Override
+ public void visit(DocIdSetIterator iterator) throws IOException {
+ result.or(iterator);
+ cost[0] += iterator.cost();
+ }
+
@Override
public void visit(int docID, byte[] t) {
if (result.get(docID) == false) {
@@ -514,6 +520,12 @@ abstract class SpatialQuery extends Query {
cost[0]++;
}
+ @Override
+ public void visit(DocIdSetIterator iterator) throws IOException {
+ result.or(iterator);
+ cost[0] += iterator.cost();
+ }
+
@Override
public void visit(int docID, byte[] t) {
if (excluded.get(docID) == false) {
@@ -564,6 +576,11 @@ abstract class SpatialQuery extends Query {
excluded.set(docID);
}
+ @Override
+ public void visit(DocIdSetIterator iterator) throws IOException {
+ excluded.or(iterator);
+ }
+
@Override
public void visit(int docID, byte[] t) {
if (excluded.get(docID) == false) {
@@ -618,6 +635,12 @@ abstract class SpatialQuery extends Query {
cost[0]--;
}
+ @Override
+ public void visit(DocIdSetIterator iterator) throws IOException {
+ result.andNot(iterator);
+ cost[0] = Math.max(0, cost[0] - iterator.cost());
+ }
+
@Override
public void visit(int docID, byte[] packedTriangle) {
if (result.get(docID)) {
@@ -660,6 +683,11 @@ abstract class SpatialQuery extends Query {
result.clear(docID);
}
+ @Override
+ public void visit(DocIdSetIterator iterator) throws IOException {
+ result.andNot(iterator);
+ }
+
@Override
public void visit(int docID, byte[] packedTriangle) {
// NO-OP
diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
index 06b7836e4be..af6645aaaf8 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
@@ -208,7 +208,7 @@ public abstract class PointRangeQuery extends Query {
}
/** Create a visitor that clears documents that do NOT match the range. */
- private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, int[] cost) {
+ private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, long[] cost) {
return new IntersectVisitor() {
@Override
@@ -217,6 +217,12 @@ public abstract class PointRangeQuery extends Query {
cost[0]--;
}
+ @Override
+ public void visit(DocIdSetIterator iterator) throws IOException {
+ result.andNot(iterator);
+ cost[0] = Math.max(0, cost[0] - iterator.cost());
+ }
+
@Override
public void visit(int docID, byte[] packedValue) {
if (matches(packedValue) == false) {
@@ -330,7 +336,7 @@ public abstract class PointRangeQuery extends Query {
// by computing the set of documents that do NOT match the range
final FixedBitSet result = new FixedBitSet(reader.maxDoc());
result.set(0, reader.maxDoc());
- int[] cost = new int[] {reader.maxDoc()};
+ long[] cost = new long[] {reader.maxDoc()};
values.intersect(getInverseIntersectVisitor(result, cost));
final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);
return new ConstantScoreScorer(weight, score(), scoreMode, iterator);
diff --git a/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java b/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
index d30a3147425..71eb31b47f3 100644
--- a/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
+++ b/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
@@ -351,16 +351,38 @@ public final class FixedBitSet extends BitSet {
}
}
+ public void andNot(DocIdSetIterator iter) throws IOException {
+ if (BitSetIterator.getFixedBitSetOrNull(iter) != null) {
+ checkUnpositioned(iter);
+ final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter);
+ assert bits != null;
+ andNot(bits);
+ } else if (iter instanceof DocBaseBitSetIterator) {
+ checkUnpositioned(iter);
+ DocBaseBitSetIterator baseIter = (DocBaseBitSetIterator) iter;
+ andNot(baseIter.getDocBase() >> 6, baseIter.getBitSet());
+ } else {
+ checkUnpositioned(iter);
+ for (int doc = iter.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iter.nextDoc()) {
+ clear(doc);
+ }
+ }
+ }
+
/** this = this AND NOT other */
public void andNot(FixedBitSet other) {
- andNot(other.bits, other.numWords);
+ andNot(0, other.bits, other.numWords);
}
- private void andNot(final long[] otherArr, final int otherNumWords) {
+ private void andNot(final int otherOffsetWords, FixedBitSet other) {
+ andNot(otherOffsetWords, other.bits, other.numWords);
+ }
+
+ private void andNot(final int otherOffsetWords, final long[] otherArr, final int otherNumWords) {
+ int pos = Math.min(numWords - otherOffsetWords, otherNumWords);
final long[] thisArr = this.bits;
- int pos = Math.min(this.numWords, otherNumWords);
while (--pos >= 0) {
- thisArr[pos] &= ~otherArr[pos];
+ thisArr[pos + otherOffsetWords] &= ~otherArr[pos];
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java b/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java
index 0c3884743c2..11cf4c0f705 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java
@@ -17,6 +17,7 @@
package org.apache.lucene.util;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Random;
import org.apache.lucene.search.DocIdSetIterator;
@@ -446,6 +447,55 @@ public class TestFixedBitSet extends BaseBitSetTestCase<FixedBitSet> {
assertEquals(bitSet1.cardinality(), intersectionCount);
}
+ public void testAndNot() throws IOException {
+ Random random = random();
+
+ int numBits2 = TestUtil.nextInt(random, 1000, 2000);
+ int numBits1 = TestUtil.nextInt(random, 1000, numBits2);
+
+ int count1 = TestUtil.nextInt(random, 0, numBits1 - 1);
+ int count2 = TestUtil.nextInt(random, 0, numBits2 - 1);
+
+ int offSetWord1 = FixedBitSet.bits2words(TestUtil.nextInt(random, 0, count1));
+ int offset1 = offSetWord1 << 6;
+ int[] bits1 = makeIntArray(random, count1, offset1, numBits1 - 1);
+ int[] bits2 = makeIntArray(random, count2, 0, numBits2 - 1);
+
+ java.util.BitSet bitSet1 = makeBitSet(bits1);
+ java.util.BitSet bitSet2 = makeBitSet(bits2);
+ bitSet2.andNot(bitSet1);
+
+ {
+ // test BitSetIterator
+ FixedBitSet fixedBitSet2 = makeFixedBitSet(bits2, numBits2);
+ DocIdSetIterator disi = new BitSetIterator(makeFixedBitSet(bits1, numBits1), count1);
+ fixedBitSet2.andNot(disi);
+ doGet(bitSet2, fixedBitSet2);
+ }
+
+ {
+ // test DocBaseBitSetIterator
+ FixedBitSet fixedBitSet2 = makeFixedBitSet(bits2, numBits2);
+ int[] offsetBits = Arrays.stream(bits1).map(i -> i - offset1).toArray();
+ DocIdSetIterator disi =
+ new DocBaseBitSetIterator(
+ makeFixedBitSet(offsetBits, numBits1 - offset1), count1, offset1);
+ fixedBitSet2.andNot(disi);
+ doGet(bitSet2, fixedBitSet2);
+ }
+
+ {
+ // test other
+ FixedBitSet fixedBitSet2 = makeFixedBitSet(bits2, numBits2);
+ int[] sorted = new int[bits1.length + 1];
+ System.arraycopy(bits1, 0, sorted, 0, bits1.length);
+ sorted[bits1.length] = DocIdSetIterator.NO_MORE_DOCS;
+ DocIdSetIterator disi = new IntArrayDocIdSet.IntArrayDocIdSetIterator(sorted, count1);
+ fixedBitSet2.andNot(disi);
+ doGet(bitSet2, fixedBitSet2);
+ }
+ }
+
// Demonstrates that the presence of ghost bits in the last used word can cause spurious failures
public void testUnionCount() {
Random random = random();