Apache Lucene open-source search software
LUCENE-10233: Use AND NOT for inverse intersector (#499)
gf21215f09bb3fab0e
diff --git a/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.javaindex b1950e5b366..4adf7452913 100644--- a/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java+++ b/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java@@ -166,7 +166,7 @@ final class LatLonPointDistanceQuery extends Query {// by computing the set of documents that do NOT match the rangefinal FixedBitSet result = new FixedBitSet(reader.maxDoc());result.set(0, reader.maxDoc());- int[] cost = new int[] {reader.maxDoc()};+ long[] cost = new long[] {reader.maxDoc()};values.intersect(getInverseIntersectVisitor(result, cost));final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);return new ConstantScoreScorer(weight, score(), scoreMode, iterator);@@ -286,7 +286,7 @@ final class LatLonPointDistanceQuery extends Query {}/** Create a visitor that clears documents that do NOT match the range. */- private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, int[] cost) {+ private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, long[] cost) {return new IntersectVisitor() {@Override@@ -295,6 +295,12 @@ final class LatLonPointDistanceQuery extends Query {cost[0]--;}+ @Override+ public void visit(DocIdSetIterator iterator) throws IOException {+ result.andNot(iterator);+ cost[0] = Math.max(0, cost[0] - iterator.cost());+ }+@Overridepublic void visit(int docID, byte[] packedValue) {if (matches(packedValue) == false) {diff --git a/lucene/core/src/java/org/apache/lucene/document/SpatialQuery.java b/lucene/core/src/java/org/apache/lucene/document/SpatialQuery.javaindex 19a5d83bcb8..c6170ecad62 100644--- a/lucene/core/src/java/org/apache/lucene/document/SpatialQuery.java+++ b/lucene/core/src/java/org/apache/lucene/document/SpatialQuery.java@@ -468,6 +468,12 @@ abstract class SpatialQuery extends Query {cost[0]++;}+ @Override+ public void visit(DocIdSetIterator iterator) throws IOException {+ result.or(iterator);+ cost[0] += iterator.cost();+ }+@Overridepublic void visit(int docID, byte[] t) {if (result.get(docID) == false) {@@ -514,6 +520,12 @@ abstract class SpatialQuery extends Query {cost[0]++;}+ @Override+ public void visit(DocIdSetIterator iterator) throws IOException {+ result.or(iterator);+ cost[0] += iterator.cost();+ }+@Overridepublic void visit(int docID, byte[] t) {if (excluded.get(docID) == false) {@@ -564,6 +576,11 @@ abstract class SpatialQuery extends Query {excluded.set(docID);}+ @Override+ public void visit(DocIdSetIterator iterator) throws IOException {+ excluded.or(iterator);+ }+@Overridepublic void visit(int docID, byte[] t) {if (excluded.get(docID) == false) {@@ -618,6 +635,12 @@ abstract class SpatialQuery extends Query {cost[0]--;}+ @Override+ public void visit(DocIdSetIterator iterator) throws IOException {+ result.andNot(iterator);+ cost[0] = Math.max(0, cost[0] - iterator.cost());+ }+@Overridepublic void visit(int docID, byte[] packedTriangle) {if (result.get(docID)) {@@ -660,6 +683,11 @@ abstract class SpatialQuery extends Query {result.clear(docID);}+ @Override+ public void visit(DocIdSetIterator iterator) throws IOException {+ result.andNot(iterator);+ }+@Overridepublic void visit(int docID, byte[] packedTriangle) {// NO-OPdiff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.javaindex 06b7836e4be..af6645aaaf8 100644--- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java@@ -208,7 +208,7 @@ public abstract class PointRangeQuery extends Query {}/** Create a visitor that clears documents that do NOT match the range. */- private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, int[] cost) {+ private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, long[] cost) {return new IntersectVisitor() {@Override@@ -217,6 +217,12 @@ public abstract class PointRangeQuery extends Query {cost[0]--;}+ @Override+ public void visit(DocIdSetIterator iterator) throws IOException {+ result.andNot(iterator);+ cost[0] = Math.max(0, cost[0] - iterator.cost());+ }+@Overridepublic void visit(int docID, byte[] packedValue) {if (matches(packedValue) == false) {@@ -330,7 +336,7 @@ public abstract class PointRangeQuery extends Query {// by computing the set of documents that do NOT match the rangefinal FixedBitSet result = new FixedBitSet(reader.maxDoc());result.set(0, reader.maxDoc());- int[] cost = new int[] {reader.maxDoc()};+ long[] cost = new long[] {reader.maxDoc()};values.intersect(getInverseIntersectVisitor(result, cost));final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);return new ConstantScoreScorer(weight, score(), scoreMode, iterator);diff --git a/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java b/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.javaindex d30a3147425..71eb31b47f3 100644--- a/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java+++ b/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java@@ -351,16 +351,38 @@ public final class FixedBitSet extends BitSet {}}+ public void andNot(DocIdSetIterator iter) throws IOException {+ if (BitSetIterator.getFixedBitSetOrNull(iter) != null) {+ checkUnpositioned(iter);+ final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter);+ assert bits != null;+ andNot(bits);+ } else if (iter instanceof DocBaseBitSetIterator) {+ checkUnpositioned(iter);+ DocBaseBitSetIterator baseIter = (DocBaseBitSetIterator) iter;+ andNot(baseIter.getDocBase() >> 6, baseIter.getBitSet());+ } else {+ checkUnpositioned(iter);+ for (int doc = iter.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iter.nextDoc()) {+ clear(doc);+ }+ }+ }+/** this = this AND NOT other */public void andNot(FixedBitSet other) {- andNot(other.bits, other.numWords);+ andNot(0, other.bits, other.numWords);}- private void andNot(final long[] otherArr, final int otherNumWords) {+ private void andNot(final int otherOffsetWords, FixedBitSet other) {+ andNot(otherOffsetWords, other.bits, other.numWords);+ }++ private void andNot(final int otherOffsetWords, final long[] otherArr, final int otherNumWords) {+ int pos = Math.min(numWords - otherOffsetWords, otherNumWords);final long[] thisArr = this.bits;- int pos = Math.min(this.numWords, otherNumWords);while (--pos >= 0) {- thisArr[pos] &= ~otherArr[pos];+ thisArr[pos + otherOffsetWords] &= ~otherArr[pos];}}diff --git a/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java b/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.javaindex 0c3884743c2..11cf4c0f705 100644--- a/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java+++ b/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java@@ -17,6 +17,7 @@package org.apache.lucene.util;import java.io.IOException;+import java.util.Arrays;import java.util.Random;import org.apache.lucene.search.DocIdSetIterator;@@ -446,6 +447,55 @@ public class TestFixedBitSet extends BaseBitSetTestCase<FixedBitSet> {assertEquals(bitSet1.cardinality(), intersectionCount);}+ public void testAndNot() throws IOException {+ Random random = random();++ int numBits2 = TestUtil.nextInt(random, 1000, 2000);+ int numBits1 = TestUtil.nextInt(random, 1000, numBits2);++ int count1 = TestUtil.nextInt(random, 0, numBits1 - 1);+ int count2 = TestUtil.nextInt(random, 0, numBits2 - 1);++ int offSetWord1 = FixedBitSet.bits2words(TestUtil.nextInt(random, 0, count1));+ int offset1 = offSetWord1 << 6;+ int[] bits1 = makeIntArray(random, count1, offset1, numBits1 - 1);+ int[] bits2 = makeIntArray(random, count2, 0, numBits2 - 1);++ java.util.BitSet bitSet1 = makeBitSet(bits1);+ java.util.BitSet bitSet2 = makeBitSet(bits2);+ bitSet2.andNot(bitSet1);++ {+ // test BitSetIterator+ FixedBitSet fixedBitSet2 = makeFixedBitSet(bits2, numBits2);+ DocIdSetIterator disi = new BitSetIterator(makeFixedBitSet(bits1, numBits1), count1);+ fixedBitSet2.andNot(disi);+ doGet(bitSet2, fixedBitSet2);+ }++ {+ // test DocBaseBitSetIterator+ FixedBitSet fixedBitSet2 = makeFixedBitSet(bits2, numBits2);+ int[] offsetBits = Arrays.stream(bits1).map(i -> i - offset1).toArray();+ DocIdSetIterator disi =+ new DocBaseBitSetIterator(+ makeFixedBitSet(offsetBits, numBits1 - offset1), count1, offset1);+ fixedBitSet2.andNot(disi);+ doGet(bitSet2, fixedBitSet2);+ }++ {+ // test other+ FixedBitSet fixedBitSet2 = makeFixedBitSet(bits2, numBits2);+ int[] sorted = new int[bits1.length + 1];+ System.arraycopy(bits1, 0, sorted, 0, bits1.length);+ sorted[bits1.length] = DocIdSetIterator.NO_MORE_DOCS;+ DocIdSetIterator disi = new IntArrayDocIdSet.IntArrayDocIdSetIterator(sorted, count1);+ fixedBitSet2.andNot(disi);+ doGet(bitSet2, fixedBitSet2);+ }+ }+// Demonstrates that the presence of ghost bits in the last used word can cause spurious failurespublic void testUnionCount() {Random random = random();