Skip to content

Apache Lucene open-source search software

Implement #docIdRunEnd in the iterators used by RoaringDocIdSet (#16077)

Ignacio Vera6a0e4611eebb
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3c66d2d0e94..15c2db97b9d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -383,6 +383,9 @@ Optimizations
* GITHUB#16072: Added efficient implementations for BitSetIterator#docIDRunEnd. (Ignacio Vera)
+* GITHUB#16077: Added efficient implementations if the method #docIDRunEnd for the iterators used
+ by RoaringDocIdSet. (Ignacio Vera)
+
Bug Fixes
---------------------
* GITHUB#15754: Fix HTMLStripCharFilter to prevent tags from incorrectly consuming subsequent
diff --git a/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java b/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java
index f41d2353507..7d6856ea592 100644
--- a/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java
+++ b/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java
@@ -77,6 +77,13 @@ public final class NotDocIdSet extends DocIdSet {
}
}
+ @Override
+ public int docIDRunEnd() {
+ assert doc != NO_MORE_DOCS;
+ assert nextSkippedDoc > doc;
+ return Math.min(nextSkippedDoc, maxDoc);
+ }
+
@Override
public long cost() {
// even if there are few docs in this set, iterating over all documents
diff --git a/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java b/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java
index 011759a0d7d..1eed8c1c0d9 100644
--- a/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java
+++ b/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java
@@ -226,6 +226,17 @@ public class RoaringDocIdSet extends DocIdSet {
bitSet.set(docId(i) - offset);
}
}
+
+ @Override
+ public int docIDRunEnd() {
+ int runEnd = doc + 1;
+ int j = i + 1;
+ while (j < docIDs.length && (docIDs[j] & 0xFFFF) == runEnd) {
+ runEnd++;
+ j++;
+ }
+ return runEnd;
+ }
};
}
}
@@ -334,6 +345,29 @@ public class RoaringDocIdSet extends DocIdSet {
}
}
+ @Override
+ public int docIDRunEnd() throws IOException {
+ assert doc != NO_MORE_DOCS && sub != null;
+ int b = block;
+ long globalEnd = ((long) b << 16) + (long) sub.docIDRunEnd();
+ // Merge with the next block when the sub-run ends exactly on a 64K boundary and the next
+ // block's first doc is the immediate successor (local doc 0).
+ while (globalEnd == ((long) (b + 1) << 16) && globalEnd < (long) Integer.MAX_VALUE) {
+ int nb = b + 1;
+ if (nb >= docIdSets.length || docIdSets[nb] == null) {
+ break;
+ }
+ DocIdSetIterator nextIt = docIdSets[nb].iterator();
+ int first = nextIt.nextDoc();
+ if (first != 0) {
+ break;
+ }
+ b = nb;
+ globalEnd = ((long) b << 16) + (long) nextIt.docIDRunEnd();
+ }
+ return globalEnd >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) globalEnd;
+ }
+
@Override
public long cost() {
return cardinality;
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java b/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java
index c4256447f4e..3109281685d 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java
@@ -18,6 +18,7 @@ package org.apache.lucene.util;
import java.io.IOException;
import java.util.BitSet;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.tests.util.BaseDocIdSetTestCase;
public class TestNotDocIdSet extends BaseDocIdSetTestCase<NotDocIdSet> {
@@ -30,4 +31,64 @@ public class TestNotDocIdSet extends BaseDocIdSetTestCase<NotDocIdSet> {
}
return new NotDocIdSet(length, new BitDocIdSet(set));
}
+
+ public void testDocIDRunEndContiguousRun() throws IOException {
+ final int maxDoc = random().nextInt(2, 1_000);
+ BitSet bs = new BitSet();
+ int start = random().nextInt(0, maxDoc - 1);
+ int end = random().nextInt(start + 1, maxDoc);
+ for (int d = start; d < end; d++) {
+ bs.set(d);
+ }
+ assertDocIDRunEndMatches(bs, maxDoc, copyOf(bs, maxDoc));
+ }
+
+ public void testRandomDocIDRunEnd() throws IOException {
+ final int iters = TEST_NIGHTLY ? 50 : 5;
+ for (int iter = 0; iter < iters; iter++) {
+ final int maxDoc = random().nextInt(50, 1 << 18);
+ final BitSet bs = randomBitSet(maxDoc, random().nextFloat());
+ if (bs.isEmpty()) {
+ continue;
+ }
+ assertDocIDRunEndMatches(bs, maxDoc, copyOf(bs, maxDoc));
+ }
+ }
+
+ private static BitSet randomBitSet(int numBits, float percentSet) {
+ final int numBitsSet = Math.min(numBits, (int) (percentSet * numBits));
+ final BitSet set = new BitSet(numBits);
+ if (numBitsSet >= numBits) {
+ set.set(0, numBits);
+ return set;
+ }
+ for (int i = 0; i < numBitsSet; ++i) {
+ while (true) {
+ final int o = random().nextInt(numBits);
+ if (!set.get(o)) {
+ set.set(o);
+ break;
+ }
+ }
+ }
+ return set;
+ }
+
+ private static int expectedDocIDRunEnd(BitSet bs, int maxDoc, int doc) {
+ int end = doc + 1;
+ while (end < maxDoc && bs.get(end)) {
+ end++;
+ }
+ return end;
+ }
+
+ private static void assertDocIDRunEndMatches(BitSet bs, int maxDoc, NotDocIdSet set)
+ throws IOException {
+ DocIdSetIterator it = set.iterator();
+ assertNotNull(it);
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ assertTrue(bs.get(doc));
+ assertEquals(expectedDocIDRunEnd(bs, maxDoc, doc), it.docIDRunEnd());
+ }
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestRoaringDocIdSet.java b/lucene/core/src/test/org/apache/lucene/util/TestRoaringDocIdSet.java
index fed8075b0f2..3eb3555d442 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestRoaringDocIdSet.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestRoaringDocIdSet.java
@@ -18,6 +18,7 @@ package org.apache.lucene.util;
import java.io.IOException;
import java.util.BitSet;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.tests.util.BaseDocIdSetTestCase;
public class TestRoaringDocIdSet extends BaseDocIdSetTestCase<RoaringDocIdSet> {
@@ -36,4 +37,88 @@ public class TestRoaringDocIdSet extends BaseDocIdSetTestCase<RoaringDocIdSet> {
super.assertEquals(numBits, ds1, ds2);
assertEquals(ds1.cardinality(), ds2.cardinality());
}
+
+ public void testDocIDRunEndContiguousWithinBlock() throws IOException {
+ final int maxDoc = random().nextInt(2, 50_000);
+ BitSet bs = new BitSet();
+ int start = random().nextInt(0, maxDoc - 1);
+ int end = random().nextInt(start + 1, maxDoc);
+ for (int d = start; d < end; d++) {
+ bs.set(d);
+ }
+ RoaringDocIdSet set = copyOf(bs, maxDoc);
+ assertDocIDRunEndMatches(bs, maxDoc, set);
+ }
+
+ public void testDocIDRunEndFinishInBoundary() throws IOException {
+ final int boundary = 1 << 16;
+ final int maxDoc = boundary + 100;
+ BitSet bs = new BitSet();
+ for (int d = boundary - 3; d < boundary; d++) {
+ bs.set(d);
+ }
+ RoaringDocIdSet set = copyOf(bs, maxDoc);
+ assertDocIDRunEndMatches(bs, maxDoc, set);
+ }
+
+ public void testDocIDRunEndAcross64KBoundary() throws IOException {
+ final int boundary = 1 << 16;
+ final int maxDoc = boundary + 100;
+ BitSet bs = new BitSet();
+ for (int d = boundary - 3; d < boundary + 7; d++) {
+ bs.set(d);
+ }
+ RoaringDocIdSet set = copyOf(bs, maxDoc);
+ assertDocIDRunEndMatches(bs, maxDoc, set);
+ }
+
+ private static BitSet randomBitSet(int numBits, float percentSet) {
+ final int numBitsSet = Math.min(numBits, (int) (percentSet * numBits));
+ final BitSet set = new BitSet(numBits);
+ if (numBitsSet >= numBits) {
+ set.set(0, numBits);
+ return set;
+ }
+ for (int i = 0; i < numBitsSet; ++i) {
+ while (true) {
+ final int o = random().nextInt(numBits);
+ if (!set.get(o)) {
+ set.set(o);
+ break;
+ }
+ }
+ }
+ return set;
+ }
+
+ public void testRandomDocIDRunEnd() throws IOException {
+ final int iters = TEST_NIGHTLY ? 50 : 5;
+ for (int iter = 0; iter < iters; iter++) {
+ final int maxDoc = random().nextInt(50, 1 << 18);
+ final BitSet bs = randomBitSet(maxDoc, random().nextFloat());
+ if (bs.isEmpty()) {
+ assertNull(copyOf(bs, maxDoc).iterator());
+ continue;
+ }
+ assertDocIDRunEndMatches(bs, maxDoc, copyOf(bs, maxDoc));
+ }
+ }
+
+ private static int expectedDocIDRunEnd(BitSet bs, int maxDoc, int doc) {
+ int end = doc + 1;
+ while (end < maxDoc && bs.get(end)) {
+ end++;
+ }
+ return end;
+ }
+
+ private static void assertDocIDRunEndMatches(BitSet bs, int maxDoc, RoaringDocIdSet set)
+ throws IOException {
+ DocIdSetIterator it = set.iterator();
+ assertNotNull(it);
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ assertTrue(bs.get(doc));
+ assertEquals(expectedDocIDRunEnd(bs, maxDoc, doc), it.docIDRunEnd());
+ }
+ }
}