Apache Lucene open-source search software
Implement #docIdRunEnd in the iterators used by RoaringDocIdSet (#16077)
Ignacio Vera6a0e4611eebb
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 3c66d2d0e94..15c2db97b9d 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -383,6 +383,9 @@ Optimizations* GITHUB#16072: Added efficient implementations for BitSetIterator#docIDRunEnd. (Ignacio Vera)+* GITHUB#16077: Added efficient implementations if the method #docIDRunEnd for the iterators used+ by RoaringDocIdSet. (Ignacio Vera)+Bug Fixes---------------------* GITHUB#15754: Fix HTMLStripCharFilter to prevent tags from incorrectly consuming subsequentdiff --git a/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java b/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.javaindex f41d2353507..7d6856ea592 100644--- a/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java+++ b/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java@@ -77,6 +77,13 @@ public final class NotDocIdSet extends DocIdSet {}}+ @Override+ public int docIDRunEnd() {+ assert doc != NO_MORE_DOCS;+ assert nextSkippedDoc > doc;+ return Math.min(nextSkippedDoc, maxDoc);+ }+@Overridepublic long cost() {// even if there are few docs in this set, iterating over all documentsdiff --git a/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java b/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.javaindex 011759a0d7d..1eed8c1c0d9 100644--- a/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java+++ b/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java@@ -226,6 +226,17 @@ public class RoaringDocIdSet extends DocIdSet {bitSet.set(docId(i) - offset);}}++ @Override+ public int docIDRunEnd() {+ int runEnd = doc + 1;+ int j = i + 1;+ while (j < docIDs.length && (docIDs[j] & 0xFFFF) == runEnd) {+ runEnd++;+ j++;+ }+ return runEnd;+ }};}}@@ -334,6 +345,29 @@ public class RoaringDocIdSet extends DocIdSet {}}+ @Override+ public int docIDRunEnd() throws IOException {+ assert doc != NO_MORE_DOCS && sub != null;+ int b = block;+ long globalEnd = ((long) b << 16) + (long) sub.docIDRunEnd();+ // Merge with the next block when the sub-run ends exactly on a 64K boundary and the next+ // block's first doc is the immediate successor (local doc 0).+ while (globalEnd == ((long) (b + 1) << 16) && globalEnd < (long) Integer.MAX_VALUE) {+ int nb = b + 1;+ if (nb >= docIdSets.length || docIdSets[nb] == null) {+ break;+ }+ DocIdSetIterator nextIt = docIdSets[nb].iterator();+ int first = nextIt.nextDoc();+ if (first != 0) {+ break;+ }+ b = nb;+ globalEnd = ((long) b << 16) + (long) nextIt.docIDRunEnd();+ }+ return globalEnd >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) globalEnd;+ }+@Overridepublic long cost() {return cardinality;diff --git a/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java b/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.javaindex c4256447f4e..3109281685d 100644--- a/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java+++ b/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java@@ -18,6 +18,7 @@ package org.apache.lucene.util;import java.io.IOException;import java.util.BitSet;+import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.tests.util.BaseDocIdSetTestCase;public class TestNotDocIdSet extends BaseDocIdSetTestCase<NotDocIdSet> {@@ -30,4 +31,64 @@ public class TestNotDocIdSet extends BaseDocIdSetTestCase<NotDocIdSet> {}return new NotDocIdSet(length, new BitDocIdSet(set));}++ public void testDocIDRunEndContiguousRun() throws IOException {+ final int maxDoc = random().nextInt(2, 1_000);+ BitSet bs = new BitSet();+ int start = random().nextInt(0, maxDoc - 1);+ int end = random().nextInt(start + 1, maxDoc);+ for (int d = start; d < end; d++) {+ bs.set(d);+ }+ assertDocIDRunEndMatches(bs, maxDoc, copyOf(bs, maxDoc));+ }++ public void testRandomDocIDRunEnd() throws IOException {+ final int iters = TEST_NIGHTLY ? 50 : 5;+ for (int iter = 0; iter < iters; iter++) {+ final int maxDoc = random().nextInt(50, 1 << 18);+ final BitSet bs = randomBitSet(maxDoc, random().nextFloat());+ if (bs.isEmpty()) {+ continue;+ }+ assertDocIDRunEndMatches(bs, maxDoc, copyOf(bs, maxDoc));+ }+ }++ private static BitSet randomBitSet(int numBits, float percentSet) {+ final int numBitsSet = Math.min(numBits, (int) (percentSet * numBits));+ final BitSet set = new BitSet(numBits);+ if (numBitsSet >= numBits) {+ set.set(0, numBits);+ return set;+ }+ for (int i = 0; i < numBitsSet; ++i) {+ while (true) {+ final int o = random().nextInt(numBits);+ if (!set.get(o)) {+ set.set(o);+ break;+ }+ }+ }+ return set;+ }++ private static int expectedDocIDRunEnd(BitSet bs, int maxDoc, int doc) {+ int end = doc + 1;+ while (end < maxDoc && bs.get(end)) {+ end++;+ }+ return end;+ }++ private static void assertDocIDRunEndMatches(BitSet bs, int maxDoc, NotDocIdSet set)+ throws IOException {+ DocIdSetIterator it = set.iterator();+ assertNotNull(it);+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {+ assertTrue(bs.get(doc));+ assertEquals(expectedDocIDRunEnd(bs, maxDoc, doc), it.docIDRunEnd());+ }+ }}diff --git a/lucene/core/src/test/org/apache/lucene/util/TestRoaringDocIdSet.java b/lucene/core/src/test/org/apache/lucene/util/TestRoaringDocIdSet.javaindex fed8075b0f2..3eb3555d442 100644--- a/lucene/core/src/test/org/apache/lucene/util/TestRoaringDocIdSet.java+++ b/lucene/core/src/test/org/apache/lucene/util/TestRoaringDocIdSet.java@@ -18,6 +18,7 @@ package org.apache.lucene.util;import java.io.IOException;import java.util.BitSet;+import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.tests.util.BaseDocIdSetTestCase;public class TestRoaringDocIdSet extends BaseDocIdSetTestCase<RoaringDocIdSet> {@@ -36,4 +37,88 @@ public class TestRoaringDocIdSet extends BaseDocIdSetTestCase<RoaringDocIdSet> {super.assertEquals(numBits, ds1, ds2);assertEquals(ds1.cardinality(), ds2.cardinality());}++ public void testDocIDRunEndContiguousWithinBlock() throws IOException {+ final int maxDoc = random().nextInt(2, 50_000);+ BitSet bs = new BitSet();+ int start = random().nextInt(0, maxDoc - 1);+ int end = random().nextInt(start + 1, maxDoc);+ for (int d = start; d < end; d++) {+ bs.set(d);+ }+ RoaringDocIdSet set = copyOf(bs, maxDoc);+ assertDocIDRunEndMatches(bs, maxDoc, set);+ }++ public void testDocIDRunEndFinishInBoundary() throws IOException {+ final int boundary = 1 << 16;+ final int maxDoc = boundary + 100;+ BitSet bs = new BitSet();+ for (int d = boundary - 3; d < boundary; d++) {+ bs.set(d);+ }+ RoaringDocIdSet set = copyOf(bs, maxDoc);+ assertDocIDRunEndMatches(bs, maxDoc, set);+ }++ public void testDocIDRunEndAcross64KBoundary() throws IOException {+ final int boundary = 1 << 16;+ final int maxDoc = boundary + 100;+ BitSet bs = new BitSet();+ for (int d = boundary - 3; d < boundary + 7; d++) {+ bs.set(d);+ }+ RoaringDocIdSet set = copyOf(bs, maxDoc);+ assertDocIDRunEndMatches(bs, maxDoc, set);+ }++ private static BitSet randomBitSet(int numBits, float percentSet) {+ final int numBitsSet = Math.min(numBits, (int) (percentSet * numBits));+ final BitSet set = new BitSet(numBits);+ if (numBitsSet >= numBits) {+ set.set(0, numBits);+ return set;+ }+ for (int i = 0; i < numBitsSet; ++i) {+ while (true) {+ final int o = random().nextInt(numBits);+ if (!set.get(o)) {+ set.set(o);+ break;+ }+ }+ }+ return set;+ }++ public void testRandomDocIDRunEnd() throws IOException {+ final int iters = TEST_NIGHTLY ? 50 : 5;+ for (int iter = 0; iter < iters; iter++) {+ final int maxDoc = random().nextInt(50, 1 << 18);+ final BitSet bs = randomBitSet(maxDoc, random().nextFloat());+ if (bs.isEmpty()) {+ assertNull(copyOf(bs, maxDoc).iterator());+ continue;+ }+ assertDocIDRunEndMatches(bs, maxDoc, copyOf(bs, maxDoc));+ }+ }++ private static int expectedDocIDRunEnd(BitSet bs, int maxDoc, int doc) {+ int end = doc + 1;+ while (end < maxDoc && bs.get(end)) {+ end++;+ }+ return end;+ }++ private static void assertDocIDRunEndMatches(BitSet bs, int maxDoc, RoaringDocIdSet set)+ throws IOException {+ DocIdSetIterator it = set.iterator();+ assertNotNull(it);+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {+ assertTrue(bs.get(doc));+ assertEquals(expectedDocIDRunEnd(bs, maxDoc, doc), it.docIDRunEnd());+ }+ }}