Apache Lucene open-source search software
Updating Dense#intoBitSet to properly set upTo if it exceeds bitset size (#14922)
Panagiotis Bailis7271b5f2e2dc
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 66666a723f0..252260076d8 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -234,6 +234,9 @@ Bug Fixes* GITHUB#14838: Make it possible to extend Patience/Seeded knn queries (Tommaso Teofili)+* GITHUB#14922: Fix for IndexedDISI.Dense#intoBitSetWithinBlock to take into account the provided bitset size+ and avoid throwing IndexOutOfBoundsException (Panagiotis Bailis)+Build---------------------* Upgrade forbiddenapis to version 3.9. (Uwe Schindler)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.javaindex 1d64be1a928..83350e5ce95 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java@@ -27,6 +27,7 @@ import org.apache.lucene.store.RandomAccessInput;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BitSetIterator;import org.apache.lucene.util.FixedBitSet;+import org.apache.lucene.util.MathUtil;import org.apache.lucene.util.RoaringDocIdSet;/**@@ -473,7 +474,7 @@ public final class IndexedDISI extends AbstractDocIdSetIterator {@Overridepublic void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {- assert doc >= offset;+ assert doc >= offset : "offset=" + offset + " doc=" + doc;while (doc < upTo && method.intoBitSetWithinBlock(this, upTo, bitSet, offset) == false) {readBlockHeader();boolean found = method.advanceWithinBlock(this, block);@@ -719,10 +720,10 @@ public final class IndexedDISI extends AbstractDocIdSetIterator {if (disi.bitSet == null) {disi.bitSet = new FixedBitSet(BLOCK_SIZE);}-- int sourceFrom = disi.doc & 0xFFFF;- int sourceTo = Math.min(upTo - disi.block, BLOCK_SIZE);int destFrom = disi.doc - offset;+ int destTo = MathUtil.unsignedMin(upTo, offset + bitSet.length());+ int sourceFrom = disi.doc & 0xFFFF;+ int sourceTo = Math.min(destTo - disi.block, BLOCK_SIZE);long fp = disi.slice.getFilePointer();disi.slice.seek(fp - Long.BYTES); // seek back a long to include current word (disi.word).@@ -731,13 +732,24 @@ public final class IndexedDISI extends AbstractDocIdSetIterator {FixedBitSet.orRange(disi.bitSet, sourceFrom, bitSet, destFrom, sourceTo - sourceFrom);int blockEnd = disi.block | 0xFFFF;- if (upTo > blockEnd) {+ if (destTo > blockEnd) {disi.slice.seek(disi.blockEnd);disi.index += disi.bitSet.cardinality(sourceFrom, sourceTo);return false;} else {disi.slice.seek(fp);- return advanceWithinBlock(disi, upTo);+ boolean found = advanceWithinBlock(disi, destTo);+ if (found && disi.doc < upTo) {+ throw new IllegalStateException(+ "There are bits set in the source bitset that are not accounted for."+ + " doc="+ + disi.doc+ + " upTo="+ + upTo+ + " disi.block="+ + disi.block);+ }+ return found;}}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestIndexedDISI.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestIndexedDISI.javaindex 135019fe387..974abff9f24 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestIndexedDISI.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestIndexedDISI.java@@ -388,6 +388,36 @@ public class TestIndexedDISI extends LuceneTestCase {}}+ public void testDenseBitSizeLessThanBlockSize() throws IOException {+ final byte denseRankPower = (byte) (random().nextInt(7) + 7);+ try (Directory dir = newDirectory()) {+ // initialize a maxDoc that is less than IndexedDISI.BLOCK_SIZE+ int maxDoc = random().nextInt(4096 * 2, 65536);+ FixedBitSet set = new FixedBitSet(maxDoc);+ for (int i = 0; i < maxDoc; i += 2) { // Set every other to ensure dense+ set.set(i);+ }+ int jumpTableEntryCount; // this should always be 0 given that maxDoc < BLOCK_SIZE+ long length;+ try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {+ jumpTableEntryCount =+ IndexedDISI.writeBitSet(+ new BitSetIterator(set, set.cardinality()), out, denseRankPower);+ length = out.getFilePointer();+ assertTrue(+ "jumpTableEntryCount should be 0 for dense bitsets with size < BLOCK_SIZE",+ 0 == jumpTableEntryCount);+ }+ try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {+ IndexedDISI disi =+ new IndexedDISI(in, 0L, length, jumpTableEntryCount, denseRankPower, set.cardinality());+ FixedBitSet disiSet = new FixedBitSet(maxDoc);+ // This would throw IOOB if bitset size is not handled correctly as per #14882+ disiSet.or(disi);+ }+ }+ }+public void testIllegalDenseRankPower() throws IOException {// Legal values