Apache Lucene open-source search software
Decode doc ids in BKD leaves with auto-vectorized loops (#14203)
gf21219472dcad0ebf
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 9603ab872e8..8d0bf31ceb6 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -178,6 +178,8 @@ Optimizations* GITHUB#14301: Avoid unnecessary evaluations and skipping documents. (hanbj)+# GITHUB#14203: Decode doc ids in BKD leaves with auto-vectorized loops when using DEFAULT_MAX_POINTS_IN_LEAF_NODE. (Guo Feng)+Bug Fixes---------------------diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsFormat.javaindex 64e61991eb1..a59b0181463 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsFormat.java@@ -17,11 +17,13 @@package org.apache.lucene.codecs.lucene90;import java.io.IOException;+import java.util.Map;import org.apache.lucene.codecs.PointsFormat;import org.apache.lucene.codecs.PointsReader;import org.apache.lucene.codecs.PointsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.util.bkd.BKDWriter;/*** Lucene 9.0 point format, which encodes dimensional values in a block KD-tree structure for fast@@ -59,18 +61,40 @@ public final class Lucene90PointsFormat extends PointsFormat {public static final String META_EXTENSION = "kdm";static final int VERSION_START = 0;- static final int VERSION_CURRENT = VERSION_START;+ static final int VERSION_BKD_VECTORIZED_BPV24 = 1;+ static final int VERSION_CURRENT = VERSION_BKD_VECTORIZED_BPV24;++ private static final Map<Integer, Integer> VERSION_TO_BKD_VERSION =+ Map.of(+ VERSION_START, BKDWriter.VERSION_META_FILE,+ VERSION_BKD_VECTORIZED_BPV24, BKDWriter.VERSION_VECTORIZED_DOCID);++ private final int version;/** Sole constructor */- public Lucene90PointsFormat() {}+ public Lucene90PointsFormat() {+ this(VERSION_CURRENT);+ }++ /** Constructor that takes a version. This is used for testing with older versions. */+ Lucene90PointsFormat(int version) {+ if (VERSION_TO_BKD_VERSION.containsKey(version) == false) {+ throw new IllegalArgumentException("Invalid version: " + version);+ }+ this.version = version;+ }@Overridepublic PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {- return new Lucene90PointsWriter(state);+ return new Lucene90PointsWriter(state, version);}@Overridepublic PointsReader fieldsReader(SegmentReadState state) throws IOException {return new Lucene90PointsReader(state);}++ static int bkdVersion(int version) {+ return VERSION_TO_BKD_VERSION.get(version);+ }}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.javaindex 2de98e55688..456d12cd962 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java@@ -46,16 +46,18 @@ public class Lucene90PointsWriter extends PointsWriter {final SegmentWriteState writeState;final int maxPointsInLeafNode;final double maxMBSortInHeap;+ final int version;private boolean finished;/** Full constructor */public Lucene90PointsWriter(- SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap)+ SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap, int version)throws IOException {assert writeState.fieldInfos.hasPointValues();this.writeState = writeState;this.maxPointsInLeafNode = maxPointsInLeafNode;this.maxMBSortInHeap = maxMBSortInHeap;+ this.version = version;String dataFileName =IndexFileNames.segmentFileName(writeState.segmentInfo.name,@@ -105,6 +107,12 @@ public class Lucene90PointsWriter extends PointsWriter {}}+ public Lucene90PointsWriter(+ SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap)+ throws IOException {+ this(writeState, maxPointsInLeafNode, maxMBSortInHeap, Lucene90PointsFormat.VERSION_CURRENT);+ }+/*** Uses the defaults values for {@code maxPointsInLeafNode} (512) and {@code maxMBSortInHeap}* (16.0)@@ -113,7 +121,17 @@ public class Lucene90PointsWriter extends PointsWriter {this(writeState,BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE,- BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP);+ BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP,+ Lucene90PointsFormat.VERSION_CURRENT);+ }++ /** Constructor that takes a version. This is used for testing with older versions. */+ Lucene90PointsWriter(SegmentWriteState writeState, int version) throws IOException {+ this(+ writeState,+ BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE,+ BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP,+ version);}@Override@@ -135,7 +153,8 @@ public class Lucene90PointsWriter extends PointsWriter {writeState.segmentInfo.name,config,maxMBSortInHeap,- values.size())) {+ values.size(),+ Lucene90PointsFormat.bkdVersion(version))) {if (values instanceof MutablePointTree) {IORunnable finalizer =@@ -233,7 +252,8 @@ public class Lucene90PointsWriter extends PointsWriter {writeState.segmentInfo.name,config,maxMBSortInHeap,- totMaxSize)) {+ totMaxSize,+ Lucene90PointsFormat.bkdVersion(version))) {List<PointValues> pointValues = new ArrayList<>();List<MergeState.DocMap> docMaps = new ArrayList<>();for (int i = 0; i < mergeState.pointsReaders.length; i++) {diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.javaindex 17b0cb79845..40f995e61ea 100644--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java@@ -34,7 +34,6 @@ import org.apache.lucene.util.MathUtil;* @lucene.experimental*/public class BKDReader extends PointValues {-final BKDConfig config;final int numLeaves;final IndexInput in;@@ -269,7 +268,7 @@ public class BKDReader extends PointValues {1,minPackedValue,maxPackedValue,- new BKDReaderDocIDSetIterator(config.maxPointsInLeafNode()),+ new BKDReaderDocIDSetIterator(config.maxPointsInLeafNode(), version),new byte[config.packedBytesLength()],new byte[config.packedIndexBytesLength()],new byte[config.packedIndexBytesLength()],@@ -598,7 +597,8 @@ public class BKDReader extends PointValues {// How many points are stored in this leaf cell:int count = leafNodes.readVInt();// No need to call grow(), it has been called up-front- docIdsWriter.readInts(leafNodes, count, visitor);+ // Borrow scratchIterator.docIds as decoding buffer+ docIdsWriter.readInts(leafNodes, count, visitor, scratchIterator.docIDs);} else {pushLeft();addAll(visitor, grown);@@ -1036,9 +1036,9 @@ public class BKDReader extends PointValues {final int[] docIDs;private final DocIdsWriter docIdsWriter;- public BKDReaderDocIDSetIterator(int maxPointsInLeafNode) {+ public BKDReaderDocIDSetIterator(int maxPointsInLeafNode, int version) {this.docIDs = new int[maxPointsInLeafNode];- this.docIdsWriter = new DocIdsWriter(maxPointsInLeafNode);+ this.docIdsWriter = new DocIdsWriter(maxPointsInLeafNode, version);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.javaindex 2ce7d7e87ef..5db1af560c2 100644--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java@@ -85,7 +85,8 @@ public class BKDWriter implements Closeable {public static final int VERSION_SELECTIVE_INDEXING = 6;public static final int VERSION_LOW_CARDINALITY_LEAVES = 7;public static final int VERSION_META_FILE = 9;- public static final int VERSION_CURRENT = VERSION_META_FILE;+ public static final int VERSION_VECTORIZED_DOCID = 10;+ public static final int VERSION_CURRENT = VERSION_VECTORIZED_DOCID;/** Number of splits before we compute the exact bounding box of an inner node. */private static final int SPLITS_BEFORE_EXACT_BOUNDS = 4;@@ -103,6 +104,7 @@ public class BKDWriter implements Closeable {final TrackingDirectoryWrapper tempDir;final String tempFileNamePrefix;final double maxMBSortInHeap;+ final int version;final byte[] scratchDiff;final byte[] scratch;@@ -139,6 +141,29 @@ public class BKDWriter implements Closeable {BKDConfig config,double maxMBSortInHeap,long totalPointCount) {+ this(+ maxDoc,+ tempDir,+ tempFileNamePrefix,+ config,+ maxMBSortInHeap,+ totalPointCount,+ BKDWriter.VERSION_CURRENT);+ }++ /** This ctor should be only used for testing with older versions. */+ public BKDWriter(+ int maxDoc,+ Directory tempDir,+ String tempFileNamePrefix,+ BKDConfig config,+ double maxMBSortInHeap,+ long totalPointCount,+ int version) {+ if (version < VERSION_START || version > VERSION_CURRENT) {+ throw new IllegalArgumentException("Version out of range: " + version);+ }+ this.version = version;verifyParams(maxMBSortInHeap, totalPointCount);// We use tracking dir to deal with removing files on exception, so each place that// creates temp files doesn't need crazy try/finally/sucess logic:@@ -165,7 +190,7 @@ public class BKDWriter implements Closeable {// Maximum number of points we hold in memory at any timemaxPointsSortInHeap = (int) ((maxMBSortInHeap * 1024 * 1024) / (config.bytesPerDoc()));- docIdsWriter = new DocIdsWriter(config.maxPointsInLeafNode());+ docIdsWriter = new DocIdsWriter(config.maxPointsInLeafNode(), version);// Finally, we must be able to hold at least the leaf node in heap during build:if (maxPointsSortInHeap < config.maxPointsInLeafNode()) {throw new IllegalArgumentException(@@ -1245,7 +1270,7 @@ public class BKDWriter implements Closeable {byte[] packedIndex,long dataStartFP)throws IOException {- CodecUtil.writeHeader(metaOut, CODEC_NAME, VERSION_CURRENT);+ CodecUtil.writeHeader(metaOut, CODEC_NAME, version);metaOut.writeVInt(config.numDims());metaOut.writeVInt(config.numIndexDims());metaOut.writeVInt(countPerLeaf);diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.javaindex d188e1881d5..bcb7d964868 100644--- a/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java@@ -52,13 +52,16 @@ final class DocIdsWriter {*/private final IntsRef scratchIntsRef = new IntsRef();+ private final int version;+{// This is here to not rely on the default constructor of IntsRef to set offset to 0scratchIntsRef.offset = 0;}- DocIdsWriter(int maxPointsInLeaf) {- scratch = new int[maxPointsInLeaf];+ DocIdsWriter(int maxPointsInLeaf, int version) {+ this.scratch = new int[maxPointsInLeaf];+ this.version = version;}void writeDocIds(int[] docIds, int start, int count, DataOutput out) throws IOException {@@ -101,7 +104,7 @@ final class DocIdsWriter {scratch[i] = docIds[start + i] - min;}out.writeVInt(min);- final int halfLen = count >>> 1;+ final int halfLen = count >> 1;for (int i = 0; i < halfLen; ++i) {scratch[i] = scratch[halfLen + i] | (scratch[i] << 16);}@@ -114,31 +117,28 @@ final class DocIdsWriter {} else {if (max <= 0xFFFFFF) {out.writeByte(BPV_24);- // write them the same way we are reading them.- int i;- for (i = 0; i < count - 7; i += 8) {- int doc1 = docIds[start + i];- int doc2 = docIds[start + i + 1];- int doc3 = docIds[start + i + 2];- int doc4 = docIds[start + i + 3];- int doc5 = docIds[start + i + 4];- int doc6 = docIds[start + i + 5];- int doc7 = docIds[start + i + 6];- int doc8 = docIds[start + i + 7];- long l1 = (doc1 & 0xffffffL) << 40 | (doc2 & 0xffffffL) << 16 | ((doc3 >>> 8) & 0xffffL);- long l2 =- (doc3 & 0xffL) << 56- | (doc4 & 0xffffffL) << 32- | (doc5 & 0xffffffL) << 8- | ((doc6 >> 16) & 0xffL);- long l3 = (doc6 & 0xffffL) << 48 | (doc7 & 0xffffffL) << 24 | (doc8 & 0xffffffL);- out.writeLong(l1);- out.writeLong(l2);- out.writeLong(l3);- }- for (; i < count; ++i) {- out.writeShort((short) (docIds[start + i] >>> 8));- out.writeByte((byte) docIds[start + i]);+ if (version < BKDWriter.VERSION_VECTORIZED_DOCID) {+ writeScalarInts24(docIds, start, count, out);+ } else {+ // encode the docs in the format that can be vectorized decoded.+ final int quarter = count >> 2;+ final int numInts = quarter * 3;+ for (int i = 0; i < numInts; i++) {+ scratch[i] = docIds[i + start] << 8;+ }+ for (int i = 0; i < quarter; i++) {+ final int longIdx = i + numInts + start;+ scratch[i] |= docIds[longIdx] & 0xFF;+ scratch[i + quarter] |= (docIds[longIdx] >>> 8) & 0xFF;+ scratch[i + quarter * 2] |= docIds[longIdx] >>> 16;+ }+ for (int i = 0; i < numInts; i++) {+ out.writeInt(scratch[i]);+ }+ for (int i = quarter << 2; i < count; ++i) {+ out.writeShort((short) docIds[start + i]);+ out.writeByte((byte) (docIds[start + i] >>> 16));+ }}} else {out.writeByte(BPV_32);@@ -149,6 +149,35 @@ final class DocIdsWriter {}}+ private static void writeScalarInts24(int[] docIds, int start, int count, DataOutput out)+ throws IOException {+ int i;+ for (i = 0; i < count - 7; i += 8) {+ int doc1 = docIds[start + i];+ int doc2 = docIds[start + i + 1];+ int doc3 = docIds[start + i + 2];+ int doc4 = docIds[start + i + 3];+ int doc5 = docIds[start + i + 4];+ int doc6 = docIds[start + i + 5];+ int doc7 = docIds[start + i + 6];+ int doc8 = docIds[start + i + 7];+ long l1 = (doc1 & 0xffffffL) << 40 | (doc2 & 0xffffffL) << 16 | ((doc3 >>> 8) & 0xffffL);+ long l2 =+ (doc3 & 0xffL) << 56+ | (doc4 & 0xffffffL) << 32+ | (doc5 & 0xffffffL) << 8+ | ((doc6 >> 16) & 0xffL);+ long l3 = (doc6 & 0xffffL) << 48 | (doc7 & 0xffffffL) << 24 | (doc8 & 0xffffffL);+ out.writeLong(l1);+ out.writeLong(l2);+ out.writeLong(l3);+ }+ for (; i < count; ++i) {+ out.writeShort((short) (docIds[start + i] >>> 8));+ out.writeByte((byte) docIds[start + i]);+ }+ }+private static void writeIdsAsBitSet(int[] docIds, int start, int count, DataOutput out)throws IOException {int min = docIds[start];@@ -196,7 +225,11 @@ final class DocIdsWriter {readDelta16(in, count, docIDs);break;case BPV_24:- readInts24(in, count, docIDs);+ if (version < BKDWriter.VERSION_VECTORIZED_DOCID) {+ readScalarInts24(in, count, docIDs);+ } else {+ readInts24(in, count, docIDs);+ }break;case BPV_32:readInts32(in, count, docIDs);@@ -248,21 +281,65 @@ final class DocIdsWriter {assert pos == count : "pos: " + pos + ", count: " + count;}- private static void readDelta16(IndexInput in, int count, int[] docIDs) throws IOException {+ private static void readDelta16(IndexInput in, int count, int[] docIds) throws IOException {final int min = in.readVInt();- final int halfLen = count >>> 1;- in.readInts(docIDs, 0, halfLen);- for (int i = 0; i < halfLen; ++i) {- int l = docIDs[i];+ final int half = count >> 1;+ in.readInts(docIds, 0, half);+ if (count == BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE) {+ // Same format, but enabling the JVM to specialize the decoding logic for the default number+ // of points per node proved to help on benchmarks+ decode16(docIds, BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE / 2, min);+ } else {+ decode16(docIds, half, min);+ }+ // read the remaining doc if count is odd.+ for (int i = half << 1; i < count; i++) {+ docIds[i] = Short.toUnsignedInt(in.readShort()) + min;+ }+ }++ private static void decode16(int[] docIDs, int half, int min) {+ for (int i = 0; i < half; ++i) {+ final int l = docIDs[i];docIDs[i] = (l >>> 16) + min;- docIDs[halfLen + i] = (l & 0xFFFF) + min;+ docIDs[i + half] = (l & 0xFFFF) + min;}- if ((count & 1) == 1) {- docIDs[count - 1] = Short.toUnsignedInt(in.readShort()) + min;+ }++ private void readInts24(IndexInput in, int count, int[] docIDs) throws IOException {+ int quarter = count >> 2;+ int numInts = quarter * 3;+ in.readInts(scratch, 0, numInts);+ if (count == BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE) {+ // Same format, but enabling the JVM to specialize the decoding logic for the default number+ // of points per node proved to help on benchmarks+ decode24(+ docIDs,+ scratch,+ BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE / 4,+ BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE / 4 * 3);+ } else {+ decode24(docIDs, scratch, quarter, numInts);+ }+ // Now read the remaining 0, 1, 2 or 3 values+ for (int i = quarter << 2; i < count; ++i) {+ docIDs[i] = (in.readShort() & 0xFFFF) | (in.readByte() & 0xFF) << 16;+ }+ }++ private static void decode24(int[] docIDs, int[] scratch, int quarter, int numInts) {+ for (int i = 0; i < numInts; ++i) {+ docIDs[i] = scratch[i] >>> 8;+ }+ for (int i = 0; i < quarter; i++) {+ docIDs[i + numInts] =+ (scratch[i] & 0xFF)+ | ((scratch[i + quarter] & 0xFF) << 8)+ | ((scratch[i + quarter * 2] & 0xFF) << 16);}}- private static void readInts24(IndexInput in, int count, int[] docIDs) throws IOException {+ private static void readScalarInts24(IndexInput in, int count, int[] docIDs) throws IOException {int i;for (i = 0; i < count - 7; i += 8) {long l1 = in.readLong();@@ -290,7 +367,8 @@ final class DocIdsWriter {* Read {@code count} integers and feed the result directly to {@link* IntersectVisitor#visit(int)}.*/- void readInts(IndexInput in, int count, IntersectVisitor visitor) throws IOException {+ void readInts(IndexInput in, int count, IntersectVisitor visitor, int[] buffer)+ throws IOException {final int bpv = in.readByte();switch (bpv) {case CONTINUOUS_IDS:@@ -303,7 +381,11 @@ final class DocIdsWriter {readDelta16(in, count, visitor);break;case BPV_24:- readInts24(in, count, visitor);+ if (version < BKDWriter.VERSION_VECTORIZED_DOCID) {+ readScalarInts24(in, count, visitor);+ } else {+ readInts24(in, count, visitor, buffer);+ }break;case BPV_32:readInts32(in, count, visitor);@@ -343,8 +425,17 @@ final class DocIdsWriter {visitor.visit(scratchIntsRef);}- private void readInts24(IndexInput in, int count, IntersectVisitor visitor) throws IOException {- readInts24(in, count, scratch);+ private void readInts24(IndexInput in, int count, IntersectVisitor visitor, int[] buffer)+ throws IOException {+ readInts24(in, count, buffer);+ scratchIntsRef.ints = buffer;+ scratchIntsRef.length = count;+ visitor.visit(scratchIntsRef);+ }++ private void readScalarInts24(IndexInput in, int count, IntersectVisitor visitor)+ throws IOException {+ readScalarInts24(in, count, scratch);scratchIntsRef.ints = scratch;scratchIntsRef.length = count;visitor.visit(scratchIntsRef);diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PointsFormatV0.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PointsFormatV0.javanew file mode 100644index 00000000000..fd27eef0488--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PointsFormatV0.java@@ -0,0 +1,35 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene90;++import org.apache.lucene.codecs.Codec;+import org.apache.lucene.codecs.PointsFormat;+import org.apache.lucene.tests.codecs.asserting.AssertingCodec;+import org.apache.lucene.tests.index.BasePointsFormatTestCase;++public class TestLucene90PointsFormatV0 extends BasePointsFormatTestCase {++ @Override+ protected Codec getCodec() {+ return new AssertingCodec() {+ @Override+ public PointsFormat pointsFormat() {+ return new Lucene90PointsFormat(Lucene90PointsFormat.VERSION_START);+ }+ };+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.javaindex 086eb460e56..e0ccea1eea0 100644--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java@@ -36,11 +36,18 @@ import org.apache.lucene.util.Constants;public class TestDocIdsWriter extends LuceneTestCase {+ private static final int[] VERSIONS =+ new int[] {BKDWriter.VERSION_META_FILE, BKDWriter.VERSION_CURRENT};+public void testRandom() throws Exception {int numIters = atLeast(100);try (Directory dir = newDirectory()) {for (int iter = 0; iter < numIters; ++iter) {- int[] docIDs = new int[1 + random().nextInt(5000)];+ int count =+ random().nextBoolean()+ ? 1 + random().nextInt(5000)+ : BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE;+ int[] docIDs = new int[count];final int bpv = TestUtil.nextInt(random(), 1, 32);for (int i = 0; i < docIDs.length; ++i) {docIDs[i] = TestUtil.nextInt(random(), 0, (1 << bpv) - 1);@@ -69,7 +76,11 @@ public class TestDocIdsWriter extends LuceneTestCase {int numIters = atLeast(100);try (Directory dir = newDirectory()) {for (int iter = 0; iter < numIters; ++iter) {- int[] docIDs = new int[1 + random().nextInt(5000)];+ int count =+ random().nextBoolean()+ ? 1 + random().nextInt(5000)+ : BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE;+ int[] docIDs = new int[count];int min = random().nextInt(1000);final int bpv = TestUtil.nextInt(random(), 1, 16);for (int i = 0; i < docIDs.length; ++i) {@@ -113,7 +124,9 @@ public class TestDocIdsWriter extends LuceneTestCase {private void test(Directory dir, int[] ints) throws Exception {final long len;- DocIdsWriter docIdsWriter = new DocIdsWriter(ints.length);+ // It is hard to get BPV24-encoded docs in TextLuceneXXPointsFormat, test bwc here as well.+ final int version = VERSIONS[random().nextInt(VERSIONS.length)];+ DocIdsWriter docIdsWriter = new DocIdsWriter(ints.length, version);try (IndexOutput out = dir.createOutput("tmp", IOContext.DEFAULT)) {docIdsWriter.writeDocIds(ints, 0, ints.length, out);len = out.getFilePointer();@@ -149,7 +162,8 @@ public class TestDocIdsWriter extends LuceneTestCase {public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {throw new UnsupportedOperationException();}- });+ },+ new int[ints.length]);assertArrayEquals(ints, read);assertEquals(len, in.getFilePointer());}