Skip to content

Apache Lucene open-source search software

Pack file pointers when buikding BKD trees (#14393)

Ignacio Vera510ce5fc87ff
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d0e0cd6474e..ee3304272cc 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -200,6 +200,8 @@ Optimizations
* GITHUB#14373: Optimized `ParallelLeafReader` to improve term vector fetching efficiency. (Divyansh Agrawal)
+* GITHUB#14393: Pack file pointers using the packed monotonic builder when building BKD trees. (Ignacio Vera)
+
Bug Fixes
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 3fb80e5c303..22b3ca90225 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -28,7 +28,6 @@ import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.internal.hppc.LongArrayList;
import org.apache.lucene.store.ByteBuffersDataOutput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataOutput;
@@ -43,9 +42,12 @@ import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedLengthBytesRefArray;
import org.apache.lucene.util.IORunnable;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LongValues;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.bkd.BKDUtil.ByteArrayPredicate;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PackedLongValues;
// TODO
// - allow variable length byte[] (across docs and dims), but this is quite a bit more hairy
@@ -582,8 +584,21 @@ public class BKDWriter implements Closeable {
scratchBytesRef1.length = config.bytesPerDim();
scratchBytesRef1.bytes = splitPackedValues;
+ final LongValues leafFPLongValues =
+ new LongValues() {
+ @Override
+ public long get(long index) {
+ return leafBlockFPs[(int) index];
+ }
+ };
- return makeWriter(metaOut, indexOut, splitDimensionValues, leafBlockFPs, dataStartFP);
+ return makeWriter(
+ metaOut,
+ indexOut,
+ splitDimensionValues,
+ leafFPLongValues,
+ leafBlockFPs.length,
+ dataStartFP);
}
/* In the 1D case, we can simply sort points in ascending order and use the
@@ -678,7 +693,8 @@ public class BKDWriter implements Closeable {
final IndexOutput metaOut, indexOut, dataOut;
final long dataStartFP;
- final LongArrayList leafBlockFPs = new LongArrayList();
+ private final PackedLongValues.Builder leafBlockFPs =
+ PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
final FixedLengthBytesRefArray leafBlockStartValues =
new FixedLengthBytesRefArray(config.packedIndexBytesLength());
final byte[] leafValues = new byte[config.maxPointsInLeafNode() * config.packedBytesLength()];
@@ -708,7 +724,6 @@ public class BKDWriter implements Closeable {
this.indexOut = indexOut;
this.dataOut = dataOut;
this.dataStartFP = dataOut.getFilePointer();
-
lastPackedValue = new byte[config.packedBytesLength()];
}
@@ -773,11 +788,12 @@ public class BKDWriter implements Closeable {
scratchBytesRef1.length = config.packedIndexBytesLength();
scratchBytesRef1.offset = 0;
assert leafBlockStartValues.size() + 1 == leafBlockFPs.size();
+ final LongValues leafFPLongValues = leafBlockFPs.build();
BKDTreeLeafNodes leafNodes =
new BKDTreeLeafNodes() {
@Override
public long getLeafLP(int index) {
- return leafBlockFPs.get(index);
+ return leafFPLongValues.get(index);
}
@Override
@@ -792,7 +808,7 @@ public class BKDWriter implements Closeable {
@Override
public int numLeaves() {
- return leafBlockFPs.size();
+ return Math.toIntExact(leafBlockFPs.size());
}
};
return () -> {
@@ -823,7 +839,7 @@ public class BKDWriter implements Closeable {
leafBlockStartValues.append(scratchBytesRef1);
}
leafBlockFPs.add(dataOut.getFilePointer());
- checkMaxLeafNodeCount(leafBlockFPs.size());
+ checkMaxLeafNodeCount(Math.toIntExact(leafBlockFPs.size()));
// Find per-dim common prefix:
commonPrefixLengths[0] =
@@ -955,7 +971,8 @@ public class BKDWriter implements Closeable {
// +1 because leaf count is power of 2 (e.g. 8), and innerNodeCount is power of 2 minus 1 (e.g.
// 7)
- long[] leafBlockFPs = new long[numLeaves];
+ final PackedLongValues.Builder leafBlockFPs =
+ PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
// Make sure the math above "worked":
assert pointCount / numLeaves <= config.maxPointsInLeafNode()
@@ -987,8 +1004,10 @@ public class BKDWriter implements Closeable {
splitPackedValues,
splitDimensionValues,
leafBlockFPs,
+ numLeaves,
new int[config.maxPointsInLeafNode()]);
assert Arrays.equals(parentSplits, new int[config.numIndexDims()]);
+ assert leafBlockFPs.size() == numLeaves;
// If no exception, we should have cleaned everything up:
assert tempDir.getCreatedFiles().isEmpty();
@@ -1002,22 +1021,30 @@ public class BKDWriter implements Closeable {
}
}
+ LongValues leafBlockLongValues = leafBlockFPs.build();
scratchBytesRef1.bytes = splitPackedValues;
scratchBytesRef1.length = config.bytesPerDim();
- return makeWriter(metaOut, indexOut, splitDimensionValues, leafBlockFPs, dataStartFP);
+ return makeWriter(
+ metaOut,
+ indexOut,
+ splitDimensionValues,
+ leafBlockLongValues,
+ Math.toIntExact(leafBlockFPs.size()),
+ dataStartFP);
}
private IORunnable makeWriter(
IndexOutput metaOut,
IndexOutput indexOut,
byte[] splitDimensionValues,
- long[] leafBlockFPs,
+ LongValues leafBlockFPs,
+ int numLeaves,
long dataStartFP) {
BKDTreeLeafNodes leafNodes =
new BKDTreeLeafNodes() {
@Override
public long getLeafLP(int index) {
- return leafBlockFPs[index];
+ return leafBlockFPs.get(index);
}
@Override
@@ -1033,7 +1060,7 @@ public class BKDWriter implements Closeable {
@Override
public int numLeaves() {
- return leafBlockFPs.length;
+ return numLeaves;
}
};
@@ -1903,7 +1930,8 @@ public class BKDWriter implements Closeable {
int[] parentSplits,
byte[] splitPackedValues,
byte[] splitDimensionValues,
- long[] leafBlockFPs,
+ PackedLongValues.Builder leafBlockFPs,
+ int totalNumLeaves,
int[] spareDocIds)
throws IOException {
@@ -1961,7 +1989,7 @@ public class BKDWriter implements Closeable {
int leafCardinality = heapSource.computeCardinality(from, to, commonPrefixLengths);
// Save the block file pointer:
- leafBlockFPs[leavesOffset] = out.getFilePointer();
+ leafBlockFPs.add(out.getFilePointer());
// System.out.println(" write leaf block @ fp=" + out.getFilePointer());
// Write docIDs first, as their own chunk, so that at intersect time we can add all docIDs w/o
@@ -2003,7 +2031,7 @@ public class BKDWriter implements Closeable {
// split dimensions. Because it is an expensive operation, the frequency we recompute the
// bounds is given
// by SPLITS_BEFORE_EXACT_BOUNDS.
- if (numLeaves != leafBlockFPs.length
+ if (numLeaves != totalNumLeaves
&& config.numIndexDims() > 2
&& Arrays.stream(parentSplits).sum() % SPLITS_BEFORE_EXACT_BOUNDS == 0) {
computePackedValueBounds(points, minPackedValue, maxPackedValue);
@@ -2011,8 +2039,8 @@ public class BKDWriter implements Closeable {
splitDim = split(minPackedValue, maxPackedValue, parentSplits);
}
- assert numLeaves <= leafBlockFPs.length
- : "numLeaves=" + numLeaves + " leafBlockFPs.length=" + leafBlockFPs.length;
+ assert numLeaves <= totalNumLeaves
+ : "numLeaves=" + numLeaves + " totalNumLeaves=" + totalNumLeaves;
// How many leaves will be in the left tree:
final int numLeftLeafNodes = getNumLeftLeafNodes(numLeaves);
@@ -2078,6 +2106,7 @@ public class BKDWriter implements Closeable {
splitPackedValues,
splitDimensionValues,
leafBlockFPs,
+ totalNumLeaves,
spareDocIds);
// Recurse on right tree:
@@ -2093,6 +2122,7 @@ public class BKDWriter implements Closeable {
splitPackedValues,
splitDimensionValues,
leafBlockFPs,
+ totalNumLeaves,
spareDocIds);
parentSplits[splitDim]--;