Apache Lucene open-source search software
Hide the internal data structure of HeapPointWriter (#12762)
Ignacio Vera74085cd1b083
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 8c85e5bce33..93bdc076b4d 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -363,6 +363,8 @@ Otheroverflows and slices that are too large. Some bits of code are simplified. Documentation is updated and expanded.(Stefan Vodita)+* GITHUB#12762: Refactor BKD HeapPointWriter to hide the internal data structure. (Ignacio Vera)+======================== Lucene 9.8.0 =======================API Changesdiff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDRadixSelector.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDRadixSelector.javaindex 662f21b6c86..62586641e2c 100644--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDRadixSelector.java+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDRadixSelector.java@@ -19,8 +19,6 @@ package org.apache.lucene.util.bkd;import java.io.IOException;import java.util.Arrays;import org.apache.lucene.store.Directory;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.ArrayUtil.ByteArrayComparator;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.IntroSelector;import org.apache.lucene.util.IntroSorter;@@ -181,12 +179,8 @@ public final class BKDRadixSelector {break;} else {// Check common prefix and adjust histogram- final int startIndex =- (dimCommonPrefix > config.bytesPerDim) ? config.bytesPerDim : dimCommonPrefix;- final int endIndex =- (commonPrefixPosition > config.bytesPerDim)- ? config.bytesPerDim- : commonPrefixPosition;+ final int startIndex = Math.min(dimCommonPrefix, config.bytesPerDim);+ final int endIndex = Math.min(commonPrefixPosition, config.bytesPerDim);packedValueDocID = pointValue.packedValueDocIDBytes();int j =Arrays.mismatch(@@ -427,24 +421,13 @@ public final class BKDRadixSelector {@Overrideprotected int byteAt(int i, int k) {assert k >= 0 : "negative prefix " + k;- if (k < dimCmpBytes) {- // dim bytes- return points.block[i * config.bytesPerDoc + dimOffset + k] & 0xff;- } else {- // data bytes- return points.block[i * config.bytesPerDoc + dataOffset + k] & 0xff;- }+ return points.byteAt(i, k < dimCmpBytes ? dimOffset + k : dataOffset + k);}@Overrideprotected Selector getFallbackSelector(int d) {final int skypedBytes = d + commonPrefixLength;final int dimStart = dim * config.bytesPerDim;- // data length is composed by the data dimensions plus the docID- final int dataLength =- (config.numDims - config.numIndexDims) * config.bytesPerDim + Integer.BYTES;- final ByteArrayComparator dimComparator =- ArrayUtil.getUnsignedComparator(config.bytesPerDim);return new IntroSelector() {@Override@@ -455,61 +438,31 @@ public final class BKDRadixSelector {@Overrideprotected void setPivot(int i) {if (skypedBytes < config.bytesPerDim) {- System.arraycopy(- points.block,- i * config.bytesPerDoc + dim * config.bytesPerDim,- scratch,- 0,- config.bytesPerDim);+ points.copyDim(i, dimStart, scratch, 0);}- System.arraycopy(- points.block,- i * config.bytesPerDoc + config.packedIndexBytesLength,- scratch,- config.bytesPerDim,- dataLength);+ points.copyDataDimsAndDoc(i, scratch, config.bytesPerDim);}@Overrideprotected int compare(int i, int j) {if (skypedBytes < config.bytesPerDim) {- int iOffset = i * config.bytesPerDoc;- int jOffset = j * config.bytesPerDoc;- int cmp =- dimComparator.compare(- points.block, iOffset + dimStart, points.block, jOffset + dimStart);+ int cmp = points.compareDim(i, j, dimStart);if (cmp != 0) {return cmp;}}- int iOffset = i * config.bytesPerDoc + config.packedIndexBytesLength;- int jOffset = j * config.bytesPerDoc + config.packedIndexBytesLength;- return Arrays.compareUnsigned(- points.block,- iOffset,- iOffset + dataLength,- points.block,- jOffset,- jOffset + dataLength);+ return points.compareDataDimsAndDoc(i, j);}@Overrideprotected int comparePivot(int j) {if (skypedBytes < config.bytesPerDim) {- int jOffset = j * config.bytesPerDoc;- int cmp = dimComparator.compare(scratch, 0, points.block, jOffset + dimStart);+ int cmp = points.compareDim(j, scratch, 0, dimStart);if (cmp != 0) {return cmp;}}- int jOffset = j * config.bytesPerDoc + config.packedIndexBytesLength;- return Arrays.compareUnsigned(- scratch,- config.bytesPerDim,- config.bytesPerDim + dataLength,- points.block,- jOffset,- jOffset + dataLength);+ return points.compareDataDimsAndDoc(j, scratch, config.bytesPerDim);}};}@@ -538,13 +491,7 @@ public final class BKDRadixSelector {@Overrideprotected int byteAt(int i, int k) {assert k >= 0 : "negative prefix " + k;- if (k < dimCmpBytes) {- // dim bytes- return points.block[i * config.bytesPerDoc + dimOffset + k] & 0xff;- } else {- // data bytes- return points.block[i * config.bytesPerDoc + dataOffset + k] & 0xff;- }+ return points.byteAt(i, k < dimCmpBytes ? dimOffset + k : dataOffset + k);}@Override@@ -556,11 +503,6 @@ public final class BKDRadixSelector {protected Sorter getFallbackSorter(int k) {final int skypedBytes = k + commonPrefixLength;final int dimStart = dim * config.bytesPerDim;- // data length is composed by the data dimensions plus the docID- final int dataLength =- (config.numDims - config.numIndexDims) * config.bytesPerDim + Integer.BYTES;- final ByteArrayComparator dimComparator =- ArrayUtil.getUnsignedComparator(config.bytesPerDim);return new IntroSorter() {@Override@@ -571,61 +513,31 @@ public final class BKDRadixSelector {@Overrideprotected void setPivot(int i) {if (skypedBytes < config.bytesPerDim) {- System.arraycopy(- points.block,- i * config.bytesPerDoc + dim * config.bytesPerDim,- scratch,- 0,- config.bytesPerDim);+ points.copyDim(i, dimStart, scratch, 0);}- System.arraycopy(- points.block,- i * config.bytesPerDoc + config.packedIndexBytesLength,- scratch,- config.bytesPerDim,- dataLength);+ points.copyDataDimsAndDoc(i, scratch, config.bytesPerDim);}@Overrideprotected int compare(int i, int j) {if (skypedBytes < config.bytesPerDim) {- int iOffset = i * config.bytesPerDoc;- int jOffset = j * config.bytesPerDoc;- int cmp =- dimComparator.compare(- points.block, iOffset + dimStart, points.block, jOffset + dimStart);+ final int cmp = points.compareDim(i, j, dimStart);if (cmp != 0) {return cmp;}}- int iOffset = i * config.bytesPerDoc + config.packedIndexBytesLength;- int jOffset = j * config.bytesPerDoc + config.packedIndexBytesLength;- return Arrays.compareUnsigned(- points.block,- iOffset,- iOffset + dataLength,- points.block,- jOffset,- jOffset + dataLength);+ return points.compareDataDimsAndDoc(i, j);}@Overrideprotected int comparePivot(int j) {if (skypedBytes < config.bytesPerDim) {- int jOffset = j * config.bytesPerDoc;- int cmp = dimComparator.compare(scratch, 0, points.block, jOffset + dimStart);+ int cmp = points.compareDim(j, scratch, 0, dimStart);if (cmp != 0) {return cmp;}}- int jOffset = j * config.bytesPerDoc + config.packedIndexBytesLength;- return Arrays.compareUnsigned(- scratch,- config.bytesPerDim,- config.bytesPerDim + dataLength,- points.block,- jOffset,- jOffset + dataLength);+ return points.compareDataDimsAndDoc(j, scratch, config.bytesPerDim);}};}diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.javaindex 093b6dd5d54..2b79ad704be 100644--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java@@ -16,8 +16,7 @@*/package org.apache.lucene.util.bkd;-import org.apache.lucene.util.BitUtil;-import org.apache.lucene.util.BytesRef;+import java.util.function.IntFunction;/*** Utility class to read buffered points from in-heap arrays.@@ -26,22 +25,13 @@ import org.apache.lucene.util.BytesRef;*/public final class HeapPointReader implements PointReader {private int curRead;- final byte[] block;- final BKDConfig config;- final int end;- private final HeapPointValue pointValue;+ private final int end;+ private final IntFunction<PointValue> points;- public HeapPointReader(BKDConfig config, byte[] block, int start, int end) {- this.block = block;+ HeapPointReader(IntFunction<PointValue> points, int start, int end) {curRead = start - 1;this.end = end;- this.config = config;- if (start < end) {- this.pointValue = new HeapPointValue(config, block);- } else {- // no values- this.pointValue = null;- }+ this.points = points;}@Override@@ -52,46 +42,9 @@ public final class HeapPointReader implements PointReader {@Overridepublic PointValue pointValue() {- pointValue.setOffset(curRead * config.bytesPerDoc);- return pointValue;+ return points.apply(curRead);}@Overridepublic void close() {}-- /** Reusable implementation for a point value on-heap */- static class HeapPointValue implements PointValue {-- final BytesRef packedValue;- final BytesRef packedValueDocID;- final int packedValueLength;-- HeapPointValue(BKDConfig config, byte[] value) {- this.packedValueLength = config.packedBytesLength;- this.packedValue = new BytesRef(value, 0, packedValueLength);- this.packedValueDocID = new BytesRef(value, 0, config.bytesPerDoc);- }-- /** Sets a new value by changing the offset. */- public void setOffset(int offset) {- packedValue.offset = offset;- packedValueDocID.offset = offset;- }-- @Override- public BytesRef packedValue() {- return packedValue;- }-- @Override- public int docID() {- int position = packedValueDocID.offset + packedValueLength;- return (int) BitUtil.VH_BE_INT.get(packedValueDocID.bytes, position);- }-- @Override- public BytesRef packedValueDocIDBytes() {- return packedValueDocID;- }- }}diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.javaindex 117ce6f80f1..5a918d166d8 100644--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java@@ -17,6 +17,7 @@package org.apache.lucene.util.bkd;import java.util.Arrays;+import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BitUtil;import org.apache.lucene.util.BytesRef;@@ -26,22 +27,26 @@ import org.apache.lucene.util.BytesRef;* @lucene.internal*/public final class HeapPointWriter implements PointWriter {- public final byte[] block;+ private final byte[] block;final int size;- final BKDConfig config;+ private final BKDConfig config;private final byte[] scratch;+ private final ArrayUtil.ByteArrayComparator dimComparator;+ // length is composed by the data dimensions plus the docID+ private final int dataDimsAndDocLength;private int nextWrite;private boolean closed;-- private HeapPointReader.HeapPointValue pointValue;+ private final HeapPointValue pointValue;public HeapPointWriter(BKDConfig config, int size) {this.config = config;this.block = new byte[config.bytesPerDoc * size];this.size = size;+ this.dimComparator = ArrayUtil.getUnsignedComparator(config.bytesPerDim);+ this.dataDimsAndDocLength = config.bytesPerDoc - config.packedIndexBytesLength;this.scratch = new byte[config.bytesPerDoc];if (size > 0) {- pointValue = new HeapPointReader.HeapPointValue(config, block);+ pointValue = new HeapPointValue(config, block);} else {// no valuespointValue = null;@@ -65,10 +70,9 @@ public final class HeapPointWriter implements PointWriter {+ packedValue.length+ "]";assert nextWrite < size : "nextWrite=" + (nextWrite + 1) + " vs size=" + size;- System.arraycopy(- packedValue, 0, block, nextWrite * config.bytesPerDoc, config.packedBytesLength);- int position = nextWrite * config.bytesPerDoc + config.packedBytesLength;- BitUtil.VH_BE_INT.set(block, position, docID);+ final int position = nextWrite * config.bytesPerDoc;+ System.arraycopy(packedValue, 0, block, position, config.packedBytesLength);+ BitUtil.VH_BE_INT.set(block, position + config.packedBytesLength, docID);nextWrite++;}@@ -76,27 +80,23 @@ public final class HeapPointWriter implements PointWriter {public void append(PointValue pointValue) {assert closed == false : "point writer is already closed";assert nextWrite < size : "nextWrite=" + (nextWrite + 1) + " vs size=" + size;- BytesRef packedValueDocID = pointValue.packedValueDocIDBytes();+ final BytesRef packedValueDocID = pointValue.packedValueDocIDBytes();assert packedValueDocID.length == config.bytesPerDoc: "[packedValue] must have length ["+ (config.bytesPerDoc)+ "] but was ["+ packedValueDocID.length+ "]";+ final int position = nextWrite * config.bytesPerDoc;System.arraycopy(- packedValueDocID.bytes,- packedValueDocID.offset,- block,- nextWrite * config.bytesPerDoc,- config.bytesPerDoc);+ packedValueDocID.bytes, packedValueDocID.offset, block, position, config.bytesPerDoc);nextWrite++;}- public void swap(int i, int j) {-- int indexI = i * config.bytesPerDoc;- int indexJ = j * config.bytesPerDoc;-+ /** Swaps the point at point {@code i} with the point at position {@code j} */+ void swap(int i, int j) {+ final int indexI = i * config.bytesPerDoc;+ final int indexJ = j * config.bytesPerDoc;// scratch1 = values[i]System.arraycopy(block, indexI, scratch, 0, config.bytesPerDoc);// values[i] = values[j]@@ -105,19 +105,100 @@ public final class HeapPointWriter implements PointWriter {System.arraycopy(scratch, 0, block, indexJ, config.bytesPerDoc);}+ /** Return the byte at position {@code k} of the point at position {@code i} */+ int byteAt(int i, int k) {+ return block[i * config.bytesPerDoc + k] & 0xff;+ }++ /**+ * Copy the dimension {@code dim} of the point at position {@code i} in the provided {@code bytes}+ * at the given offset+ */+ void copyDim(int i, int dim, byte[] bytes, int offset) {+ System.arraycopy(block, i * config.bytesPerDoc + dim, bytes, offset, config.bytesPerDim);+ }++ /**+ * Copy the data dimensions and doc value of the point at position {@code i} in the provided+ * {@code bytes} at the given offset+ */+ void copyDataDimsAndDoc(int i, byte[] bytes, int offset) {+ System.arraycopy(+ block,+ i * config.bytesPerDoc + config.packedIndexBytesLength,+ bytes,+ offset,+ dataDimsAndDocLength);+ }++ /**+ * Compares the dimension {@code dim} value of the point at position {@code i} with the point at+ * position {@code j}+ */+ int compareDim(int i, int j, int dim) {+ final int iOffset = i * config.bytesPerDoc + dim;+ final int jOffset = j * config.bytesPerDoc + dim;+ return compareDim(block, iOffset, block, jOffset);+ }++ /**+ * Compares the dimension {@code dim} value of the point at position {@code j} with the provided+ * value+ */+ int compareDim(int j, byte[] dimValue, int offset, int dim) {+ final int jOffset = j * config.bytesPerDoc + dim;+ return compareDim(dimValue, offset, block, jOffset);+ }++ private int compareDim(byte[] blockI, int offsetI, byte[] blockJ, int offsetJ) {+ return dimComparator.compare(blockI, offsetI, blockJ, offsetJ);+ }++ /**+ * Compares the data dimensions and doc values of the point at position {@code i} with the point+ * at position {@code j}+ */+ int compareDataDimsAndDoc(int i, int j) {+ final int iOffset = i * config.bytesPerDoc + config.packedIndexBytesLength;+ final int jOffset = j * config.bytesPerDoc + config.packedIndexBytesLength;+ return compareDataDimsAndDoc(block, iOffset, block, jOffset);+ }++ /**+ * Compares the data dimensions and doc values of the point at position {@code j} with the+ * provided value+ */+ int compareDataDimsAndDoc(int j, byte[] dataDimsAndDocs, int offset) {+ final int jOffset = j * config.bytesPerDoc + config.packedIndexBytesLength;+ return compareDataDimsAndDoc(dataDimsAndDocs, offset, block, jOffset);+ }++ private int compareDataDimsAndDoc(byte[] blockI, int offsetI, byte[] blockJ, int offsetJ) {+ return Arrays.compareUnsigned(+ blockI,+ offsetI,+ offsetI + dataDimsAndDocLength,+ blockJ,+ offsetJ,+ offsetJ + dataDimsAndDocLength);+ }++ /** Computes the cardinality of the points between {@code from} tp {@code to} */public int computeCardinality(int from, int to, int[] commonPrefixLengths) {int leafCardinality = 1;for (int i = from + 1; i < to; i++) {+ final int pointOffset = (i - 1) * config.bytesPerDoc;+ final int nextPointOffset = pointOffset + config.bytesPerDoc;for (int dim = 0; dim < config.numDims; dim++) {final int start = dim * config.bytesPerDim + commonPrefixLengths[dim];final int end = dim * config.bytesPerDim + config.bytesPerDim;if (Arrays.mismatch(block,- i * config.bytesPerDoc + start,- i * config.bytesPerDoc + end,+ nextPointOffset + start,+ nextPointOffset + end,block,- (i - 1) * config.bytesPerDoc + start,- (i - 1) * config.bytesPerDoc + end)+ pointOffset + start,+ pointOffset + end)!= -1) {leafCardinality++;break;@@ -139,7 +220,8 @@ public final class HeapPointWriter implements PointWriter {: "start=" + start + " length=" + length + " docIDs.length=" + size;assert start + length <= nextWrite: "start=" + start + " length=" + length + " nextWrite=" + nextWrite;- return new HeapPointReader(config, block, (int) start, Math.toIntExact(start + length));+ return new HeapPointReader(+ this::getPackedValueSlice, (int) start, Math.toIntExact(start + length));}@Override@@ -154,4 +236,40 @@ public final class HeapPointWriter implements PointWriter {public String toString() {return "HeapPointWriter(count=" + nextWrite + " size=" + size + ")";}++ /** Reusable implementation for a point value on-heap */+ private static class HeapPointValue implements PointValue {++ private final BytesRef packedValue;+ private final BytesRef packedValueDocID;+ private final int packedValueLength;++ HeapPointValue(BKDConfig config, byte[] value) {+ this.packedValueLength = config.packedBytesLength;+ this.packedValue = new BytesRef(value, 0, packedValueLength);+ this.packedValueDocID = new BytesRef(value, 0, config.bytesPerDoc);+ }++ /** Sets a new value by changing the offset. */+ void setOffset(int offset) {+ packedValue.offset = offset;+ packedValueDocID.offset = offset;+ }++ @Override+ public BytesRef packedValue() {+ return packedValue;+ }++ @Override+ public int docID() {+ int position = packedValueDocID.offset + packedValueLength;+ return (int) BitUtil.VH_BE_INT.get(packedValueDocID.bytes, position);+ }++ @Override+ public BytesRef packedValueDocIDBytes() {+ return packedValueDocID;+ }+ }}