Apache Lucene open-source search software
Simplify bytes arrays in NumericLeafComparator to long (#13246)
gf2121203cf7b553ea
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex f96bd4c8b50..55fdf3687c2 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -119,6 +119,8 @@ New FeaturesImprovements---------------------+* GITHUB#13246: Simplify bytes comparison as long comparison in NumericComparator. (Guo feng)+* LUCENE-10416: Update Korean Dictionary to mecab-ko-dic-2.1.1-20180720 for Nori.(Uihyun Kim)diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/DoubleComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/DoubleComparator.javaindex 5cefdb3ebe0..938f793e22b 100644--- a/lucene/core/src/java/org/apache/lucene/search/comparators/DoubleComparator.java+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/DoubleComparator.java@@ -18,10 +18,10 @@package org.apache.lucene.search.comparators;import java.io.IOException;-import org.apache.lucene.document.DoublePoint;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.search.LeafFieldComparator;import org.apache.lucene.search.Pruning;+import org.apache.lucene.util.NumericUtils;/*** Comparator based on {@link Double#compare} for {@code numHits}. This comparator provides a@@ -54,6 +54,16 @@ public class DoubleComparator extends NumericComparator<Double> {return Double.valueOf(values[slot]);}+ @Override+ protected long missingValueAsComparableLong() {+ return NumericUtils.doubleToSortableLong(missingValue);+ }++ @Override+ protected long sortableBytesToLong(byte[] bytes) {+ return NumericUtils.sortableBytesToLong(bytes, 0);+ }+@Overridepublic LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {return new DoubleLeafComparator(context);@@ -97,23 +107,13 @@ public class DoubleComparator extends NumericComparator<Double> {}@Override- protected int compareMissingValueWithBottomValue() {- return Double.compare(missingValue, bottom);- }-- @Override- protected int compareMissingValueWithTopValue() {- return Double.compare(missingValue, topValue);- }-- @Override- protected void encodeBottom(byte[] packedValue) {- DoublePoint.encodeDimension(bottom, packedValue, 0);+ protected long bottomAsComparableLong() {+ return NumericUtils.doubleToSortableLong(bottom);}@Override- protected void encodeTop(byte[] packedValue) {- DoublePoint.encodeDimension(topValue, packedValue, 0);+ protected long topAsComparableLong() {+ return NumericUtils.doubleToSortableLong(topValue);}}}diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/FloatComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/FloatComparator.javaindex 7e58ca0faa2..7d57568f001 100644--- a/lucene/core/src/java/org/apache/lucene/search/comparators/FloatComparator.java+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/FloatComparator.java@@ -18,10 +18,10 @@package org.apache.lucene.search.comparators;import java.io.IOException;-import org.apache.lucene.document.FloatPoint;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.search.LeafFieldComparator;import org.apache.lucene.search.Pruning;+import org.apache.lucene.util.NumericUtils;/*** Comparator based on {@link Float#compare} for {@code numHits}. This comparator provides a@@ -54,6 +54,16 @@ public class FloatComparator extends NumericComparator<Float> {return Float.valueOf(values[slot]);}+ @Override+ protected long missingValueAsComparableLong() {+ return NumericUtils.floatToSortableInt(missingValue);+ }++ @Override+ protected long sortableBytesToLong(byte[] bytes) {+ return NumericUtils.sortableBytesToInt(bytes, 0);+ }+@Overridepublic LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {return new FloatLeafComparator(context);@@ -97,23 +107,13 @@ public class FloatComparator extends NumericComparator<Float> {}@Override- protected int compareMissingValueWithBottomValue() {- return Float.compare(missingValue, bottom);- }-- @Override- protected int compareMissingValueWithTopValue() {- return Float.compare(missingValue, topValue);- }-- @Override- protected void encodeBottom(byte[] packedValue) {- FloatPoint.encodeDimension(bottom, packedValue, 0);+ protected long bottomAsComparableLong() {+ return NumericUtils.floatToSortableInt(bottom);}@Override- protected void encodeTop(byte[] packedValue) {- FloatPoint.encodeDimension(topValue, packedValue, 0);+ protected long topAsComparableLong() {+ return NumericUtils.floatToSortableInt(topValue);}}}diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/IntComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/IntComparator.javaindex d88a858a0be..78c1fd93a92 100644--- a/lucene/core/src/java/org/apache/lucene/search/comparators/IntComparator.java+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/IntComparator.java@@ -18,10 +18,10 @@package org.apache.lucene.search.comparators;import java.io.IOException;-import org.apache.lucene.document.IntPoint;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.search.LeafFieldComparator;import org.apache.lucene.search.Pruning;+import org.apache.lucene.util.NumericUtils;/*** Comparator based on {@link Integer#compare} for {@code numHits}. This comparator provides a@@ -54,6 +54,16 @@ public class IntComparator extends NumericComparator<Integer> {return Integer.valueOf(values[slot]);}+ @Override+ protected long missingValueAsComparableLong() {+ return missingValue;+ }++ @Override+ protected long sortableBytesToLong(byte[] bytes) {+ return NumericUtils.sortableBytesToInt(bytes, 0);+ }+@Overridepublic LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {return new IntLeafComparator(context);@@ -97,23 +107,13 @@ public class IntComparator extends NumericComparator<Integer> {}@Override- protected int compareMissingValueWithBottomValue() {- return Integer.compare(missingValue, bottom);- }-- @Override- protected int compareMissingValueWithTopValue() {- return Integer.compare(missingValue, topValue);- }-- @Override- protected void encodeBottom(byte[] packedValue) {- IntPoint.encodeDimension(bottom, packedValue, 0);+ protected long bottomAsComparableLong() {+ return bottom;}@Override- protected void encodeTop(byte[] packedValue) {- IntPoint.encodeDimension(topValue, packedValue, 0);+ protected long topAsComparableLong() {+ return topValue;}}}diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/LongComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/LongComparator.javaindex 49c9011108d..20f5f930288 100644--- a/lucene/core/src/java/org/apache/lucene/search/comparators/LongComparator.java+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/LongComparator.java@@ -18,10 +18,10 @@package org.apache.lucene.search.comparators;import java.io.IOException;-import org.apache.lucene.document.LongPoint;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.search.LeafFieldComparator;import org.apache.lucene.search.Pruning;+import org.apache.lucene.util.NumericUtils;/*** Comparator based on {@link Long#compare} for {@code numHits}. This comparator provides a skipping@@ -54,6 +54,16 @@ public class LongComparator extends NumericComparator<Long> {return Long.valueOf(values[slot]);}+ @Override+ protected long missingValueAsComparableLong() {+ return missingValue;+ }++ @Override+ protected long sortableBytesToLong(byte[] bytes) {+ return NumericUtils.sortableBytesToLong(bytes, 0);+ }+@Overridepublic LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {return new LongLeafComparator(context);@@ -97,23 +107,13 @@ public class LongComparator extends NumericComparator<Long> {}@Override- protected int compareMissingValueWithBottomValue() {- return Long.compare(missingValue, bottom);- }-- @Override- protected int compareMissingValueWithTopValue() {- return Long.compare(missingValue, topValue);- }-- @Override- protected void encodeBottom(byte[] packedValue) {- LongPoint.encodeDimension(bottom, packedValue, 0);+ protected long bottomAsComparableLong() {+ return bottom;}@Override- protected void encodeTop(byte[] packedValue) {- LongPoint.encodeDimension(topValue, packedValue, 0);+ protected long topAsComparableLong() {+ return topValue;}}}diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.javaindex b52e611d45a..a0cd748f94c 100644--- a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java@@ -29,10 +29,7 @@ import org.apache.lucene.search.LeafFieldComparator;import org.apache.lucene.search.Pruning;import org.apache.lucene.search.Scorable;import org.apache.lucene.search.Scorer;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.ArrayUtil.ByteArrayComparator;import org.apache.lucene.util.DocIdSetBuilder;-import org.apache.lucene.util.NumericUtils;/*** Abstract numeric comparator for comparing numeric values. This comparator provides a skipping@@ -49,10 +46,10 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatoprivate static final int MIN_SKIP_INTERVAL = 32;private static final int MAX_SKIP_INTERVAL = 8192;protected final T missingValue;+ private final long missingValueAsLong;protected final String field;protected final boolean reverse;private final int bytesCount; // how many bytes are used to encode this number- private final ByteArrayComparator bytesComparator;protected boolean topValueSet;protected boolean singleSort; // singleSort is true, if sort is based on a single sort field.@@ -64,10 +61,10 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatoString field, T missingValue, boolean reverse, Pruning pruning, int bytesCount) {this.field = field;this.missingValue = missingValue;+ this.missingValueAsLong = missingValueAsComparableLong();this.reverse = reverse;this.pruning = pruning;this.bytesCount = bytesCount;- this.bytesComparator = ArrayUtil.getUnsignedComparator(bytesCount);}@Override@@ -85,6 +82,14 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatopruning = Pruning.NONE;}+ protected abstract long missingValueAsComparableLong();++ /**+ * Decode sortable bytes to long. It should be consistent with the codec that {@link PointValues}+ * of this field is using.+ */+ protected abstract long sortableBytesToLong(byte[] bytes);+/** Leaf comparator for {@link NumericComparator} that provides skipping functionality */public abstract class NumericLeafComparator implements LeafFieldComparator {private final LeafReaderContext context;@@ -94,8 +99,12 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato// if skipping functionality should be enabled on this segmentprivate final boolean enableSkipping;private final int maxDoc;- private byte[] minValueAsBytes;- private byte[] maxValueAsBytes;++ /** According to {@link FieldComparator#setTopValue}, topValueSet is final in leafComparator */+ private final boolean leafTopSet = topValueSet;++ private long minValueAsLong = Long.MIN_VALUE;+ private long maxValueAsLong = Long.MAX_VALUE;private DocIdSetIterator competitiveIterator;private long iteratorCost = -1;@@ -134,6 +143,9 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatothis.enableSkipping = true; // skipping is enabled when points are availablethis.maxDoc = context.reader().maxDoc();this.competitiveIterator = DocIdSetIterator.all(maxDoc);+ if (leafTopSet) {+ encodeTop();+ }} else {this.pointTree = null;this.enableSkipping = false;@@ -193,7 +205,7 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatoprivate void updateCompetitiveIterator() throws IOException {if (enableSkipping == false|| hitsThresholdReached == false- || (queueFull == false && topValueSet == false)) return;+ || (leafTopSet == false && queueFull == false)) return;// if some documents have missing points, check that missing values prohibits optimizationif ((pointValues.getDocCount() < maxDoc) && isMissingValueCompetitive()) {return; // we can't filter out documents, as documents with missing values are competitive@@ -205,24 +217,9 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato&& (updateCounter & (currentSkipInterval - 1)) != currentSkipInterval - 1) {return;}- if (reverse == false) {- if (queueFull) { // bottom is available only when queue is full- maxValueAsBytes = maxValueAsBytes == null ? new byte[bytesCount] : maxValueAsBytes;- encodeBottom();- }- if (topValueSet) {- minValueAsBytes = minValueAsBytes == null ? new byte[bytesCount] : minValueAsBytes;- encodeTop();- }- } else {- if (queueFull) { // bottom is available only when queue is full- minValueAsBytes = minValueAsBytes == null ? new byte[bytesCount] : minValueAsBytes;- encodeBottom();- }- if (topValueSet) {- maxValueAsBytes = maxValueAsBytes == null ? new byte[bytesCount] : maxValueAsBytes;- encodeTop();- }++ if (queueFull) {+ encodeBottom();}DocIdSetBuilder result = new DocIdSetBuilder(maxDoc);@@ -248,42 +245,32 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatoif (docID <= maxDocVisited) {return; // already visited or skipped}- if (maxValueAsBytes != null) {- int cmp = bytesComparator.compare(packedValue, 0, maxValueAsBytes, 0);-- if (cmp > 0) return;+ long l = sortableBytesToLong(packedValue);+ if (l >= minValueAsLong && l <= maxValueAsLong) {+ adder.add(docID); // doc is competitive}- if (minValueAsBytes != null) {- int cmp = bytesComparator.compare(packedValue, 0, minValueAsBytes, 0);-- if (cmp < 0) return;- }- adder.add(docID); // doc is competitive}@Overridepublic PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- if (maxValueAsBytes != null) {- int cmp = bytesComparator.compare(minPackedValue, 0, maxValueAsBytes, 0);+ long min = sortableBytesToLong(minPackedValue);+ long max = sortableBytesToLong(maxPackedValue);++ if (min > maxValueAsLong || max < minValueAsLong) {// 1. cmp ==0 and pruning==Pruning.GREATER_THAN_OR_EQUAL_TO : if the sort is- // ascending then maxValueAsBytes is bottom's next less value, so it is competitive- // 2. cmp ==0 and pruning==Pruning.GREATER_THAN: maxValueAsBytes equals to+ // ascending then maxValueAsLong is bottom's next less value, so it is competitive+ // 2. cmp ==0 and pruning==Pruning.GREATER_THAN: maxValueAsLong equals to// bottom, but there are multiple comparators, so it could be competitive- if (cmp > 0) return PointValues.Relation.CELL_OUTSIDE_QUERY;- }- if (minValueAsBytes != null) {- int cmp = bytesComparator.compare(maxPackedValue, 0, minValueAsBytes, 0);- if (cmp < 0) return PointValues.Relation.CELL_OUTSIDE_QUERY;+ return PointValues.Relation.CELL_OUTSIDE_QUERY;}- if ((maxValueAsBytes != null- && bytesComparator.compare(maxPackedValue, 0, maxValueAsBytes, 0) > 0)- || (minValueAsBytes != null- && bytesComparator.compare(minPackedValue, 0, minValueAsBytes, 0) < 0)) {++ if (min < minValueAsLong || max > maxValueAsLong) {return PointValues.Relation.CELL_CROSSES_QUERY;}return PointValues.Relation.CELL_INSIDE_QUERY;}};+final long threshold = iteratorCost >>> 3;if (PointValues.isEstimatedPointCountGreaterThanOrEqualTo(visitor, pointTree, threshold)) {@@ -321,42 +308,46 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato/*** If {@link NumericComparator#pruning} equals {@link Pruning#GREATER_THAN_OR_EQUAL_TO}, we- * could better tune the {@link NumericLeafComparator#maxValueAsBytes}/{@link- * NumericLeafComparator#minValueAsBytes}. For instance, if the sort is ascending and bottom+ * could better tune the {@link NumericLeafComparator#maxValueAsLong}/{@link+ * NumericLeafComparator#minValueAsLong}. For instance, if the sort is ascending and bottom* value is 5, we will use a range on [MIN_VALUE, 4].*/private void encodeBottom() {if (reverse == false) {- encodeBottom(maxValueAsBytes);- if (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO) {- NumericUtils.nextDown(maxValueAsBytes);+ maxValueAsLong = bottomAsComparableLong();+ if (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && maxValueAsLong != Long.MIN_VALUE) {+ maxValueAsLong--;}} else {- encodeBottom(minValueAsBytes);- if (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO) {- NumericUtils.nextUp(minValueAsBytes);+ minValueAsLong = bottomAsComparableLong();+ if (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && minValueAsLong != Long.MAX_VALUE) {+ minValueAsLong++;}}}/*** If {@link NumericComparator#pruning} equals {@link Pruning#GREATER_THAN_OR_EQUAL_TO}, we- * could better tune the {@link NumericLeafComparator#maxValueAsBytes}/{@link- * NumericLeafComparator#minValueAsBytes}. For instance, if the sort is ascending and top value+ * could better tune the {@link NumericLeafComparator#minValueAsLong}/{@link+ * NumericLeafComparator#minValueAsLong}. For instance, if the sort is ascending and top value* is 3, we will use a range on [4, MAX_VALUE].*/private void encodeTop() {if (reverse == false) {- encodeTop(minValueAsBytes);- // we could not tune the top value in page search- if (singleSort && pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && queueFull) {- NumericUtils.nextUp(minValueAsBytes);+ minValueAsLong = topAsComparableLong();+ if (singleSort+ && pruning == Pruning.GREATER_THAN_OR_EQUAL_TO+ && queueFull+ && minValueAsLong != Long.MAX_VALUE) {+ minValueAsLong++;}} else {- encodeTop(maxValueAsBytes);- // we could not tune the top value in page search- if (singleSort && pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && queueFull) {- NumericUtils.nextDown(maxValueAsBytes);+ maxValueAsLong = topAsComparableLong();+ if (singleSort+ && pruning == Pruning.GREATER_THAN_OR_EQUAL_TO+ && queueFull+ && maxValueAsLong != Long.MIN_VALUE) {+ maxValueAsLong--;}}}@@ -365,14 +356,14 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato// if queue is full, always compare with bottom,// if not, check if we can compare with topValueif (queueFull) {- int result = compareMissingValueWithBottomValue();+ int result = Long.compare(missingValueAsLong, bottomAsComparableLong());// in reverse (desc) sort missingValue is competitive when it's greater or equal to bottom,// in asc sort missingValue is competitive when it's smaller or equal to bottomreturn reverse? (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO ? result > 0 : result >= 0): (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO ? result < 0 : result <= 0);- } else if (topValueSet) {- int result = compareMissingValueWithTopValue();+ } else if (leafTopSet) {+ int result = Long.compare(missingValueAsLong, topAsComparableLong());// in reverse (desc) sort missingValue is competitive when it's smaller or equal to// topValue,// in asc sort missingValue is competitive when it's greater or equal to topValue@@ -411,18 +402,8 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato};}- /**- * in ascending sort, missing value is competitive when it is less or equal(maybe there are two- * or more comparators) than bottom value. if there is only one comparator(See {@link- * Pruning#GREATER_THAN_OR_EQUAL_TO}), missing value is competitive only when it is less than- * bottom value. vice versa in descending sort.- */- protected abstract int compareMissingValueWithTopValue();-- protected abstract int compareMissingValueWithBottomValue();-- protected abstract void encodeBottom(byte[] packedValue);+ protected abstract long bottomAsComparableLong();- protected abstract void encodeTop(byte[] packedValue);+ protected abstract long topAsComparableLong();}}diff --git a/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java b/lucene/core/src/java/org/apache/lucene/util/NumericUtils.javaindex 0c64b42da3f..1d0a27138e0 100644--- a/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java+++ b/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java@@ -132,52 +132,6 @@ public final class NumericUtils {}}- /**- * Modify in-place the given bytes to the next value of the same length that compares greater than- * the current value. This returns false if, and only if, the value is currently equal to the- * maximum possible value.- */- public static boolean nextUp(byte[] bytes) {- boolean isMax = true;- for (byte b : bytes) {- if (b != (byte) 0xff) {- isMax = false;- break;- }- }- if (isMax) return false;-- for (int i = bytes.length - 1; i >= 0; --i) {- if (++bytes[i] != 0) {- break;- }- }- return true;- }-- /**- * Modify in-place the given bytes to the previous value of the same length that compares less- * than the current value. This returns false if, and only if, the value is currently equal to the- * minimum possible value.- */- public static boolean nextDown(byte[] bytes) {- boolean isMin = true;- for (byte b : bytes) {- if (b != 0) {- isMin = false;- break;- }- }- if (isMin) return false;-- for (int i = bytes.length - 1; i >= 0; --i) {- if (bytes[i]-- != 0) {- break;- }- }- return true;- }-/*** Encodes an integer {@code value} such that unsigned byte order comparison is consistent with* {@link Integer#compare(int, int)}diff --git a/lucene/core/src/test/org/apache/lucene/util/TestNumericUtils.java b/lucene/core/src/test/org/apache/lucene/util/TestNumericUtils.javaindex 1e117ebc7f1..9089b5b4c53 100644--- a/lucene/core/src/test/org/apache/lucene/util/TestNumericUtils.java+++ b/lucene/core/src/test/org/apache/lucene/util/TestNumericUtils.java@@ -616,32 +616,4 @@ public class TestNumericUtils extends LuceneTestCase {Integer.signum(leftValue.compareTo(rightValue)), Integer.signum(left.compareTo(right)));}}-- public void testNextUp() {- for (int i : new int[] {Integer.MIN_VALUE, -256, -255, -1, 0, 1, 255, 256, Integer.MAX_VALUE}) {- byte[] b = new byte[Integer.BYTES];- NumericUtils.intToSortableBytes(i, b, 0);- if (i == Integer.MAX_VALUE) {- assertFalse(NumericUtils.nextUp(b));- assertEquals(i, NumericUtils.sortableBytesToInt(b, 0));- } else {- assertTrue(NumericUtils.nextUp(b));- assertEquals(i + 1, NumericUtils.sortableBytesToInt(b, 0));- }- }- }-- public void testNextDown() {- for (int i : new int[] {Integer.MIN_VALUE, -256, -255, -1, 0, 1, 255, 256, Integer.MAX_VALUE}) {- byte[] b = new byte[Integer.BYTES];- NumericUtils.intToSortableBytes(i, b, 0);- if (i == Integer.MIN_VALUE) {- assertFalse(NumericUtils.nextDown(b));- assertEquals(i, NumericUtils.sortableBytesToInt(b, 0));- } else {- assertTrue(NumericUtils.nextDown(b));- assertEquals(i - 1, NumericUtils.sortableBytesToInt(b, 0));- }- }- }}