Skip to content

Apache Lucene open-source search software

Use SIMD in fixed-cardinality SortedNumericDocValues rangeIntoBitSet() (#16283)

Costin Leau42e71d8e73f1
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 456e95f8d2f..2a6cf9ecec4 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -320,6 +320,9 @@ Optimizations
* GITHUB#16160: Improve numeric doc values range query performance for dense fields that use GCD or delta encoding. (Costin Leau)
+* GITHUB#16283: Use Panama Vector API to SIMD-evaluate fixed-cardinality sorted numeric range queries in
+ rangeIntoBitSet. (Costin Leau)
+
Bug Fixes
---------------------
(No changes)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
index 811f11c5343..f3599924c32 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
@@ -562,29 +562,6 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
return (int) cardinality;
}
- private static void sortedNumericScalarRangeIntoBitSet(
- LongValues values,
- int fromDoc,
- int toDoc,
- int cardinality,
- long minValue,
- long maxValue,
- FixedBitSet bitSet,
- int offset) {
- for (int doc = fromDoc; doc < toDoc; doc++) {
- long valueOffset = (long) doc * cardinality;
- for (int i = 0; i < cardinality; i++) {
- long value = values.get(valueOffset + i);
- if (value >= minValue) {
- if (value <= maxValue) {
- bitSet.set(doc - offset);
- }
- break;
- }
- }
- }
- }
-
private static boolean sortedNumericMatchesRange(
LongValues values, long start, long end, long minValue, long maxValue) {
for (long valueOffset = start; valueOffset < end; valueOffset++) {
@@ -2024,7 +2001,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
}
int cardinality = denseFixedCardinality;
if (cardinality > 1) {
- sortedNumericScalarRangeIntoBitSet(
+ DOC_VALUES_RANGE_SUPPORT.sortedNumericRangeIntoBitSet(
values, fromDoc, endDoc, cardinality, minValue, maxValue, bitSet, offset);
return;
}
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java
index 7b8b4e2c949..db8a79e1605 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java
@@ -50,4 +50,36 @@ public interface DocValuesRangeSupport {
long maxValue,
FixedBitSet bitSet,
int offset);
+
+ /**
+ * Fills {@code bitSet} with docs in {@code [fromDoc, toDoc)} whose sorted numeric values contain
+ * at least one value in {@code [minValue, maxValue]}.
+ *
+ * <p>This method only supports <b>fixed-cardinality</b> fields where every document has exactly
+ * {@code cardinality} values stored contiguously starting at index {@code doc * cardinality}.
+ *
+ * @param cardinality number of values per document (must be &gt; 0)
+ */
+ default void sortedNumericRangeIntoBitSet(
+ LongValues values,
+ int fromDoc,
+ int toDoc,
+ int cardinality,
+ long minValue,
+ long maxValue,
+ FixedBitSet bitSet,
+ int offset) {
+ for (int doc = fromDoc; doc < toDoc; doc++) {
+ long valueOffset = (long) doc * cardinality;
+ for (int i = 0; i < cardinality; i++) {
+ long value = values.get(valueOffset + i);
+ if (value >= minValue) {
+ if (value <= maxValue) {
+ bitSet.set(doc - offset);
+ }
+ break;
+ }
+ }
+ }
+ }
}
diff --git a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java
index 72fef3c31b6..a1a6ca1591e 100644
--- a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java
+++ b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java
@@ -69,4 +69,61 @@ final class PanamaDocValuesRangeSupport implements DocValuesRangeSupport {
}
}
}
+
+ @Override
+ public void sortedNumericRangeIntoBitSet(
+ LongValues values,
+ int fromDoc,
+ int toDoc,
+ int cardinality,
+ long minValue,
+ long maxValue,
+ FixedBitSet bitSet,
+ int offset) {
+ assert cardinality > 0 : "cardinality must be positive: " + cardinality;
+ final int vectorLen = LONG_SPECIES.length();
+ final int docsPerVector = vectorLen / cardinality;
+ if (docsPerVector == 0 || vectorLen % cardinality != 0) {
+ DocValuesRangeSupport.super.sortedNumericRangeIntoBitSet(
+ values, fromDoc, toDoc, cardinality, minValue, maxValue, bitSet, offset);
+ return;
+ }
+
+ final long[] scratch = new long[vectorLen];
+ final int vectorDocEnd = fromDoc + (toDoc - fromDoc) / docsPerVector * docsPerVector;
+ int doc = fromDoc;
+ for (; doc < vectorDocEnd; doc += docsPerVector) {
+ long valueOffset = (long) doc * cardinality;
+ for (int lane = 0; lane < vectorLen; lane++) {
+ scratch[lane] = values.get(valueOffset + lane);
+ }
+ LongVector vector = LongVector.fromArray(LONG_SPECIES, scratch, 0);
+ // Flat range check on all lanes. Equivalent to the scalar early-break on sorted values:
+ // a value outside [min,max] that is >= min must be > max, so its lane stays unset, and
+ // collapseToDocMask OR-reduces lanes per doc to match "any value in range" semantics.
+ VectorMask<Long> inRange =
+ vector
+ .compare(VectorOperators.GE, minValue)
+ .and(vector.compare(VectorOperators.LE, maxValue));
+ long docMask = collapseToDocMask(inRange.toLong(), cardinality, docsPerVector);
+ if (docMask != 0) {
+ bitSet.orMask(doc - offset, docMask, docsPerVector);
+ }
+ }
+
+ DocValuesRangeSupport.super.sortedNumericRangeIntoBitSet(
+ values, doc, toDoc, cardinality, minValue, maxValue, bitSet, offset);
+ }
+
+ private static long collapseToDocMask(long valueMask, int cardinality, int docsPerVector) {
+ final long valueMaskPerDoc = (1L << cardinality) - 1L;
+ long docMask = 0L;
+ for (int d = 0; d < docsPerVector; d++) {
+ long perDoc = (valueMask >>> (d * cardinality)) & valueMaskPerDoc;
+ if (perDoc != 0) {
+ docMask |= 1L << d;
+ }
+ }
+ return docMask;
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java
index ccadb56b3af..00cb26aca1d 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java
@@ -764,6 +764,17 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe
private void doTestSortedNumericRangeIntoBitSet(boolean dense, boolean fixedCardinality)
throws Exception {
+ doTestSortedNumericRangeIntoBitSet(dense, fixedCardinality, 4);
+ }
+
+ public void testSortedNumericRangeIntoBitSetVaryingCardinality() throws Exception {
+ for (int cardinality : new int[] {2, 3, 4, 5, 7, 8}) {
+ doTestSortedNumericRangeIntoBitSet(true, true, cardinality);
+ }
+ }
+
+ private void doTestSortedNumericRangeIntoBitSet(
+ boolean dense, boolean fixedCardinality, int fixedCardinalityValue) throws Exception {
int numDocs = 4096 * 2;
try (Directory dir = newDirectory()) {
IndexWriterConfig iwc = new IndexWriterConfig().setCodec(new Lucene104Codec());
@@ -771,7 +782,7 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe
for (int docID = 0; docID < numDocs; docID++) {
Document doc = new Document();
if (dense || docID % 3 != 0) {
- int valueCount = fixedCardinality ? 4 : 1 + (docID & 3);
+ int valueCount = fixedCardinality ? fixedCardinalityValue : 1 + (docID & 3);
long firstValue = (docID * 13L) % 100;
for (int i = 0; i < valueCount; i++) {
doc.add(SortedNumericDocValuesField.indexedField("sn", firstValue + i * 3L));