Apache Lucene open-source search software
Use SIMD in fixed-cardinality SortedNumericDocValues rangeIntoBitSet() (#16283)
Costin Leau42e71d8e73f1
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 456e95f8d2f..2a6cf9ecec4 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -320,6 +320,9 @@ Optimizations* GITHUB#16160: Improve numeric doc values range query performance for dense fields that use GCD or delta encoding. (Costin Leau)+* GITHUB#16283: Use Panama Vector API to SIMD-evaluate fixed-cardinality sorted numeric range queries in+ rangeIntoBitSet. (Costin Leau)+Bug Fixes---------------------(No changes)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.javaindex 811f11c5343..f3599924c32 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java@@ -562,29 +562,6 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {return (int) cardinality;}- private static void sortedNumericScalarRangeIntoBitSet(- LongValues values,- int fromDoc,- int toDoc,- int cardinality,- long minValue,- long maxValue,- FixedBitSet bitSet,- int offset) {- for (int doc = fromDoc; doc < toDoc; doc++) {- long valueOffset = (long) doc * cardinality;- for (int i = 0; i < cardinality; i++) {- long value = values.get(valueOffset + i);- if (value >= minValue) {- if (value <= maxValue) {- bitSet.set(doc - offset);- }- break;- }- }- }- }-private static boolean sortedNumericMatchesRange(LongValues values, long start, long end, long minValue, long maxValue) {for (long valueOffset = start; valueOffset < end; valueOffset++) {@@ -2024,7 +2001,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {}int cardinality = denseFixedCardinality;if (cardinality > 1) {- sortedNumericScalarRangeIntoBitSet(+ DOC_VALUES_RANGE_SUPPORT.sortedNumericRangeIntoBitSet(values, fromDoc, endDoc, cardinality, minValue, maxValue, bitSet, offset);return;}diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.javaindex 7b8b4e2c949..db8a79e1605 100644--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesRangeSupport.java@@ -50,4 +50,36 @@ public interface DocValuesRangeSupport {long maxValue,FixedBitSet bitSet,int offset);++ /**+ * Fills {@code bitSet} with docs in {@code [fromDoc, toDoc)} whose sorted numeric values contain+ * at least one value in {@code [minValue, maxValue]}.+ *+ * <p>This method only supports <b>fixed-cardinality</b> fields where every document has exactly+ * {@code cardinality} values stored contiguously starting at index {@code doc * cardinality}.+ *+ * @param cardinality number of values per document (must be > 0)+ */+ default void sortedNumericRangeIntoBitSet(+ LongValues values,+ int fromDoc,+ int toDoc,+ int cardinality,+ long minValue,+ long maxValue,+ FixedBitSet bitSet,+ int offset) {+ for (int doc = fromDoc; doc < toDoc; doc++) {+ long valueOffset = (long) doc * cardinality;+ for (int i = 0; i < cardinality; i++) {+ long value = values.get(valueOffset + i);+ if (value >= minValue) {+ if (value <= maxValue) {+ bitSet.set(doc - offset);+ }+ break;+ }+ }+ }+ }}diff --git a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.javaindex 72fef3c31b6..a1a6ca1591e 100644--- a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java+++ b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesRangeSupport.java@@ -69,4 +69,61 @@ final class PanamaDocValuesRangeSupport implements DocValuesRangeSupport {}}}++ @Override+ public void sortedNumericRangeIntoBitSet(+ LongValues values,+ int fromDoc,+ int toDoc,+ int cardinality,+ long minValue,+ long maxValue,+ FixedBitSet bitSet,+ int offset) {+ assert cardinality > 0 : "cardinality must be positive: " + cardinality;+ final int vectorLen = LONG_SPECIES.length();+ final int docsPerVector = vectorLen / cardinality;+ if (docsPerVector == 0 || vectorLen % cardinality != 0) {+ DocValuesRangeSupport.super.sortedNumericRangeIntoBitSet(+ values, fromDoc, toDoc, cardinality, minValue, maxValue, bitSet, offset);+ return;+ }++ final long[] scratch = new long[vectorLen];+ final int vectorDocEnd = fromDoc + (toDoc - fromDoc) / docsPerVector * docsPerVector;+ int doc = fromDoc;+ for (; doc < vectorDocEnd; doc += docsPerVector) {+ long valueOffset = (long) doc * cardinality;+ for (int lane = 0; lane < vectorLen; lane++) {+ scratch[lane] = values.get(valueOffset + lane);+ }+ LongVector vector = LongVector.fromArray(LONG_SPECIES, scratch, 0);+ // Flat range check on all lanes. Equivalent to the scalar early-break on sorted values:+ // a value outside [min,max] that is >= min must be > max, so its lane stays unset, and+ // collapseToDocMask OR-reduces lanes per doc to match "any value in range" semantics.+ VectorMask<Long> inRange =+ vector+ .compare(VectorOperators.GE, minValue)+ .and(vector.compare(VectorOperators.LE, maxValue));+ long docMask = collapseToDocMask(inRange.toLong(), cardinality, docsPerVector);+ if (docMask != 0) {+ bitSet.orMask(doc - offset, docMask, docsPerVector);+ }+ }++ DocValuesRangeSupport.super.sortedNumericRangeIntoBitSet(+ values, doc, toDoc, cardinality, minValue, maxValue, bitSet, offset);+ }++ private static long collapseToDocMask(long valueMask, int cardinality, int docsPerVector) {+ final long valueMaskPerDoc = (1L << cardinality) - 1L;+ long docMask = 0L;+ for (int d = 0; d < docsPerVector; d++) {+ long perDoc = (valueMask >>> (d * cardinality)) & valueMaskPerDoc;+ if (perDoc != 0) {+ docMask |= 1L << d;+ }+ }+ return docMask;+ }}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.javaindex ccadb56b3af..00cb26aca1d 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java@@ -764,6 +764,17 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTeprivate void doTestSortedNumericRangeIntoBitSet(boolean dense, boolean fixedCardinality)throws Exception {+ doTestSortedNumericRangeIntoBitSet(dense, fixedCardinality, 4);+ }++ public void testSortedNumericRangeIntoBitSetVaryingCardinality() throws Exception {+ for (int cardinality : new int[] {2, 3, 4, 5, 7, 8}) {+ doTestSortedNumericRangeIntoBitSet(true, true, cardinality);+ }+ }++ private void doTestSortedNumericRangeIntoBitSet(+ boolean dense, boolean fixedCardinality, int fixedCardinalityValue) throws Exception {int numDocs = 4096 * 2;try (Directory dir = newDirectory()) {IndexWriterConfig iwc = new IndexWriterConfig().setCodec(new Lucene104Codec());@@ -771,7 +782,7 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTefor (int docID = 0; docID < numDocs; docID++) {Document doc = new Document();if (dense || docID % 3 != 0) {- int valueCount = fixedCardinality ? 4 : 1 + (docID & 3);+ int valueCount = fixedCardinality ? fixedCardinalityValue : 1 + (docID & 3);long firstValue = (docID * 13L) % 100;for (int i = 0; i < valueCount; i++) {doc.add(SortedNumericDocValuesField.indexedField("sn", firstValue + i * 3L));