Skip to content

Apache Lucene open-source search software

Optimize encoded numeric range bitsets (#16160)

Costin Leau9b68f633fdd6
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 8ecc7015817..456e95f8d2f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -318,6 +318,8 @@ Optimizations
* GITHUB#16316: Lazily allocate buffers in MaxScoreBulkScorer. These are now only allocated when going
via code paths that actually use them. (Alan Woodward)
+* GITHUB#16160: Improve numeric doc values range query performance for dense fields that use GCD or delta encoding. (Costin Leau)
+
Bug Fixes
---------------------
(No changes)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GcdDeltaRangeIntoBitSetBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GcdDeltaRangeIntoBitSetBenchmark.java
new file mode 100644
index 00000000000..2cba7a0d688
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GcdDeltaRangeIntoBitSetBenchmark.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/** Benchmarks range queries over dense numeric doc values encoded as raw, delta, GCD, or both. */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 3, time = 3)
+@Measurement(iterations = 5, time = 5)
+public class GcdDeltaRangeIntoBitSetBenchmark {
+
+ private static final String FIELD = "val";
+ private static final String LEAD_FIELD = "lead";
+ private static final String LEAD_VALUE = "yes";
+ private static final String NONE = "none";
+ private static final String DELTA_ONLY = "delta_only";
+ private static final String GCD_1000 = "gcd_1000";
+ private static final String GCD_100_DELTA = "gcd_100_delta";
+ private static final long DOMAIN = 10_000_000L;
+ private static final long DELTA = 1_700_000_000_000L;
+
+ private Directory dir;
+ private DirectoryReader reader;
+ private IndexSearcher searcher;
+ private Path path;
+ private Query query;
+
+ @Param({"1000000"})
+ public int numDocs;
+
+ @Param({NONE, DELTA_ONLY, GCD_1000, GCD_100_DELTA})
+ public String encoding;
+
+ @Param({"0.01", "0.1", "0.5"})
+ public double selectivity;
+
+ @Setup(Level.Trial)
+ public void setup() throws Exception {
+ path = Files.createTempDirectory("gcdDeltaRangeIntoBitSet");
+ dir = MMapDirectory.open(path);
+
+ Random random = new Random(0);
+ try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig())) {
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField(FIELD, valueForDoc(encoding, i, random)));
+ doc.add(new StringField(LEAD_FIELD, LEAD_VALUE, Field.Store.NO));
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ }
+
+ reader = DirectoryReader.open(dir);
+ searcher = new IndexSearcher(reader);
+ query = rangeQuery(encoding, selectivity);
+ }
+
+ private static long valueForDoc(String encoding, int doc, Random random) {
+ if (doc == 0) {
+ return minimumValue(encoding);
+ } else if (doc == 1 && encoding.equals(GCD_100_DELTA)) {
+ // Anchor entry.gcd to exactly 100 for the GCD_100_DELTA encoding: random multiples of 100
+ // could otherwise share a larger common factor under some seeds, which would change the
+ // shape of the encoded values and what the benchmark measures.
+ return DELTA + 100L;
+ }
+
+ long value = random.nextLong(0, DOMAIN);
+ switch (encoding) {
+ case NONE:
+ return value;
+ case DELTA_ONLY:
+ return DELTA + value;
+ case GCD_1000:
+ return value * 1_000L;
+ case GCD_100_DELTA:
+ return DELTA + value * 100L;
+ default:
+ throw new IllegalArgumentException("Unknown encoding: " + encoding);
+ }
+ }
+
+ private static long minimumValue(String encoding) {
+ switch (encoding) {
+ case NONE:
+ case GCD_1000:
+ return 0;
+ case DELTA_ONLY:
+ case GCD_100_DELTA:
+ return DELTA;
+ default:
+ throw new IllegalArgumentException("Unknown encoding: " + encoding);
+ }
+ }
+
+ private static Query rangeQuery(String encoding, double selectivity) {
+ long range = Math.max(1, (long) (DOMAIN * selectivity));
+ long min = (DOMAIN - range) / 2;
+ long max = min + range;
+ Query rangeQuery =
+ SortedNumericDocValuesField.newSlowRangeQuery(
+ FIELD, actualValue(encoding, min), actualValue(encoding, max));
+ return new BooleanQuery.Builder()
+ .add(new TermQuery(new Term(LEAD_FIELD, LEAD_VALUE)), Occur.FILTER)
+ .add(rangeQuery, Occur.FILTER)
+ .build();
+ }
+
+ private static long actualValue(String encoding, long value) {
+ switch (encoding) {
+ case NONE:
+ return value;
+ case DELTA_ONLY:
+ return DELTA + value;
+ case GCD_1000:
+ return value * 1_000L;
+ case GCD_100_DELTA:
+ return DELTA + value * 100L;
+ default:
+ throw new IllegalArgumentException("Unknown encoding: " + encoding);
+ }
+ }
+
+ @TearDown(Level.Trial)
+ public void tearDown() throws Exception {
+ reader.close();
+ dir.close();
+ if (Files.exists(path)) {
+ try (Stream<Path> walk = Files.walk(path)) {
+ walk.sorted(Comparator.reverseOrder())
+ .forEach(
+ p -> {
+ try {
+ Files.delete(p);
+ } catch (IOException _) {
+ }
+ });
+ }
+ }
+ }
+
+ @Benchmark
+ @Fork(
+ value = 1,
+ jvmArgsAppend = {"-Xmx2g", "-Xms2g", "-XX:+AlwaysPreTouch"})
+ public int rangeQueryDefaultProvider() throws IOException {
+ return searcher.count(query);
+ }
+
+ @Benchmark
+ @Fork(
+ value = 1,
+ jvmArgsAppend = {
+ "--add-modules",
+ "jdk.incubator.vector",
+ "-Xmx2g",
+ "-Xms2g",
+ "-XX:+AlwaysPreTouch"
+ })
+ public int rangeQueryPanamaProvider() throws IOException {
+ return searcher.count(query);
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
index 568b0ef01a5..811f11c5343 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
@@ -481,6 +481,72 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
values, fromDoc, toDoc, minValue, maxValue, bitSet, offset);
}
+ /**
+ * Maps the raw query bounds {@code [minValue, maxValue]} into the encoded domain so the SIMD
+ * kernel in {@link org.apache.lucene.internal.vectorization.DocValuesRangeSupport} can run
+ * directly on packed values: {@code [encodedMin, encodedMax] = [ceil((min - delta) / mul),
+ * floor((max - delta) / mul)]}. Open bounds (e.g. {@code Long.MIN_VALUE} or {@code
+ * Long.MAX_VALUE}) are saturated to the encoded domain so they keep the SIMD path even when
+ * {@code min - delta} or {@code max - delta} would overflow.
+ */
+ private static void rangeGcdDeltaIntoBitSet(
+ LongValues values,
+ int fromDoc,
+ int toDoc,
+ long minValue,
+ long maxValue,
+ long mul,
+ long delta,
+ FixedBitSet bitSet,
+ int offset) {
+ assert mul > 0;
+ long encodedMin = saturatingShiftLower(minValue, delta);
+ long encodedMax = saturatingShiftUpper(maxValue, delta);
+ if (mul != 1) {
+ // Math.ceilDiv / Math.floorDiv never overflow for mul > 0 (only Long.MIN_VALUE / -1 does),
+ // so the SIMD path is always taken; no fallback to the per-doc decoded loop is required.
+ encodedMin = Math.ceilDiv(encodedMin, mul);
+ encodedMax = Math.floorDiv(encodedMax, mul);
+ }
+ encodedMin = Math.max(0, encodedMin);
+ if (encodedMin <= encodedMax) {
+ rangeIntoBitSet(values, fromDoc, toDoc, encodedMin, encodedMax, bitSet, offset);
+ }
+ }
+
+ /**
+ * Returns {@code minValue - delta}, saturating to {@code Long.MIN_VALUE} when the real value
+ * would underflow (every non-negative stored value satisfies the lower bound) or to {@code
+ * Long.MAX_VALUE} when it would overflow (no stored value can satisfy the lower bound). Stored
+ * values are non-negative, so the caller can keep using the SIMD path with these saturated
+ * sentinels.
+ */
+ private static long saturatingShiftLower(long minValue, long delta) {
+ try {
+ return Math.subtractExact(minValue, delta);
+ } catch (
+ @SuppressWarnings("unused")
+ ArithmeticException overflow) {
+ return delta > 0 ? Long.MIN_VALUE : Long.MAX_VALUE;
+ }
+ }
+
+ /**
+ * Symmetric counterpart of {@link #saturatingShiftLower}: returns {@code maxValue - delta},
+ * saturating to {@code Long.MAX_VALUE} when the real value would overflow (every stored value
+ * satisfies the upper bound) or to {@code Long.MIN_VALUE} when it would underflow (no stored
+ * value can satisfy the upper bound).
+ */
+ private static long saturatingShiftUpper(long maxValue, long delta) {
+ try {
+ return Math.subtractExact(maxValue, delta);
+ } catch (
+ @SuppressWarnings("unused")
+ ArithmeticException overflow) {
+ return delta < 0 ? Long.MAX_VALUE : Long.MIN_VALUE;
+ }
+ }
+
private static int fixedCardinality(
SortedNumericEntry entry, DocValuesSkipperEntry skipperEntry) {
if (skipperEntry == null
@@ -960,13 +1026,8 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
long maxValue,
FixedBitSet bitSet,
int offset) {
- // Per-doc evaluation for gcd/delta encoded fields
- for (int d = fromDoc; d < toDoc; d++) {
- long v = mul * values.get(d) + delta;
- if (v >= minValue && v <= maxValue) {
- bitSet.set(d - offset);
- }
- }
+ Lucene90DocValuesProducer.rangeGcdDeltaIntoBitSet(
+ values, fromDoc, toDoc, minValue, maxValue, mul, delta, bitSet, offset);
}
};
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java
index 76e01cae13c..ccadb56b3af 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
+import java.util.function.LongUnaryOperator;
import org.apache.lucene.codecs.lucene104.Lucene104Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.NumericDocValuesField;
@@ -634,6 +635,117 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe
}
}
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithDeltaEncoding() throws Exception {
+ long delta = 1_000_000L;
+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "delta-only encoded range must match decoded evaluation",
+ values,
+ delta + 127,
+ delta + 4097);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithGcdEncoding() throws Exception {
+ long[] values = rangeValues(DOC_COUNT, doc -> doc * 1_000L);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "gcd encoded range must match decoded evaluation", values, 123_456L, 4_567_890L);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithGcdAndDeltaEncoding() throws Exception {
+ long delta = 1_000_000L;
+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "gcd+delta encoded range must match decoded evaluation",
+ values,
+ delta + 123,
+ delta + 456_789);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWhenGcdRangeFallsBetweenValues()
+ throws Exception {
+ long[] values = rangeValues(DOC_COUNT, doc -> 10L + doc * 5L);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "gcd encoded gap range must match no docs", values, 11, 14);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithOpenLowerBound() throws Exception {
+ long delta = 1_000_000L;
+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "open Long.MIN_VALUE lower bound must saturate and match all docs up to the upper bound",
+ values,
+ Long.MIN_VALUE,
+ delta + 127);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithOpenUpperBound() throws Exception {
+ long delta = 1_000_000L;
+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "open Long.MAX_VALUE upper bound must saturate and match all docs from the lower bound",
+ values,
+ delta + 50L * 100L,
+ Long.MAX_VALUE);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithBothOpenBounds() throws Exception {
+ long delta = 1_000_000L;
+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "fully open range must match every doc", values, Long.MIN_VALUE, Long.MAX_VALUE);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithNegativeEncodedLowerBound()
+ throws Exception {
+ // Stored values are non-negative (min == delta), but the query lower bound is below delta so
+ // (minValue - delta) is negative even though Math.subtractExact succeeds. This exercises the
+ // Math.max(0, encodedMin) clamp.
+ long delta = 1_000_000L;
+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "negative encoded lower bound must clamp to 0 and match docs up to the upper bound",
+ values,
+ delta - 50,
+ delta + 100);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithRangeBelowStoredValues()
+ throws Exception {
+ // The whole query range is below the stored minimum, so encodedMin > encodedMax after the
+ // bound transformation and the SIMD path is skipped without iterating.
+ long delta = 1_000_000L;
+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "query range below stored values must match no docs", values, delta - 1_000L, delta - 1L);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithRangeAboveStoredValues()
+ throws Exception {
+ long delta = 1_000_000L;
+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);
+ long max = delta + (DOC_COUNT - 1L) * 100L;
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "query range above stored values must match no docs", values, max + 1L, max + 1_000L);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithFullGcdDeltaRange() throws Exception {
+ long delta = 1_000_000L;
+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "full gcd+delta encoded range must match all docs",
+ values,
+ delta,
+ delta + (DOC_COUNT - 1L) * 100L);
+ }
+
+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithSingleGcdDeltaValue() throws Exception {
+ long delta = 1_000_000L;
+ long value = delta + 123L * 100L;
+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);
+ assertRangeIntoBitSetMatchesPerDocEvaluation(
+ "single gcd+delta encoded value range must match one doc", values, value, value);
+ }
+
public void testSortedNumericRangeIntoBitSetDenseFixedCardinality() throws Exception {
doTestSortedNumericRangeIntoBitSet(true, true);
}
@@ -694,4 +806,47 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe
}
}
}
+
+ private static long[] rangeValues(int numDocs, LongUnaryOperator valueFunction) {
+ long[] values = new long[numDocs];
+ for (int i = 0; i < numDocs; i++) {
+ values[i] = valueFunction.applyAsLong(i);
+ }
+ return values;
+ }
+
+ private void assertRangeIntoBitSetMatchesPerDocEvaluation(
+ String message, long[] values, long rangeMin, long rangeMax) throws Exception {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig().setCodec(new Lucene104Codec());
+ try (IndexWriter w = new IndexWriter(dir, iwc)) {
+ for (long value : values) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("val", value));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ }
+
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ LeafReaderContext ctx = reader.leaves().get(0);
+ FixedBitSet expected = new FixedBitSet(values.length);
+ NumericDocValues slowDv = ctx.reader().getNumericDocValues("val");
+ for (int d = 0; d < values.length; d++) {
+ if (slowDv.advanceExact(d)) {
+ long value = slowDv.longValue();
+ if (value >= rangeMin && value <= rangeMax) {
+ expected.set(d);
+ }
+ }
+ }
+
+ FixedBitSet actual = new FixedBitSet(values.length);
+ NumericDocValues fastDv = ctx.reader().getNumericDocValues("val");
+ fastDv.rangeIntoBitSet(0, values.length, rangeMin, rangeMax, actual, 0);
+
+ assertEquals(message, expected, actual);
+ }
+ }
+ }
}