Apache Lucene open-source search software
Optimize encoded numeric range bitsets (#16160)
Costin Leau9b68f633fdd6
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 8ecc7015817..456e95f8d2f 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -318,6 +318,8 @@ Optimizations* GITHUB#16316: Lazily allocate buffers in MaxScoreBulkScorer. These are now only allocated when goingvia code paths that actually use them. (Alan Woodward)+* GITHUB#16160: Improve numeric doc values range query performance for dense fields that use GCD or delta encoding. (Costin Leau)+Bug Fixes---------------------(No changes)diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GcdDeltaRangeIntoBitSetBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GcdDeltaRangeIntoBitSetBenchmark.javanew file mode 100644index 00000000000..2cba7a0d688--- /dev/null+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GcdDeltaRangeIntoBitSetBenchmark.java@@ -0,0 +1,215 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.benchmark.jmh;++import java.io.IOException;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.Comparator;+import java.util.Random;+import java.util.concurrent.TimeUnit;+import java.util.stream.Stream;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.document.SortedNumericDocValuesField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.Term;+import org.apache.lucene.search.BooleanClause.Occur;+import org.apache.lucene.search.BooleanQuery;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.Query;+import org.apache.lucene.search.TermQuery;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.MMapDirectory;+import org.openjdk.jmh.annotations.Benchmark;+import org.openjdk.jmh.annotations.BenchmarkMode;+import org.openjdk.jmh.annotations.Fork;+import org.openjdk.jmh.annotations.Level;+import org.openjdk.jmh.annotations.Measurement;+import org.openjdk.jmh.annotations.Mode;+import org.openjdk.jmh.annotations.OutputTimeUnit;+import org.openjdk.jmh.annotations.Param;+import org.openjdk.jmh.annotations.Scope;+import org.openjdk.jmh.annotations.Setup;+import org.openjdk.jmh.annotations.State;+import org.openjdk.jmh.annotations.TearDown;+import org.openjdk.jmh.annotations.Warmup;++/** Benchmarks range queries over dense numeric doc values encoded as raw, delta, GCD, or both. */+@State(Scope.Thread)+@BenchmarkMode(Mode.Throughput)+@OutputTimeUnit(TimeUnit.SECONDS)+@Warmup(iterations = 3, time = 3)+@Measurement(iterations = 5, time = 5)+public class GcdDeltaRangeIntoBitSetBenchmark {++ private static final String FIELD = "val";+ private static final String LEAD_FIELD = "lead";+ private static final String LEAD_VALUE = "yes";+ private static final String NONE = "none";+ private static final String DELTA_ONLY = "delta_only";+ private static final String GCD_1000 = "gcd_1000";+ private static final String GCD_100_DELTA = "gcd_100_delta";+ private static final long DOMAIN = 10_000_000L;+ private static final long DELTA = 1_700_000_000_000L;++ private Directory dir;+ private DirectoryReader reader;+ private IndexSearcher searcher;+ private Path path;+ private Query query;++ @Param({"1000000"})+ public int numDocs;++ @Param({NONE, DELTA_ONLY, GCD_1000, GCD_100_DELTA})+ public String encoding;++ @Param({"0.01", "0.1", "0.5"})+ public double selectivity;++ @Setup(Level.Trial)+ public void setup() throws Exception {+ path = Files.createTempDirectory("gcdDeltaRangeIntoBitSet");+ dir = MMapDirectory.open(path);++ Random random = new Random(0);+ try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig())) {+ for (int i = 0; i < numDocs; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField(FIELD, valueForDoc(encoding, i, random)));+ doc.add(new StringField(LEAD_FIELD, LEAD_VALUE, Field.Store.NO));+ writer.addDocument(doc);+ }+ writer.forceMerge(1);+ }++ reader = DirectoryReader.open(dir);+ searcher = new IndexSearcher(reader);+ query = rangeQuery(encoding, selectivity);+ }++ private static long valueForDoc(String encoding, int doc, Random random) {+ if (doc == 0) {+ return minimumValue(encoding);+ } else if (doc == 1 && encoding.equals(GCD_100_DELTA)) {+ // Anchor entry.gcd to exactly 100 for the GCD_100_DELTA encoding: random multiples of 100+ // could otherwise share a larger common factor under some seeds, which would change the+ // shape of the encoded values and what the benchmark measures.+ return DELTA + 100L;+ }++ long value = random.nextLong(0, DOMAIN);+ switch (encoding) {+ case NONE:+ return value;+ case DELTA_ONLY:+ return DELTA + value;+ case GCD_1000:+ return value * 1_000L;+ case GCD_100_DELTA:+ return DELTA + value * 100L;+ default:+ throw new IllegalArgumentException("Unknown encoding: " + encoding);+ }+ }++ private static long minimumValue(String encoding) {+ switch (encoding) {+ case NONE:+ case GCD_1000:+ return 0;+ case DELTA_ONLY:+ case GCD_100_DELTA:+ return DELTA;+ default:+ throw new IllegalArgumentException("Unknown encoding: " + encoding);+ }+ }++ private static Query rangeQuery(String encoding, double selectivity) {+ long range = Math.max(1, (long) (DOMAIN * selectivity));+ long min = (DOMAIN - range) / 2;+ long max = min + range;+ Query rangeQuery =+ SortedNumericDocValuesField.newSlowRangeQuery(+ FIELD, actualValue(encoding, min), actualValue(encoding, max));+ return new BooleanQuery.Builder()+ .add(new TermQuery(new Term(LEAD_FIELD, LEAD_VALUE)), Occur.FILTER)+ .add(rangeQuery, Occur.FILTER)+ .build();+ }++ private static long actualValue(String encoding, long value) {+ switch (encoding) {+ case NONE:+ return value;+ case DELTA_ONLY:+ return DELTA + value;+ case GCD_1000:+ return value * 1_000L;+ case GCD_100_DELTA:+ return DELTA + value * 100L;+ default:+ throw new IllegalArgumentException("Unknown encoding: " + encoding);+ }+ }++ @TearDown(Level.Trial)+ public void tearDown() throws Exception {+ reader.close();+ dir.close();+ if (Files.exists(path)) {+ try (Stream<Path> walk = Files.walk(path)) {+ walk.sorted(Comparator.reverseOrder())+ .forEach(+ p -> {+ try {+ Files.delete(p);+ } catch (IOException _) {+ }+ });+ }+ }+ }++ @Benchmark+ @Fork(+ value = 1,+ jvmArgsAppend = {"-Xmx2g", "-Xms2g", "-XX:+AlwaysPreTouch"})+ public int rangeQueryDefaultProvider() throws IOException {+ return searcher.count(query);+ }++ @Benchmark+ @Fork(+ value = 1,+ jvmArgsAppend = {+ "--add-modules",+ "jdk.incubator.vector",+ "-Xmx2g",+ "-Xms2g",+ "-XX:+AlwaysPreTouch"+ })+ public int rangeQueryPanamaProvider() throws IOException {+ return searcher.count(query);+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.javaindex 568b0ef01a5..811f11c5343 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java@@ -481,6 +481,72 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {values, fromDoc, toDoc, minValue, maxValue, bitSet, offset);}+ /**+ * Maps the raw query bounds {@code [minValue, maxValue]} into the encoded domain so the SIMD+ * kernel in {@link org.apache.lucene.internal.vectorization.DocValuesRangeSupport} can run+ * directly on packed values: {@code [encodedMin, encodedMax] = [ceil((min - delta) / mul),+ * floor((max - delta) / mul)]}. Open bounds (e.g. {@code Long.MIN_VALUE} or {@code+ * Long.MAX_VALUE}) are saturated to the encoded domain so they keep the SIMD path even when+ * {@code min - delta} or {@code max - delta} would overflow.+ */+ private static void rangeGcdDeltaIntoBitSet(+ LongValues values,+ int fromDoc,+ int toDoc,+ long minValue,+ long maxValue,+ long mul,+ long delta,+ FixedBitSet bitSet,+ int offset) {+ assert mul > 0;+ long encodedMin = saturatingShiftLower(minValue, delta);+ long encodedMax = saturatingShiftUpper(maxValue, delta);+ if (mul != 1) {+ // Math.ceilDiv / Math.floorDiv never overflow for mul > 0 (only Long.MIN_VALUE / -1 does),+ // so the SIMD path is always taken; no fallback to the per-doc decoded loop is required.+ encodedMin = Math.ceilDiv(encodedMin, mul);+ encodedMax = Math.floorDiv(encodedMax, mul);+ }+ encodedMin = Math.max(0, encodedMin);+ if (encodedMin <= encodedMax) {+ rangeIntoBitSet(values, fromDoc, toDoc, encodedMin, encodedMax, bitSet, offset);+ }+ }++ /**+ * Returns {@code minValue - delta}, saturating to {@code Long.MIN_VALUE} when the real value+ * would underflow (every non-negative stored value satisfies the lower bound) or to {@code+ * Long.MAX_VALUE} when it would overflow (no stored value can satisfy the lower bound). Stored+ * values are non-negative, so the caller can keep using the SIMD path with these saturated+ * sentinels.+ */+ private static long saturatingShiftLower(long minValue, long delta) {+ try {+ return Math.subtractExact(minValue, delta);+ } catch (+ @SuppressWarnings("unused")+ ArithmeticException overflow) {+ return delta > 0 ? Long.MIN_VALUE : Long.MAX_VALUE;+ }+ }++ /**+ * Symmetric counterpart of {@link #saturatingShiftLower}: returns {@code maxValue - delta},+ * saturating to {@code Long.MAX_VALUE} when the real value would overflow (every stored value+ * satisfies the upper bound) or to {@code Long.MIN_VALUE} when it would underflow (no stored+ * value can satisfy the upper bound).+ */+ private static long saturatingShiftUpper(long maxValue, long delta) {+ try {+ return Math.subtractExact(maxValue, delta);+ } catch (+ @SuppressWarnings("unused")+ ArithmeticException overflow) {+ return delta < 0 ? Long.MAX_VALUE : Long.MIN_VALUE;+ }+ }+private static int fixedCardinality(SortedNumericEntry entry, DocValuesSkipperEntry skipperEntry) {if (skipperEntry == null@@ -960,13 +1026,8 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {long maxValue,FixedBitSet bitSet,int offset) {- // Per-doc evaluation for gcd/delta encoded fields- for (int d = fromDoc; d < toDoc; d++) {- long v = mul * values.get(d) + delta;- if (v >= minValue && v <= maxValue) {- bitSet.set(d - offset);- }- }+ Lucene90DocValuesProducer.rangeGcdDeltaIntoBitSet(+ values, fromDoc, toDoc, minValue, maxValue, mul, delta, bitSet, offset);}};}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.javaindex 76e01cae13c..ccadb56b3af 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java@@ -19,6 +19,7 @@ package org.apache.lucene.search;import java.util.ArrayList;import java.util.List;import java.util.Random;+import java.util.function.LongUnaryOperator;import org.apache.lucene.codecs.lucene104.Lucene104Codec;import org.apache.lucene.document.Document;import org.apache.lucene.document.NumericDocValuesField;@@ -634,6 +635,117 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe}}+ public void testRangeIntoBitSetMatchesPerDocEvaluationWithDeltaEncoding() throws Exception {+ long delta = 1_000_000L;+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "delta-only encoded range must match decoded evaluation",+ values,+ delta + 127,+ delta + 4097);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWithGcdEncoding() throws Exception {+ long[] values = rangeValues(DOC_COUNT, doc -> doc * 1_000L);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "gcd encoded range must match decoded evaluation", values, 123_456L, 4_567_890L);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWithGcdAndDeltaEncoding() throws Exception {+ long delta = 1_000_000L;+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "gcd+delta encoded range must match decoded evaluation",+ values,+ delta + 123,+ delta + 456_789);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWhenGcdRangeFallsBetweenValues()+ throws Exception {+ long[] values = rangeValues(DOC_COUNT, doc -> 10L + doc * 5L);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "gcd encoded gap range must match no docs", values, 11, 14);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWithOpenLowerBound() throws Exception {+ long delta = 1_000_000L;+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "open Long.MIN_VALUE lower bound must saturate and match all docs up to the upper bound",+ values,+ Long.MIN_VALUE,+ delta + 127);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWithOpenUpperBound() throws Exception {+ long delta = 1_000_000L;+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "open Long.MAX_VALUE upper bound must saturate and match all docs from the lower bound",+ values,+ delta + 50L * 100L,+ Long.MAX_VALUE);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWithBothOpenBounds() throws Exception {+ long delta = 1_000_000L;+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "fully open range must match every doc", values, Long.MIN_VALUE, Long.MAX_VALUE);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWithNegativeEncodedLowerBound()+ throws Exception {+ // Stored values are non-negative (min == delta), but the query lower bound is below delta so+ // (minValue - delta) is negative even though Math.subtractExact succeeds. This exercises the+ // Math.max(0, encodedMin) clamp.+ long delta = 1_000_000L;+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "negative encoded lower bound must clamp to 0 and match docs up to the upper bound",+ values,+ delta - 50,+ delta + 100);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWithRangeBelowStoredValues()+ throws Exception {+ // The whole query range is below the stored minimum, so encodedMin > encodedMax after the+ // bound transformation and the SIMD path is skipped without iterating.+ long delta = 1_000_000L;+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "query range below stored values must match no docs", values, delta - 1_000L, delta - 1L);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWithRangeAboveStoredValues()+ throws Exception {+ long delta = 1_000_000L;+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);+ long max = delta + (DOC_COUNT - 1L) * 100L;+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "query range above stored values must match no docs", values, max + 1L, max + 1_000L);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWithFullGcdDeltaRange() throws Exception {+ long delta = 1_000_000L;+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "full gcd+delta encoded range must match all docs",+ values,+ delta,+ delta + (DOC_COUNT - 1L) * 100L);+ }++ public void testRangeIntoBitSetMatchesPerDocEvaluationWithSingleGcdDeltaValue() throws Exception {+ long delta = 1_000_000L;+ long value = delta + 123L * 100L;+ long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L);+ assertRangeIntoBitSetMatchesPerDocEvaluation(+ "single gcd+delta encoded value range must match one doc", values, value, value);+ }+public void testSortedNumericRangeIntoBitSetDenseFixedCardinality() throws Exception {doTestSortedNumericRangeIntoBitSet(true, true);}@@ -694,4 +806,47 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe}}}++ private static long[] rangeValues(int numDocs, LongUnaryOperator valueFunction) {+ long[] values = new long[numDocs];+ for (int i = 0; i < numDocs; i++) {+ values[i] = valueFunction.applyAsLong(i);+ }+ return values;+ }++ private void assertRangeIntoBitSetMatchesPerDocEvaluation(+ String message, long[] values, long rangeMin, long rangeMax) throws Exception {+ try (Directory dir = newDirectory()) {+ IndexWriterConfig iwc = new IndexWriterConfig().setCodec(new Lucene104Codec());+ try (IndexWriter w = new IndexWriter(dir, iwc)) {+ for (long value : values) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("val", value));+ w.addDocument(doc);+ }+ w.forceMerge(1);+ }++ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ LeafReaderContext ctx = reader.leaves().get(0);+ FixedBitSet expected = new FixedBitSet(values.length);+ NumericDocValues slowDv = ctx.reader().getNumericDocValues("val");+ for (int d = 0; d < values.length; d++) {+ if (slowDv.advanceExact(d)) {+ long value = slowDv.longValue();+ if (value >= rangeMin && value <= rangeMax) {+ expected.set(d);+ }+ }+ }++ FixedBitSet actual = new FixedBitSet(values.length);+ NumericDocValues fastDv = ctx.reader().getNumericDocValues("val");+ fastDv.rangeIntoBitSet(0, values.length, rangeMin, rangeMax, actual, 0);++ assertEquals(message, expected, actual);+ }+ }+ }}