Skip to content

Apache Lucene open-source search software

Specialize ordinal encoding for SortedSetDocValues (#1010)

Greg Miller7c35311f291e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d834b43f211..41c5679757a 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -132,6 +132,8 @@ Optimizations
* GITHUB#984: Use primitive type data structures in FloatTaxonomyFacets and IntTaxonomyFacets
#getAllChildren() internal implementation to avoid some garbage creation. (Greg Miller)
+* GITHUB#1010: Specialize ordinal encoding for common case in SortedSetDocValues. (Greg Miller)
+
Changes in runtime behavior
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
index 60f566a888d..c3d170447ae 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
@@ -699,12 +699,12 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
throws IOException {
meta.writeInt(field.number);
meta.writeByte(Lucene90DocValuesFormat.SORTED_NUMERIC);
- doAddSortedNumericField(field, valuesProducer);
+ doAddSortedNumericField(field, valuesProducer, false);
}
- private void doAddSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer)
- throws IOException {
- long[] stats = writeValues(field, valuesProducer, false);
+ private void doAddSortedNumericField(
+ FieldInfo field, DocValuesProducer valuesProducer, boolean ords) throws IOException {
+ long[] stats = writeValues(field, valuesProducer, ords);
int numDocsWithField = Math.toIntExact(stats[0]);
long numValues = stats[1];
assert numValues >= numDocsWithField;
@@ -825,7 +825,8 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
}
};
}
- });
+ },
+ true);
addTermsDict(valuesProducer.getSortedSet(field));
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
index a7146f26fe3..6d3e03b32ab 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
@@ -1432,7 +1432,152 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
return DocValues.singleton(getSorted(entry.singleValueEntry));
}
- final SortedNumericDocValues ords = getSortedNumeric(entry.ordsEntry);
+ // Specialize the common case for ordinals: single block of packed integers.
+ SortedNumericEntry ordsEntry = entry.ordsEntry;
+ if (ordsEntry.blockShift < 0 && ordsEntry.bitsPerValue > 0) {
+ if (ordsEntry.gcd != 1 || ordsEntry.minValue != 0 || ordsEntry.table != null) {
+ throw new IllegalStateException("Ordinals shouldn't use GCD, offset or table compression");
+ }
+
+ final RandomAccessInput addressesInput =
+ data.randomAccessSlice(ordsEntry.addressesOffset, ordsEntry.addressesLength);
+ final LongValues addresses =
+ DirectMonotonicReader.getInstance(ordsEntry.addressesMeta, addressesInput);
+
+ final RandomAccessInput slice =
+ data.randomAccessSlice(ordsEntry.valuesOffset, ordsEntry.valuesLength);
+ final LongValues values = DirectReader.getInstance(slice, ordsEntry.bitsPerValue);
+
+ if (ordsEntry.docsWithFieldOffset == -1) { // dense
+ return new BaseSortedSetDocValues(entry, data) {
+
+ private final int maxDoc = Lucene90DocValuesProducer.this.maxDoc;
+ private int doc = -1;
+ private long start;
+ private long end;
+ private int count;
+
+ @Override
+ public long nextOrd() throws IOException {
+ if (start == end) {
+ return NO_MORE_ORDS;
+ }
+ return values.get(start++);
+ }
+
+ @Override
+ public boolean advanceExact(int target) throws IOException {
+ start = addresses.get(target);
+ end = addresses.get(target + 1L);
+ count = (int) (end - start);
+ doc = target;
+ return true;
+ }
+
+ @Override
+ public int docValueCount() {
+ return count;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return advance(doc + 1);
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target >= maxDoc) {
+ return doc = NO_MORE_DOCS;
+ }
+ start = addresses.get(target);
+ end = addresses.get(target + 1L);
+ count = (int) (end - start);
+ return doc = target;
+ }
+
+ @Override
+ public long cost() {
+ return maxDoc;
+ }
+ };
+ } else if (ordsEntry.docsWithFieldOffset >= 0) { // sparse but non-empty
+ final IndexedDISI disi =
+ new IndexedDISI(
+ data,
+ ordsEntry.docsWithFieldOffset,
+ ordsEntry.docsWithFieldLength,
+ ordsEntry.jumpTableEntryCount,
+ ordsEntry.denseRankPower,
+ ordsEntry.numValues);
+
+ return new BaseSortedSetDocValues(entry, data) {
+
+ boolean set;
+ long start, end;
+ int count;
+
+ @Override
+ public long nextOrd() throws IOException {
+ set();
+ if (start == end) {
+ return NO_MORE_ORDS;
+ }
+ return values.get(start++);
+ }
+
+ @Override
+ public boolean advanceExact(int target) throws IOException {
+ set = false;
+ return disi.advanceExact(target);
+ }
+
+ @Override
+ public int docValueCount() {
+ set();
+ return count;
+ }
+
+ @Override
+ public int docID() {
+ return disi.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ set = false;
+ return disi.nextDoc();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ set = false;
+ return disi.advance(target);
+ }
+
+ @Override
+ public long cost() {
+ return disi.cost();
+ }
+
+ private void set() {
+ if (set == false) {
+ final int index = disi.index();
+ start = addresses.get(index);
+ end = addresses.get(index + 1L);
+ count = (int) (end - start);
+ set = true;
+ }
+ }
+ };
+ }
+ }
+
+ final SortedNumericDocValues ords = getSortedNumeric(ordsEntry);
return new BaseSortedSetDocValues(entry, data) {
int i = 0;