Apache Lucene open-source search software
LUCENE-10405: binary and Sorted doc values are stored as BytesRef instead of BytesRefHash in memory index (#647)
Ignacio Vera4c578017af84
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 434ce56bf12..71e30cfc8ae 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -232,6 +232,9 @@ Bug Fixes* LUCENE-10407: Containing intervals could sometimes yield incorrect matches when wrappedin a disjunction. (Alan Woodward, Dawid Weiss)++* LUCENE-10405: When using the MemoryIndex, binary and Sorted doc values are stored+ as BytesRef instead of BytesRefHash so they don't have a limit on size. (Ignacio Vera)Other---------------------diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.javaindex cbc211473ec..71c408d9b8c 100644--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java@@ -578,17 +578,6 @@ public class MemoryIndex {info.numericProducer.dvLongValues[info.numericProducer.count++] = (long) docValuesValue;break;case BINARY:- if (info.binaryProducer.dvBytesValuesSet != null) {- throw new IllegalArgumentException(- "Only one value per field allowed for ["- + docValuesType- + "] doc values field ["- + fieldName- + "]");- }- info.binaryProducer.dvBytesValuesSet = new BytesRefHash(byteBlockPool);- info.binaryProducer.dvBytesValuesSet.add((BytesRef) docValuesValue);- break;case SORTED:if (info.binaryProducer.dvBytesValuesSet != null) {throw new IllegalArgumentException(@@ -598,14 +587,13 @@ public class MemoryIndex {+ fieldName+ "]");}- info.binaryProducer.dvBytesValuesSet = new BytesRefHash(byteBlockPool);- info.binaryProducer.dvBytesValuesSet.add((BytesRef) docValuesValue);+ info.binaryProducer.dvBytesValuesSet = ((BytesRef) docValuesValue).clone();break;case SORTED_SET:- if (info.binaryProducer.dvBytesValuesSet == null) {- info.binaryProducer.dvBytesValuesSet = new BytesRefHash(byteBlockPool);+ if (info.bytesRefHashProducer.dvBytesRefHashValuesSet == null) {+ info.bytesRefHashProducer.dvBytesRefHashValuesSet = new BytesRefHash(byteBlockPool);}- info.binaryProducer.dvBytesValuesSet.add((BytesRef) docValuesValue);+ info.bytesRefHashProducer.dvBytesRefHashValuesSet.add((BytesRef) docValuesValue);break;case NONE:default:@@ -866,6 +854,8 @@ public class MemoryIndex {/** the last offset encountered in this field for multi field support */private int lastOffset;+ private BytesRefHashDocValuesProducer bytesRefHashProducer;+private BinaryDocValuesProducer binaryProducer;private NumericDocValuesProducer numericProducer;@@ -884,7 +874,8 @@ public class MemoryIndex {this.fieldInfo = fieldInfo;this.sliceArray = new SliceByteStartArray(BytesRefHash.DEFAULT_CAPACITY);this.terms = new BytesRefHash(byteBlockPool, BytesRefHash.DEFAULT_CAPACITY, sliceArray);- ;++ this.bytesRefHashProducer = new BytesRefHashDocValuesProducer();this.binaryProducer = new BinaryDocValuesProducer();this.numericProducer = new NumericDocValuesProducer();}@@ -914,10 +905,8 @@ public class MemoryIndex {if (dvType == DocValuesType.NUMERIC || dvType == DocValuesType.SORTED_NUMERIC) {numericProducer.prepareForUsage();}- if (dvType == DocValuesType.BINARY- || dvType == DocValuesType.SORTED- || dvType == DocValuesType.SORTED_SET) {- binaryProducer.prepareForUsage();+ if (dvType == DocValuesType.SORTED_SET) {+ bytesRefHashProducer.prepareForUsage();}if (pointValues != null) {assert pointValues[0].bytes.length == pointValues[0].length@@ -1193,12 +1182,16 @@ public class MemoryIndex {}private static final class BinaryDocValuesProducer {+ BytesRef dvBytesValuesSet;+ }++ private static final class BytesRefHashDocValuesProducer {- BytesRefHash dvBytesValuesSet;+ BytesRefHash dvBytesRefHashValuesSet;int[] bytesIds;private void prepareForUsage() {- bytesIds = dvBytesValuesSet.sort();+ bytesIds = dvBytesRefHashValuesSet.sort();}}@@ -1316,8 +1309,7 @@ public class MemoryIndex {private SortedDocValues getSortedDocValues(String field, DocValuesType docValuesType) {Info info = getInfoForExpectedDocValuesType(field, docValuesType);if (info != null) {- BytesRef value = info.binaryProducer.dvBytesValuesSet.get(0, new BytesRef());- return sortedDocValues(value);+ return sortedDocValues(info.binaryProducer.dvBytesValuesSet);} else {return null;}@@ -1338,7 +1330,7 @@ public class MemoryIndex {Info info = getInfoForExpectedDocValuesType(field, DocValuesType.SORTED_SET);if (info != null) {return sortedSetDocValues(- info.binaryProducer.dvBytesValuesSet, info.binaryProducer.bytesIds);+ info.bytesRefHashProducer.dvBytesRefHashValuesSet, info.bytesRefHashProducer.bytesIds);} else {return null;}diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.javaindex f241693c6bf..4ce329a62ea 100644--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java@@ -430,6 +430,30 @@ public class TestMemoryIndex extends LuceneTestCase {assertEquals("quick brown fox", binaryDocValues.binaryValue().utf8ToString());}+ public void testBigBinaryDocValues() throws Exception {+ Document doc = new Document();+ byte[] bytes = new byte[33 * 1024];+ random().nextBytes(bytes);+ doc.add(new BinaryDocValuesField("binary", new BytesRef(bytes)));+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer, true, true);+ LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();+ BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("binary");+ assertEquals(0, binaryDocValues.nextDoc());+ assertArrayEquals(bytes, binaryDocValues.binaryValue().bytes);+ }++ public void testBigSortedDocValues() throws Exception {+ Document doc = new Document();+ byte[] bytes = new byte[33 * 1024];+ random().nextBytes(bytes);+ doc.add(new SortedDocValuesField("binary", new BytesRef(bytes)));+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer, true, true);+ LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();+ SortedDocValues sortedDocValues = leafReader.getSortedDocValues("binary");+ assertEquals(0, sortedDocValues.nextDoc());+ assertArrayEquals(bytes, sortedDocValues.lookupOrd(0).bytes);+ }+public void testPointValues() throws Exception {List<Function<Long, IndexableField>> fieldFunctions =Arrays.asList(