Apache Lucene open-source search software
Add support for stored fields to MemoryIndex (#11999)
Alan Woodward66127f6e6963
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex ff85eeb8fd7..2b4836a8c41 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -138,6 +138,8 @@ New Features* GITHUB#11929: MMapDirectory gives more granular control on which files topreload. (Adrien Grand, Uwe Schindler)+* GITHUB#11999: MemoryIndex now supports stored fields. (Alan Woodward)+Improvements---------------------* GITHUB#11778: Detailed part-of-speech information for particle(조사) and ending(어미) on Noridiff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.javaindex 75d6217fa2e..4925dfb8ac0 100644--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java@@ -17,10 +17,12 @@package org.apache.lucene.index.memory;import java.io.IOException;+import java.util.ArrayList;import java.util.Arrays;import java.util.Collection;import java.util.Collections;import java.util.Iterator;+import java.util.List;import java.util.Map;import java.util.Objects;import java.util.SortedMap;@@ -416,6 +418,10 @@ public class MemoryIndex {if (field.fieldType().pointDimensionCount() > 0) {storePointValues(info, field.binaryValue());}++ if (field.fieldType().stored()) {+ storeValues(info, field);+ }}/**@@ -527,6 +533,26 @@ public class MemoryIndex {info.pointValues[info.pointValuesCount++] = BytesRef.deepCopyOf(pointValue);}+ private void storeValues(Info info, IndexableField field) {+ if (info.storedValues == null) {+ info.storedValues = new ArrayList<>();+ }+ BytesRef binaryValue = field.binaryValue();+ if (binaryValue != null) {+ info.storedValues.add(binaryValue);+ return;+ }+ Number numberValue = field.numericValue();+ if (numberValue != null) {+ info.storedValues.add(numberValue);+ return;+ }+ String stringValue = field.stringValue();+ if (stringValue != null) {+ info.storedValues.add(stringValue);+ }+ }+private void storeDocValues(Info info, DocValuesType docValuesType, Object docValuesValue) {String fieldName = info.fieldInfo.name;DocValuesType existingDocValuesType = info.fieldInfo.getDocValuesType();@@ -875,6 +901,8 @@ public class MemoryIndex {private boolean preparedDocValuesAndPointValues;+ private List<Object> storedValues;+private BytesRef[] pointValues;private byte[] minPackedValue;@@ -1797,9 +1825,34 @@ public class MemoryIndex {}@Override- public void document(int docID, StoredFieldVisitor visitor) {+ public void document(int docID, StoredFieldVisitor visitor) throws IOException {if (DEBUG) System.err.println("MemoryIndexReader.document");- // no-op: there are no stored fields+ for (Info info : fields.values()) {+ StoredFieldVisitor.Status status = visitor.needsField(info.fieldInfo);+ if (status == StoredFieldVisitor.Status.STOP) {+ return;+ }+ if (status == StoredFieldVisitor.Status.NO) {+ continue;+ }+ if (info.storedValues != null) {+ for (Object value : info.storedValues) {+ if (value instanceof BytesRef bytes) {+ visitor.binaryField(info.fieldInfo, BytesRef.deepCopyOf(bytes).bytes);+ } else if (value instanceof Double d) {+ visitor.doubleField(info.fieldInfo, d);+ } else if (value instanceof Float f) {+ visitor.floatField(info.fieldInfo, f);+ } else if (value instanceof Long l) {+ visitor.longField(info.fieldInfo, l);+ } else if (value instanceof Integer i) {+ visitor.intField(info.fieldInfo, i);+ } else if (value instanceof String s) {+ visitor.stringField(info.fieldInfo, s);+ }+ }+ }+ }}@Overridediff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.javaindex 8fcb80f343a..4913e3f7cba 100644--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java@@ -22,12 +22,16 @@ import static org.hamcrest.core.StringContains.containsString;import java.io.IOException;import java.nio.charset.StandardCharsets;+import java.util.ArrayList;import java.util.Arrays;+import java.util.Collections;import java.util.List;+import java.util.Objects;import java.util.function.BiFunction;import java.util.function.Function;import java.util.stream.LongStream;import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.analysis.standard.StandardAnalyzer;import org.apache.lucene.document.BinaryDocValuesField;import org.apache.lucene.document.BinaryPoint;import org.apache.lucene.document.Document;@@ -687,4 +691,106 @@ public class TestMemoryIndex extends LuceneTestCase {+ "fields=2, terms=2, positions=3",mi.toStringDebug());}++ public void testStoredFields() throws IOException {+ List<IndexableField> fields = new ArrayList<>();+ fields.add(new StoredField("float", 1.5f));+ fields.add(new StoredField("multifloat", 2.5f));+ fields.add(new StoredField("multifloat", 3.5f));+ fields.add(new StoredField("long", 10L));+ fields.add(new StoredField("multilong", 15L));+ fields.add(new StoredField("multilong", 20L));+ fields.add(new StoredField("int", 1.7));+ fields.add(new StoredField("multiint", 2.7));+ fields.add(new StoredField("multiint", 2.8));+ fields.add(new StoredField("multiint", 2.9));+ fields.add(new StoredField("double", 3.7d));+ fields.add(new StoredField("multidouble", 4.5d));+ fields.add(new StoredField("multidouble", 4.6d));+ fields.add(new StoredField("multidouble", 4.7d));+ fields.add(new StoredField("string", "foo"));+ fields.add(new StoredField("multistring", "bar"));+ fields.add(new StoredField("multistring", "baz"));+ fields.add(new StoredField("binary", "bfoo".getBytes(StandardCharsets.UTF_8)));+ fields.add(new StoredField("multibinary", "bbar".getBytes(StandardCharsets.UTF_8)));+ fields.add(new StoredField("multibinary", "bbaz".getBytes(StandardCharsets.UTF_8)));++ Collections.shuffle(fields, random());+ Document doc = new Document();+ for (IndexableField f : fields) {+ doc.add(f);+ }++ MemoryIndex mi = MemoryIndex.fromDocument(doc, new StandardAnalyzer());+ Document d = mi.createSearcher().doc(0);++ assertContains(d, "long", 10L, IndexableField::numericValue);+ assertContains(d, "int", 1.7, IndexableField::numericValue);+ assertContains(d, "double", 3.7d, IndexableField::numericValue);+ assertContains(d, "float", 1.5f, IndexableField::numericValue);+ assertContains(d, "string", "foo", IndexableField::stringValue);+ assertBinaryContains(d, "binary", new BytesRef("bfoo"));++ assertMultiContains(d, "multilong", new Object[] {15L, 20L}, IndexableField::numericValue);+ assertMultiContains(d, "multiint", new Object[] {2.7, 2.8, 2.9}, IndexableField::numericValue);+ assertMultiContains(+ d, "multidouble", new Object[] {4.5d, 4.6d, 4.7d}, IndexableField::numericValue);+ assertMultiContains(d, "multifloat", new Object[] {2.5f, 3.5f}, IndexableField::numericValue);+ assertMultiContains(d, "multistring", new Object[] {"bar", "baz"}, IndexableField::stringValue);+ assertBinaryMultiContains(+ d, "multibinary", new BytesRef[] {new BytesRef("bbar"), new BytesRef("bbaz")});+ }++ private static void assertContains(+ Document d, String field, Object expected, Function<IndexableField, Object> value) {+ assertNotNull(d.getField(field));+ IndexableField[] fields = d.getFields(field);+ assertEquals(1, fields.length);+ assertEquals(expected, value.apply(fields[0]));+ }++ private static void assertBinaryContains(Document d, String field, BytesRef expected) {+ assertNotNull(d.getField(field));+ IndexableField[] fields = d.getFields(field);+ assertEquals(1, fields.length);+ assertEquals(0, expected.compareTo(fields[0].binaryValue()));+ }++ private static void assertMultiContains(+ Document d, String field, Object[] expected, Function<IndexableField, Object> value) {+ assertNotNull(d.get(field));+ IndexableField[] fields = d.getFields(field);+ assertEquals(expected.length, fields.length);+ for (IndexableField f : fields) {+ Object actual = value.apply(f);+ assertTrue(arrayContains(expected, actual));+ }+ }++ private static void assertBinaryMultiContains(Document d, String field, BytesRef[] expected) {+ IndexableField[] fields = d.getFields(field);+ assertEquals(expected.length, fields.length);+ for (IndexableField f : fields) {+ BytesRef actual = f.binaryValue();+ assertTrue(arrayBinaryContains(expected, actual));+ }+ }++ private static boolean arrayContains(Object[] array, Object value) {+ for (Object o : array) {+ if (Objects.equals(o, value)) {+ return true;+ }+ }+ return false;+ }++ private static boolean arrayBinaryContains(BytesRef[] array, BytesRef value) {+ for (BytesRef b : array) {+ if (b.compareTo(value) == 0) {+ return true;+ }+ }+ return false;+ }}