Apache Lucene open-source search software
Move parent field from DWPT to IndexingChain (#15960)
Tim Brooks34c9495241dc
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 66e89f00a6c..0bf409d80bb 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -394,6 +394,8 @@ Other* GITHUB#15951: Fix WindowsFS onClose race condition (Szymon Bialkowski)+* GITHUB#15960: Move parent field from DWPT to IndexingChain. (Tim Brooks)+======================= Lucene 10.4.0 =======================API Changesdiff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.javaindex 8b81a6a9b7f..e4cd3f04328 100644--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java@@ -24,7 +24,6 @@ import java.util.HashSet;import java.util.Iterator;import java.util.List;import java.util.Locale;-import java.util.NoSuchElementException;import java.util.Objects;import java.util.Set;import java.util.concurrent.TimeUnit;@@ -33,7 +32,6 @@ import java.util.concurrent.locks.Condition;import java.util.concurrent.locks.Lock;import java.util.concurrent.locks.ReentrantLock;import org.apache.lucene.codecs.Codec;-import org.apache.lucene.document.NumericDocValuesField;import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.store.Directory;@@ -140,7 +138,7 @@ final class DocumentsWriterPerThread implements Accountable, Lock {private int[] deleteDocIDs = new int[0];private int numDeletedDocIds = 0;private final int indexMajorVersionCreated;- private final IndexingChain.ReservedField<NumericDocValuesField> parentField;+ private final boolean hasParentField;DocumentsWriterPerThread(int indexMajorVersionCreated,@@ -197,13 +195,7 @@ final class DocumentsWriterPerThread implements Accountable, Lock {fieldInfos,indexWriterConfig,this::onAbortingException);- if (indexWriterConfig.getParentField() != null) {- this.parentField =- indexingChain.markAsReserved(- new NumericDocValuesField(indexWriterConfig.getParentField(), -1));- } else {- this.parentField = null;- }+ this.hasParentField = indexWriterConfig.getParentField() != null;}final void testPoint(String message) {@@ -249,12 +241,10 @@ final class DocumentsWriterPerThread implements Accountable, Lock {final Iterator<? extends Iterable<? extends IndexableField>> iterator = docs.iterator();while (iterator.hasNext()) {Iterable<? extends IndexableField> doc = iterator.next();- if (parentField != null) {- if (iterator.hasNext() == false) {- doc = addParentField(doc, parentField);- }- } else if (segmentInfo.getIndexSort() != null- && iterator.hasNext()+ final boolean isLastDoc = iterator.hasNext() == false;+ if (hasParentField == false+ && segmentInfo.getIndexSort() != null+ && isLastDoc == false&& indexMajorVersionCreated >= Version.LUCENE_10_0_0.major) {// sort is configured but parent field is missing, yet we have a doc-block// yet we must not fail if this index was created in an earlier version where this@@ -271,7 +261,7 @@ final class DocumentsWriterPerThread implements Accountable, Lock {// vs non-aborting exceptions):reserveOneDoc();try {- indexingChain.processDocument(numDocsInRAM++, doc);+ indexingChain.processDocument(numDocsInRAM++, doc, isLastDoc);} finally {onNewDocOnRAM.run();}@@ -294,34 +284,6 @@ final class DocumentsWriterPerThread implements Accountable, Lock {}}- private Iterable<? extends IndexableField> addParentField(- Iterable<? extends IndexableField> doc, IndexableField parentField) {- return () -> {- final Iterator<? extends IndexableField> first = doc.iterator();- return new Iterator<>() {- IndexableField additionalField = parentField;-- @Override- public boolean hasNext() {- return additionalField != null || first.hasNext();- }-- @Override- public IndexableField next() {- if (additionalField != null) {- IndexableField field = additionalField;- additionalField = null;- return field;- }- if (first.hasNext()) {- return first.next();- }- throw new NoSuchElementException();- }- };- };- }-private long finishDocuments(DocumentsWriterDeleteQueue.Node<?> deleteNode, int docIdUpTo) {/** here we actually finish the document in two steps 1. push the delete intodiff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.javaindex 2edefc662a5..f1f94b0db64 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java@@ -18,7 +18,6 @@ package org.apache.lucene.index;import java.io.Closeable;import java.io.IOException;-import java.io.Reader;import java.util.ArrayList;import java.util.Arrays;import java.util.Collection;@@ -40,9 +39,9 @@ import org.apache.lucene.codecs.NormsProducer;import org.apache.lucene.codecs.PointsFormat;import org.apache.lucene.codecs.PointsWriter;import org.apache.lucene.document.FieldType;-import org.apache.lucene.document.InvertableType;import org.apache.lucene.document.KnnByteVectorField;import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.document.NumericDocValuesField;import org.apache.lucene.document.StoredValue;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.Sort;@@ -95,6 +94,8 @@ final class IndexingChain implements Accountable {private final LiveIndexWriterConfig indexWriterConfig;private final int indexCreatedVersionMajor;private final Consumer<Throwable> abortingExceptionConsumer;+ private final PerField parentPf;+ private final NumericDocValuesField parentField;private boolean hasHitAbortingException;IndexingChain(@@ -140,6 +141,14 @@ final class IndexingChain implements Accountable {new FreqProxTermsWriter(intBlockAllocator, byteBlockAllocator, bytesUsed, termVectorsWriter);docValuesBytePool = new ByteBlockPool(byteBlockAllocator);+ if (indexWriterConfig.getParentField() != null) {+ this.parentField = new NumericDocValuesField(indexWriterConfig.getParentField(), -1);+ parentPf = getOrAddPerField(this.parentField.name());+ updateDocFieldSchema(this.parentField.name(), parentPf.schema, this.parentField.fieldType());+ } else {+ this.parentField = null;+ this.parentPf = null;+ }}private void onAbortingException(Throwable th) {@@ -549,7 +558,9 @@ final class IndexingChain implements Accountable {}}- void processDocument(int docID, Iterable<? extends IndexableField> document) throws IOException {+ void processDocument(+ int docID, Iterable<? extends IndexableField> document, boolean lastDocInBlock)+ throws IOException {// number of unique fields by name which need to be init in segment or full validationint fieldsNeedInitOrValidate = 0;int indexedFieldCount = 0; // number of unique fields indexed with postings@@ -565,17 +576,23 @@ final class IndexingChain implements Accountable {termsHash.startDocument();startStoredFields(docID);try {+ // Handle the parent field first (before document fields). Its schema was already+ // set up in the constructor, so we only need to set the docID and trigger+ // initializeFieldInfo on the first encounter in this segment.+ if (parentPf != null && lastDocInBlock) {+ parentPf.schema.resetJustDocId(docID);+ if (parentPf.fieldInfo == null) {+ fields[fieldsNeedInitOrValidate++] = parentPf;+ }+ }+// 1st pass over doc fields – verify that doc schema matches the index schema// build schema for each unique doc fieldfor (IndexableField field : document) {final String fieldName = field.name();final IndexableFieldType fieldType = field.fieldType();- final boolean isReserved = field.getClass() == ReservedField.class;- PerField pf =- getOrAddPerField(- fieldName, false- /* we never add reserved fields during indexing should be done during DWPT setup*/ );- if (pf.reserved != isReserved) {+ PerField pf = getOrAddPerField(fieldName);+ if (pf == parentPf) {throw new IllegalArgumentException("\"" + fieldName + "\" is a reserved field and should not be added to any document");}@@ -602,8 +619,16 @@ final class IndexingChain implements Accountable {initAndValidateFields(fieldsNeedInitOrValidate);}- // 2nd pass over doc fields – index each field- // also count the number of unique fields indexed with postings+ // 2nd pass – index parent field first, then document fields+ if (parentPf != null && lastDocInBlock) {+ // parentField is currently a NumericDocValuesField so processField always returns false+ // here, but we check defensively in case the parent field representation changes.+ if (processField(docID, parentField, parentPf)) {+ fields[indexedFieldCount] = parentPf;+ indexedFieldCount++;+ }+ }+ // 2nd pass – document fieldsdocFieldIdx = 0;for (IndexableField field : document) {if (processField(docID, field, docFields[docFieldIdx])) {@@ -792,7 +817,7 @@ final class IndexingChain implements Accountable {* Returns a previously created {@link PerField}, absorbing the type information from {@link* FieldType}, and creates a new {@link PerField} if this field name wasn't seen yet.*/- private PerField getOrAddPerField(String fieldName, boolean reserved) {+ private PerField getOrAddPerField(String fieldName) {final int hashPos = fieldName.hashCode() & hashMask;PerField pf = fieldHash[hashPos];while (pf != null && pf.fieldName.equals(fieldName) == false) {@@ -808,8 +833,7 @@ final class IndexingChain implements Accountable {schema,indexWriterConfig.getSimilarity(),indexWriterConfig.getInfoStream(),- indexWriterConfig.getAnalyzer(),- reserved);+ indexWriterConfig.getAnalyzer());pf.next = fieldHash[hashPos];fieldHash[hashPos] = pf;totalFieldCount++;@@ -1083,7 +1107,6 @@ final class IndexingChain implements Accountable {final String fieldName;final int indexCreatedVersionMajor;final FieldSchema schema;- final boolean reserved;FieldInfo fieldInfo;final Similarity similarity;@@ -1129,15 +1152,13 @@ final class IndexingChain implements Accountable {FieldSchema schema,Similarity similarity,InfoStream infoStream,- Analyzer analyzer,- boolean reserved) {+ Analyzer analyzer) {this.fieldName = fieldName;this.indexCreatedVersionMajor = indexCreatedVersionMajor;this.schema = schema;this.similarity = similarity;this.infoStream = infoStream;this.analyzer = analyzer;- this.reserved = reserved;}void reset(int docId, IndexableFieldType fieldType) {@@ -1618,77 +1639,4 @@ final class IndexingChain implements Accountable {assertSame("point num bytes", fi.getPointNumBytes(), pointNumBytes);}}-- /**- * Wraps the given field in a reserved field and registers it as reserved. Only DWPT should do- * this to mark fields as private / reserved to prevent this fieldname to be used from the outside- * of the IW / DWPT eco-system- */- <T extends IndexableField> ReservedField<T> markAsReserved(T field) {- getOrAddPerField(field.name(), true);- return new ReservedField<>(field);- }-- static final class ReservedField<T extends IndexableField> implements IndexableField {-- private final T delegate;-- private ReservedField(T delegate) {- this.delegate = delegate;- }-- T getDelegate() {- return delegate;- }-- @Override- public String name() {- return delegate.name();- }-- @Override- public IndexableFieldType fieldType() {- return delegate.fieldType();- }-- @Override- public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) {- return delegate.tokenStream(analyzer, reuse);- }-- @Override- public BytesRef binaryValue() {- return delegate.binaryValue();- }-- @Override- public String stringValue() {- return delegate.stringValue();- }-- @Override- public CharSequence getCharSequenceValue() {- return delegate.getCharSequenceValue();- }-- @Override- public Reader readerValue() {- return delegate.readerValue();- }-- @Override- public Number numericValue() {- return delegate.numericValue();- }-- @Override- public StoredValue storedValue() {- return delegate.storedValue();- }-- @Override- public InvertableType invertableType() {- return delegate.invertableType();- }- }}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.javaindex d8b46a9a5b6..1cf401e279f 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java@@ -4950,6 +4950,45 @@ public class TestIndexWriter extends LuceneTestCase {}}+ public void testSingleDocBlockWritesParentField() throws IOException {+ try (Directory dir = newDirectory()) {+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));+ iwc.setParentField("parent");+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {+ // Single-document "block" — the lone doc is both first and last in the block+ Document single = new Document();+ single.add(new StringField("id", "s0", Field.Store.YES));+ writer.addDocuments(List.of(single));++ // Multi-document block for comparison+ Document child = new Document();+ child.add(new StringField("id", "c0", Field.Store.YES));+ Document parent = new Document();+ parent.add(new StringField("id", "p0", Field.Store.YES));+ writer.addDocuments(List.of(child, parent));+ }++ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ // 3 docs total: doc0=single, doc1=child, doc2=parent+ assertEquals(3, reader.numDocs());+ LeafReader leaf = reader.leaves().get(0).reader();+ NumericDocValues parentDV = leaf.getNumericDocValues("parent");+ assertNotNull(parentDV);++ // doc 0 (the single-doc block) should be a parent+ assertEquals(0, parentDV.nextDoc());+ assertEquals(-1, parentDV.longValue());++ // doc 2 (last doc of the multi-doc block) should be a parent+ assertEquals(2, parentDV.nextDoc());+ assertEquals(-1, parentDV.longValue());++ // no more parents+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, parentDV.nextDoc());+ }+ }+ }+public void testDocValuesMixedSkippingIndex() throws Exception {try (Directory dir = newDirectory()) {try (IndexWriter writer =