Skip to content

Apache Lucene open-source search software

Move parent field from DWPT to IndexingChain (#15960)

Tim Brooks34c9495241dc
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 66e89f00a6c..0bf409d80bb 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -394,6 +394,8 @@ Other
* GITHUB#15951: Fix WindowsFS onClose race condition (Szymon Bialkowski)
+* GITHUB#15960: Move parent field from DWPT to IndexingChain. (Tim Brooks)
+
======================= Lucene 10.4.0 =======================
API Changes
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
index 8b81a6a9b7f..e4cd3f04328 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
@@ -24,7 +24,6 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
-import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -33,7 +32,6 @@ import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
@@ -140,7 +138,7 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
private int[] deleteDocIDs = new int[0];
private int numDeletedDocIds = 0;
private final int indexMajorVersionCreated;
- private final IndexingChain.ReservedField<NumericDocValuesField> parentField;
+ private final boolean hasParentField;
DocumentsWriterPerThread(
int indexMajorVersionCreated,
@@ -197,13 +195,7 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
fieldInfos,
indexWriterConfig,
this::onAbortingException);
- if (indexWriterConfig.getParentField() != null) {
- this.parentField =
- indexingChain.markAsReserved(
- new NumericDocValuesField(indexWriterConfig.getParentField(), -1));
- } else {
- this.parentField = null;
- }
+ this.hasParentField = indexWriterConfig.getParentField() != null;
}
final void testPoint(String message) {
@@ -249,12 +241,10 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
final Iterator<? extends Iterable<? extends IndexableField>> iterator = docs.iterator();
while (iterator.hasNext()) {
Iterable<? extends IndexableField> doc = iterator.next();
- if (parentField != null) {
- if (iterator.hasNext() == false) {
- doc = addParentField(doc, parentField);
- }
- } else if (segmentInfo.getIndexSort() != null
- && iterator.hasNext()
+ final boolean isLastDoc = iterator.hasNext() == false;
+ if (hasParentField == false
+ && segmentInfo.getIndexSort() != null
+ && isLastDoc == false
&& indexMajorVersionCreated >= Version.LUCENE_10_0_0.major) {
// sort is configured but parent field is missing, yet we have a doc-block
// yet we must not fail if this index was created in an earlier version where this
@@ -271,7 +261,7 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
// vs non-aborting exceptions):
reserveOneDoc();
try {
- indexingChain.processDocument(numDocsInRAM++, doc);
+ indexingChain.processDocument(numDocsInRAM++, doc, isLastDoc);
} finally {
onNewDocOnRAM.run();
}
@@ -294,34 +284,6 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
}
}
- private Iterable<? extends IndexableField> addParentField(
- Iterable<? extends IndexableField> doc, IndexableField parentField) {
- return () -> {
- final Iterator<? extends IndexableField> first = doc.iterator();
- return new Iterator<>() {
- IndexableField additionalField = parentField;
-
- @Override
- public boolean hasNext() {
- return additionalField != null || first.hasNext();
- }
-
- @Override
- public IndexableField next() {
- if (additionalField != null) {
- IndexableField field = additionalField;
- additionalField = null;
- return field;
- }
- if (first.hasNext()) {
- return first.next();
- }
- throw new NoSuchElementException();
- }
- };
- };
- }
-
private long finishDocuments(DocumentsWriterDeleteQueue.Node<?> deleteNode, int docIdUpTo) {
/*
* here we actually finish the document in two steps 1. push the delete into
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index 2edefc662a5..f1f94b0db64 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
-import java.io.Reader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -40,9 +39,9 @@ import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PointsWriter;
import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.InvertableType;
import org.apache.lucene.document.KnnByteVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredValue;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Sort;
@@ -95,6 +94,8 @@ final class IndexingChain implements Accountable {
private final LiveIndexWriterConfig indexWriterConfig;
private final int indexCreatedVersionMajor;
private final Consumer<Throwable> abortingExceptionConsumer;
+ private final PerField parentPf;
+ private final NumericDocValuesField parentField;
private boolean hasHitAbortingException;
IndexingChain(
@@ -140,6 +141,14 @@ final class IndexingChain implements Accountable {
new FreqProxTermsWriter(
intBlockAllocator, byteBlockAllocator, bytesUsed, termVectorsWriter);
docValuesBytePool = new ByteBlockPool(byteBlockAllocator);
+ if (indexWriterConfig.getParentField() != null) {
+ this.parentField = new NumericDocValuesField(indexWriterConfig.getParentField(), -1);
+ parentPf = getOrAddPerField(this.parentField.name());
+ updateDocFieldSchema(this.parentField.name(), parentPf.schema, this.parentField.fieldType());
+ } else {
+ this.parentField = null;
+ this.parentPf = null;
+ }
}
private void onAbortingException(Throwable th) {
@@ -549,7 +558,9 @@ final class IndexingChain implements Accountable {
}
}
- void processDocument(int docID, Iterable<? extends IndexableField> document) throws IOException {
+ void processDocument(
+ int docID, Iterable<? extends IndexableField> document, boolean lastDocInBlock)
+ throws IOException {
// number of unique fields by name which need to be init in segment or full validation
int fieldsNeedInitOrValidate = 0;
int indexedFieldCount = 0; // number of unique fields indexed with postings
@@ -565,17 +576,23 @@ final class IndexingChain implements Accountable {
termsHash.startDocument();
startStoredFields(docID);
try {
+ // Handle the parent field first (before document fields). Its schema was already
+ // set up in the constructor, so we only need to set the docID and trigger
+ // initializeFieldInfo on the first encounter in this segment.
+ if (parentPf != null && lastDocInBlock) {
+ parentPf.schema.resetJustDocId(docID);
+ if (parentPf.fieldInfo == null) {
+ fields[fieldsNeedInitOrValidate++] = parentPf;
+ }
+ }
+
// 1st pass over doc fields – verify that doc schema matches the index schema
// build schema for each unique doc field
for (IndexableField field : document) {
final String fieldName = field.name();
final IndexableFieldType fieldType = field.fieldType();
- final boolean isReserved = field.getClass() == ReservedField.class;
- PerField pf =
- getOrAddPerField(
- fieldName, false
- /* we never add reserved fields during indexing should be done during DWPT setup*/ );
- if (pf.reserved != isReserved) {
+ PerField pf = getOrAddPerField(fieldName);
+ if (pf == parentPf) {
throw new IllegalArgumentException(
"\"" + fieldName + "\" is a reserved field and should not be added to any document");
}
@@ -602,8 +619,16 @@ final class IndexingChain implements Accountable {
initAndValidateFields(fieldsNeedInitOrValidate);
}
- // 2nd pass over doc fields – index each field
- // also count the number of unique fields indexed with postings
+ // 2nd pass – index parent field first, then document fields
+ if (parentPf != null && lastDocInBlock) {
+ // parentField is currently a NumericDocValuesField so processField always returns false
+ // here, but we check defensively in case the parent field representation changes.
+ if (processField(docID, parentField, parentPf)) {
+ fields[indexedFieldCount] = parentPf;
+ indexedFieldCount++;
+ }
+ }
+ // 2nd pass – document fields
docFieldIdx = 0;
for (IndexableField field : document) {
if (processField(docID, field, docFields[docFieldIdx])) {
@@ -792,7 +817,7 @@ final class IndexingChain implements Accountable {
* Returns a previously created {@link PerField}, absorbing the type information from {@link
* FieldType}, and creates a new {@link PerField} if this field name wasn't seen yet.
*/
- private PerField getOrAddPerField(String fieldName, boolean reserved) {
+ private PerField getOrAddPerField(String fieldName) {
final int hashPos = fieldName.hashCode() & hashMask;
PerField pf = fieldHash[hashPos];
while (pf != null && pf.fieldName.equals(fieldName) == false) {
@@ -808,8 +833,7 @@ final class IndexingChain implements Accountable {
schema,
indexWriterConfig.getSimilarity(),
indexWriterConfig.getInfoStream(),
- indexWriterConfig.getAnalyzer(),
- reserved);
+ indexWriterConfig.getAnalyzer());
pf.next = fieldHash[hashPos];
fieldHash[hashPos] = pf;
totalFieldCount++;
@@ -1083,7 +1107,6 @@ final class IndexingChain implements Accountable {
final String fieldName;
final int indexCreatedVersionMajor;
final FieldSchema schema;
- final boolean reserved;
FieldInfo fieldInfo;
final Similarity similarity;
@@ -1129,15 +1152,13 @@ final class IndexingChain implements Accountable {
FieldSchema schema,
Similarity similarity,
InfoStream infoStream,
- Analyzer analyzer,
- boolean reserved) {
+ Analyzer analyzer) {
this.fieldName = fieldName;
this.indexCreatedVersionMajor = indexCreatedVersionMajor;
this.schema = schema;
this.similarity = similarity;
this.infoStream = infoStream;
this.analyzer = analyzer;
- this.reserved = reserved;
}
void reset(int docId, IndexableFieldType fieldType) {
@@ -1618,77 +1639,4 @@ final class IndexingChain implements Accountable {
assertSame("point num bytes", fi.getPointNumBytes(), pointNumBytes);
}
}
-
- /**
- * Wraps the given field in a reserved field and registers it as reserved. Only DWPT should do
- * this to mark fields as private / reserved to prevent this fieldname to be used from the outside
- * of the IW / DWPT eco-system
- */
- <T extends IndexableField> ReservedField<T> markAsReserved(T field) {
- getOrAddPerField(field.name(), true);
- return new ReservedField<>(field);
- }
-
- static final class ReservedField<T extends IndexableField> implements IndexableField {
-
- private final T delegate;
-
- private ReservedField(T delegate) {
- this.delegate = delegate;
- }
-
- T getDelegate() {
- return delegate;
- }
-
- @Override
- public String name() {
- return delegate.name();
- }
-
- @Override
- public IndexableFieldType fieldType() {
- return delegate.fieldType();
- }
-
- @Override
- public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) {
- return delegate.tokenStream(analyzer, reuse);
- }
-
- @Override
- public BytesRef binaryValue() {
- return delegate.binaryValue();
- }
-
- @Override
- public String stringValue() {
- return delegate.stringValue();
- }
-
- @Override
- public CharSequence getCharSequenceValue() {
- return delegate.getCharSequenceValue();
- }
-
- @Override
- public Reader readerValue() {
- return delegate.readerValue();
- }
-
- @Override
- public Number numericValue() {
- return delegate.numericValue();
- }
-
- @Override
- public StoredValue storedValue() {
- return delegate.storedValue();
- }
-
- @Override
- public InvertableType invertableType() {
- return delegate.invertableType();
- }
- }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index d8b46a9a5b6..1cf401e279f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -4950,6 +4950,45 @@ public class TestIndexWriter extends LuceneTestCase {
}
}
+ public void testSingleDocBlockWritesParentField() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setParentField("parent");
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ // Single-document "block" — the lone doc is both first and last in the block
+ Document single = new Document();
+ single.add(new StringField("id", "s0", Field.Store.YES));
+ writer.addDocuments(List.of(single));
+
+ // Multi-document block for comparison
+ Document child = new Document();
+ child.add(new StringField("id", "c0", Field.Store.YES));
+ Document parent = new Document();
+ parent.add(new StringField("id", "p0", Field.Store.YES));
+ writer.addDocuments(List.of(child, parent));
+ }
+
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ // 3 docs total: doc0=single, doc1=child, doc2=parent
+ assertEquals(3, reader.numDocs());
+ LeafReader leaf = reader.leaves().get(0).reader();
+ NumericDocValues parentDV = leaf.getNumericDocValues("parent");
+ assertNotNull(parentDV);
+
+ // doc 0 (the single-doc block) should be a parent
+ assertEquals(0, parentDV.nextDoc());
+ assertEquals(-1, parentDV.longValue());
+
+ // doc 2 (last doc of the multi-doc block) should be a parent
+ assertEquals(2, parentDV.nextDoc());
+ assertEquals(-1, parentDV.longValue());
+
+ // no more parents
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, parentDV.nextDoc());
+ }
+ }
+ }
+
public void testDocValuesMixedSkippingIndex() throws Exception {
try (Directory dir = newDirectory()) {
try (IndexWriter writer =