Skip to content

Apache Lucene open-source search software

Fix stored-fields frame leak when PerField#finish throws (#16553)

Michael Marshalld8ffa2a68668
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3108ae5c72c..6134ad5759e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -214,6 +214,9 @@ Optimizations
Bug Fixes
---------------------
+* GITHUB#16553: Fix stored-fields corruption when PerField#finish throws: the stored-fields frame
+ was left open, causing "Wrote N docs, finish called with numDocs=M" at flush. (Michael Marshall)
+
* GITHUB#16552: Reconnect HNSW nodes whose out-degree decays across successive
merge-reuse generations, preventing progressive recall loss under low delete
rates. (Ethan Baker)
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index b0e29104682..03dab23d76b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -659,11 +659,14 @@ final class IndexingChain implements Accountable {
}
} finally {
if (hasHitAbortingException == false) {
- // Finish each indexed field name seen in the document:
- for (int i = 0; i < indexedFieldCount; i++) {
- fields[i].finish(docID);
+ try {
+ // Finish each indexed field name seen in the document:
+ for (int i = 0; i < indexedFieldCount; i++) {
+ fields[i].finish(docID);
+ }
+ } finally {
+ finishStoredFields();
}
- finishStoredFields();
// TODO: for broken docs, optimize termsHash.finishDocument
try {
termsHash.finishDocument(docID);
@@ -918,11 +921,14 @@ final class IndexingChain implements Accountable {
}
} finally {
if (hasHitAbortingException == false) {
- for (int i = 0; i < indexedFieldCount; i++) {
- fields[i].finish(segDocID);
- }
- if (hasStored) {
- finishStoredFields();
+ try {
+ for (int i = 0; i < indexedFieldCount; i++) {
+ fields[i].finish(segDocID);
+ }
+ } finally {
+ if (hasStored) {
+ finishStoredFields();
+ }
}
if (hasInverted) {
try {
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
index 469d8f91588..7b5a67fd029 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
@@ -49,6 +49,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
@@ -2302,6 +2303,27 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
}
}
+ public void testPerFieldFinishExceptionDoesNotCorruptStoredFields() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig();
+ iwc.setSimilarity(
+ new BM25Similarity() {
+ @Override
+ public long computeNorm(FieldInvertState state) {
+ return 0; // invalid return that triggers an exception
+ }
+ });
+ IndexWriter w = new IndexWriter(dir, iwc);
+
+ Document bad = new Document();
+ bad.add(new TextField("text", "hello world", Field.Store.NO));
+ expectThrows(IllegalStateException.class, () -> w.addDocument(bad));
+
+ w.commit();
+ w.close();
+ dir.close();
+ }
+
public void testExceptionJustBeforeFlushWithPointValues() throws Exception {
Directory dir = newDirectory();
Analyzer analyzer =