Apache Lucene open-source search software
Fix stored-fields frame leak when PerField#finish throws (#16553)
Michael Marshalld8ffa2a68668
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 3108ae5c72c..6134ad5759e 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -214,6 +214,9 @@ OptimizationsBug Fixes---------------------+* GITHUB#16553: Fix stored-fields corruption when PerField#finish throws: the stored-fields frame+ was left open, causing "Wrote N docs, finish called with numDocs=M" at flush. (Michael Marshall)+* GITHUB#16552: Reconnect HNSW nodes whose out-degree decays across successivemerge-reuse generations, preventing progressive recall loss under low deleterates. (Ethan Baker)diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.javaindex b0e29104682..03dab23d76b 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java@@ -659,11 +659,14 @@ final class IndexingChain implements Accountable {}} finally {if (hasHitAbortingException == false) {- // Finish each indexed field name seen in the document:- for (int i = 0; i < indexedFieldCount; i++) {- fields[i].finish(docID);+ try {+ // Finish each indexed field name seen in the document:+ for (int i = 0; i < indexedFieldCount; i++) {+ fields[i].finish(docID);+ }+ } finally {+ finishStoredFields();}- finishStoredFields();// TODO: for broken docs, optimize termsHash.finishDocumenttry {termsHash.finishDocument(docID);@@ -918,11 +921,14 @@ final class IndexingChain implements Accountable {}} finally {if (hasHitAbortingException == false) {- for (int i = 0; i < indexedFieldCount; i++) {- fields[i].finish(segDocID);- }- if (hasStored) {- finishStoredFields();+ try {+ for (int i = 0; i < indexedFieldCount; i++) {+ fields[i].finish(segDocID);+ }+ } finally {+ if (hasStored) {+ finishStoredFields();+ }}if (hasInverted) {try {diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.javaindex 469d8f91588..7b5a67fd029 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java@@ -49,6 +49,7 @@ import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.MatchAllDocsQuery;import org.apache.lucene.search.PhraseQuery;+import org.apache.lucene.search.similarities.BM25Similarity;import org.apache.lucene.store.AlreadyClosedException;import org.apache.lucene.store.ByteBuffersDirectory;import org.apache.lucene.store.Directory;@@ -2302,6 +2303,27 @@ public class TestIndexWriterExceptions extends LuceneTestCase {}}+ public void testPerFieldFinishExceptionDoesNotCorruptStoredFields() throws Exception {+ Directory dir = newDirectory();+ IndexWriterConfig iwc = newIndexWriterConfig();+ iwc.setSimilarity(+ new BM25Similarity() {+ @Override+ public long computeNorm(FieldInvertState state) {+ return 0; // invalid return that triggers an exception+ }+ });+ IndexWriter w = new IndexWriter(dir, iwc);++ Document bad = new Document();+ bad.add(new TextField("text", "hello world", Field.Store.NO));+ expectThrows(IllegalStateException.class, () -> w.addDocument(bad));++ w.commit();+ w.close();+ dir.close();+ }+public void testExceptionJustBeforeFlushWithPointValues() throws Exception {Directory dir = newDirectory();Analyzer analyzer =