Apache Lucene open-source search software
SegmentInfos: Ensure IOExceptions are not hidden behind CorruptSegmentInfoException (#16788)
Alexander Reelsen330ee57b167e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 610c4534ffa..14c6f2387aa 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -212,7 +212,8 @@ Improvements* GITHUB#7820: CheckIndex now reports which segment's .si file could not be read, rather than only thata commit point was unreadable. SegmentInfos throws CorruptSegmentInfoException, carrying the segment- name and the underlying failure. (Gokul Manoj, Serhiy Bzhezytskyy)+ name and the underlying failure, when a .si file is missing or corrupt; other I/O failures, such as+ transient ones, propagate unchanged. (Gokul Manoj, Serhiy Bzhezytskyy, Alex Reelsen)Optimizations---------------------diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.javaindex 74b52723d4e..e6ecb6b2eae 100644--- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java@@ -45,7 +45,6 @@ import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexOutput;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.StringHelper;-import org.apache.lucene.util.ThreadInterruptedException;import org.apache.lucene.util.Version;/**@@ -409,11 +408,12 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfofinal SegmentInfo info;try {info = codec.segmentInfoFormat().read(directory, segName, segmentID, IOContext.READONCE);- } catch (ThreadInterruptedException e) {- throw e;- } catch (Exception | AssertionError e) {- // Corruption in a .si file can surface as almost anything the codec's reader happens to do- // with the bad bytes, so catch broadly, but keep the root cause: it is what names the file.+ } catch (CorruptIndexException+ | EOFException+ | NoSuchFileException+ | FileNotFoundException e) {+ // The .si is missing, truncated or failed its checksum. Anything else, e.g. a transient I/O+ // failure, is no evidence of corruption and must propagate as-is.throw new CorruptSegmentInfoException(segName,"segment info file: " + segName + ".si cannot be read - it may be missing or corrupt",diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.javaindex e38a697ef2c..17113ed1c24 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java@@ -37,6 +37,8 @@ import org.apache.lucene.search.MatchAllDocsQuery;import org.apache.lucene.search.Sort;import org.apache.lucene.search.SortField;import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FilterDirectory;+import org.apache.lucene.store.FilterIndexInput;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.IndexOutput;@@ -360,6 +362,65 @@ public class TestCheckIndex extends BaseTestCheckIndex {}}+ public void testSegmentInfoIOExceptionDoesNotThrowCorruptionException() throws Exception {+ for (String failure : List.of("fail-open", "fail-read")) {+ try (Directory dir = newDirectory()) {+ IndexWriterConfig iwc = new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE);+ try (IndexWriter iw = new IndexWriter(dir, iwc)) {+ for (int seg = 0; seg < 2; seg++) {+ Document doc = new Document();+ doc.add(new StringField("id", "d" + seg, Field.Store.NO));+ iw.addDocument(doc);+ iw.commit();+ }+ }++ Directory failingDir =+ new FilterDirectory(dir) {+ @Override+ public IndexInput openInput(String name, IOContext context) throws IOException {+ if (name.equals("_1.si") == false) {+ return super.openInput(name, context);+ }+ if (failure.equals("fail-open")) {+ throw new IOException("simulated I/O failure opening " + name);+ }+ IndexInput in = super.openInput(name, context);+ return new FilterIndexInput("failing(" + in + ")", in) {+ @Override+ public byte readByte() throws IOException {+ throw new IOException("simulated I/O failure reading " + name);+ }++ @Override+ public void readBytes(byte[] b, int offset, int len) throws IOException {+ throw new IOException("simulated I/O failure reading " + name);+ }+ };+ }+ };++ IOException e =+ expectThrows(+ IOException.class,+ () ->+ SegmentInfos.readCommit(+ failingDir, SegmentInfos.getLastCommitSegmentsFileName(failingDir), 0));+ assertFalse(failure + ": " + e, e instanceof CorruptIndexException);+ assertTrue(failure + ": " + e, e.getMessage().startsWith("simulated I/O failure"));++ // ... and CheckIndex does not blame the segment for it+ try (CheckIndex checker = new CheckIndex(failingDir)) {+ CheckIndex.Status status = checker.checkIndex();++ assertFalse(failure, status.clean);+ assertTrue(failure, status.missingSegments);+ assertNull(failure, status.brokenSegmentName);+ }+ }+ }+ }+/** Rewrites {@code name} keeping only its first 70% of bytes. */private static void truncate(Directory dir, String name) throws IOException {byte[] bytes;diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java b/lucene/core/src/test/org/apache/lucene/index/TestTransactions.javaindex c7cefb84ea4..49703ac8a7c 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java@@ -206,10 +206,7 @@ public class TestTransactions extends LuceneTestCase {} catch (Exception e) {// can be rethrown as RuntimeException if it happens during a close listenerif (!e.getMessage().contains("on purpose")) {- // Caught "on-purpose" IOException can be rethrown as CorruptSegmentInfoException- if (e instanceof CorruptSegmentInfoException == false) {- throw e;- }+ throw e;}// release resourcesIOUtils.closeWhileHandlingException(r1, r2);