Skip to content

Apache Lucene open-source search software

SegmentInfos: Ensure IOExceptions are not hidden behind CorruptSegmentInfoException (#16788)

Alexander Reelsen330ee57b167e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 610c4534ffa..14c6f2387aa 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -212,7 +212,8 @@ Improvements
* GITHUB#7820: CheckIndex now reports which segment's .si file could not be read, rather than only that
a commit point was unreadable. SegmentInfos throws CorruptSegmentInfoException, carrying the segment
- name and the underlying failure. (Gokul Manoj, Serhiy Bzhezytskyy)
+ name and the underlying failure, when a .si file is missing or corrupt; other I/O failures, such as
+ transient ones, propagate unchanged. (Gokul Manoj, Serhiy Bzhezytskyy, Alex Reelsen)
Optimizations
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
index 74b52723d4e..e6ecb6b2eae 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
@@ -45,7 +45,6 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.Version;
/**
@@ -409,11 +408,12 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
final SegmentInfo info;
try {
info = codec.segmentInfoFormat().read(directory, segName, segmentID, IOContext.READONCE);
- } catch (ThreadInterruptedException e) {
- throw e;
- } catch (Exception | AssertionError e) {
- // Corruption in a .si file can surface as almost anything the codec's reader happens to do
- // with the bad bytes, so catch broadly, but keep the root cause: it is what names the file.
+ } catch (CorruptIndexException
+ | EOFException
+ | NoSuchFileException
+ | FileNotFoundException e) {
+ // The .si is missing, truncated or failed its checksum. Anything else, e.g. a transient I/O
+ // failure, is no evidence of corruption and must propagate as-is.
throw new CorruptSegmentInfoException(
segName,
"segment info file: " + segName + ".si cannot be read - it may be missing or corrupt",
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
index e38a697ef2c..17113ed1c24 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
@@ -37,6 +37,8 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -360,6 +362,65 @@ public class TestCheckIndex extends BaseTestCheckIndex {
}
}
+ public void testSegmentInfoIOExceptionDoesNotThrowCorruptionException() throws Exception {
+ for (String failure : List.of("fail-open", "fail-read")) {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE);
+ try (IndexWriter iw = new IndexWriter(dir, iwc)) {
+ for (int seg = 0; seg < 2; seg++) {
+ Document doc = new Document();
+ doc.add(new StringField("id", "d" + seg, Field.Store.NO));
+ iw.addDocument(doc);
+ iw.commit();
+ }
+ }
+
+ Directory failingDir =
+ new FilterDirectory(dir) {
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ if (name.equals("_1.si") == false) {
+ return super.openInput(name, context);
+ }
+ if (failure.equals("fail-open")) {
+ throw new IOException("simulated I/O failure opening " + name);
+ }
+ IndexInput in = super.openInput(name, context);
+ return new FilterIndexInput("failing(" + in + ")", in) {
+ @Override
+ public byte readByte() throws IOException {
+ throw new IOException("simulated I/O failure reading " + name);
+ }
+
+ @Override
+ public void readBytes(byte[] b, int offset, int len) throws IOException {
+ throw new IOException("simulated I/O failure reading " + name);
+ }
+ };
+ }
+ };
+
+ IOException e =
+ expectThrows(
+ IOException.class,
+ () ->
+ SegmentInfos.readCommit(
+ failingDir, SegmentInfos.getLastCommitSegmentsFileName(failingDir), 0));
+ assertFalse(failure + ": " + e, e instanceof CorruptIndexException);
+ assertTrue(failure + ": " + e, e.getMessage().startsWith("simulated I/O failure"));
+
+ // ... and CheckIndex does not blame the segment for it
+ try (CheckIndex checker = new CheckIndex(failingDir)) {
+ CheckIndex.Status status = checker.checkIndex();
+
+ assertFalse(failure, status.clean);
+ assertTrue(failure, status.missingSegments);
+ assertNull(failure, status.brokenSegmentName);
+ }
+ }
+ }
+ }
+
/** Rewrites {@code name} keeping only its first 70% of bytes. */
private static void truncate(Directory dir, String name) throws IOException {
byte[] bytes;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java b/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java
index c7cefb84ea4..49703ac8a7c 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java
@@ -206,10 +206,7 @@ public class TestTransactions extends LuceneTestCase {
} catch (Exception e) {
// can be rethrown as RuntimeException if it happens during a close listener
if (!e.getMessage().contains("on purpose")) {
- // Caught "on-purpose" IOException can be rethrown as CorruptSegmentInfoException
- if (e instanceof CorruptSegmentInfoException == false) {
- throw e;
- }
+ throw e;
}
// release resources
IOUtils.closeWhileHandlingException(r1, r2);