Skip to content

Apache Lucene open-source search software

#14607: Index open performs version check on each segment, ignores indexCreatedVersionMajor

Rahul Goswamidd70ed5801e3
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 73a1120c012..6a553602382 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -131,6 +131,8 @@ Other
* GITHUB#14761: Use more Comparators for PriorityQueue implementations. (Simon Cooper)
* GITHUB#14817: Refactor some complex uses of PriorityQueue to use Comparators. (Simon Cooper)
+* GITHUB#14607: Index open performs version check on each segment, ignores indexCreatedVersionMajor (Rahul Goswami)
+
======================= Lucene 10.4.0 =======================
API Changes
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java
index 246e6fe01b2..af3cf28c93a 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java
@@ -45,6 +45,7 @@ import org.apache.lucene.tests.store.BaseDirectoryWrapper;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.Version;
public class TestAncientIndicesCompatibility extends LuceneTestCase {
static final Set<String> UNSUPPORTED_INDEXES;
@@ -199,7 +200,7 @@ public class TestAncientIndicesCompatibility extends LuceneTestCase {
checker.setInfoStream(new PrintStream(bos, false, UTF_8));
checker.setLevel(CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS);
CheckIndex.Status indexStatus = checker.checkIndex();
- if (version.startsWith("8.") || version.startsWith("9.")) {
+ if (getVersion(version).onOrAfter(Version.fromBits(8, 6, 0))) {
assertTrue(indexStatus.clean);
} else {
assertFalse(indexStatus.clean);
@@ -217,6 +218,18 @@ public class TestAncientIndicesCompatibility extends LuceneTestCase {
}
}
+ private Version getVersion(String version) {
+ if (version.startsWith("5x")) {
+ // couple of indices in unsupported_indices.txt start with "5x'
+ return Version.fromBits(5, 0, 0);
+ }
+ String[] versionBitsStr = version.split("[.\\-]");
+ return Version.fromBits(
+ Integer.parseInt(versionBitsStr[0]),
+ Integer.parseInt(versionBitsStr[1]),
+ Integer.parseInt(versionBitsStr[2]));
+ }
+
// #12895: test on a carefully crafted 9.8.0 index (from a small contiguous subset
// of wikibigall unique terms) that shows the read-time exception of
// IntersectTermsEnum (used by WildcardQuery)
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java
index 77dc6d2412c..15088e14fb6 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java
@@ -864,7 +864,7 @@ public class TestBasicBackwardsCompatibility extends BackwardsCompatibilityTestB
assertTrue(
ex.getMessage()
.contains(
- "This Lucene version only supports indexes created with major version "
+ "This Lucene version only supports indexes with major version "
+ Version.LATEST.major
+ " or later"));
// now open with allowed min version
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
index 204bd89b350..c6042baf0b0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
@@ -328,7 +328,7 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
throw new IndexFormatTooOldException(
input, magic, CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC);
}
- format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_74, VERSION_CURRENT);
+ format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_86, VERSION_CURRENT);
byte[] id = new byte[StringHelper.ID_LENGTH];
input.readBytes(id, 0, id.length);
CodecUtil.checkIndexHeaderSuffix(input, Long.toString(generation, Character.MAX_RADIX));
@@ -346,30 +346,12 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
input);
}
- if (indexCreatedVersion < minSupportedMajorVersion) {
- throw new IndexFormatTooOldException(
- input,
- "Index created with Lucene "
- + indexCreatedVersion
- + ".x is not supported by Lucene "
- + Version.LATEST
- + ". This Lucene version only supports indexes created with major version "
- + minSupportedMajorVersion
- + " or later (found: "
- + indexCreatedVersion
- + ", minimum: "
- + minSupportedMajorVersion
- + "). To resolve this issue: (1) Re-index your data using Lucene "
- + Version.LATEST.major
- + ".x, or (2) Use an older Lucene version that supports your index format.");
- }
-
SegmentInfos infos = new SegmentInfos(indexCreatedVersion);
infos.id = id;
infos.generation = generation;
infos.lastGeneration = generation;
infos.luceneVersion = luceneVersion;
- parseSegmentInfos(directory, input, infos, format);
+ parseSegmentInfos(directory, input, infos, format, minSupportedMajorVersion);
return infos;
} catch (Throwable t) {
@@ -385,7 +367,12 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
}
private static void parseSegmentInfos(
- Directory directory, DataInput input, SegmentInfos infos, int format) throws IOException {
+ Directory directory,
+ DataInput input,
+ SegmentInfos infos,
+ int format,
+ int minSupportedMajorVersion)
+ throws IOException {
infos.version = CodecUtil.readBELong(input);
// System.out.println("READ sis version=" + infos.version);
infos.counter = input.readVLong();
@@ -402,6 +389,7 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
}
long totalDocs = 0;
+
for (int seg = 0; seg < numSegments; seg++) {
String segName = input.readString();
byte[] segmentID = new byte[StringHelper.ID_LENGTH];
@@ -495,6 +483,30 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
+ infos.indexCreatedVersionMajor,
input);
}
+
+ int createdOrSegmentMinVersion =
+ info.getMinVersion() == null
+ ? infos.indexCreatedVersionMajor
+ : info.getMinVersion().major;
+
+ // version >=7 are expected to record minVersion
+ if (info.getMinVersion() == null || info.getMinVersion().major < minSupportedMajorVersion) {
+ throw new IndexFormatTooOldException(
+ input,
+ "Index has segments derived from Lucene version "
+ + createdOrSegmentMinVersion
+ + ".x and is not supported by Lucene "
+ + Version.LATEST
+ + ". This Lucene version only supports indexes with major version "
+ + minSupportedMajorVersion
+ + " or later (found: "
+ + createdOrSegmentMinVersion
+ + ", minimum supported: "
+ + minSupportedMajorVersion
+ + "). To resolve this issue re-index your data using Lucene "
+ + minSupportedMajorVersion
+ + ".x or later.");
+ }
}
infos.userData = input.readMapOfStrings();