Apache Lucene open-source search software
Modernize BWC testing with parameterized tests (#13046)
Simon Willnauer8d9290ca36ea
diff --git a/dev-tools/scripts/addBackcompatIndexes.py b/dev-tools/scripts/addBackcompatIndexes.pyindex 3d05d85a52b..0be87ff7a55 100755--- a/dev-tools/scripts/addBackcompatIndexes.py+++ b/dev-tools/scripts/addBackcompatIndexes.py@@ -71,7 +71,7 @@ def create_and_add_index(source, indextype, index_version, current_version, temp'-Ptests.useSecurityManager=false','-p lucene/%s' % module,'test',- '--tests TestBackwardsCompatibility.%s' % test,+ '--tests TestGenerateBwcIndices.%s' % test,'-Dtests.bwcdir=%s' % temp_dir,'-Dtests.codec=default'])@@ -99,7 +99,7 @@ def create_and_add_index(source, indextype, index_version, current_version, tempdef update_backcompat_tests(types, index_version, current_version):print(' adding new indexes %s to backcompat tests...' % types, end='', flush=True)module = 'lucene/backward-codecs'- filename = '%s/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java' % module+ filename = '%s/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java' % moduleif not current_version.is_back_compat_with(index_version):matcher = re.compile(r'final String\[\] unsupportedNames = {|};')elif 'sorted' in types:@@ -165,7 +165,7 @@ def update_backcompat_tests(types, index_version, current_version):def check_backcompat_tests():print(' checking backcompat tests...', end='', flush=True)- scriptutil.run('./gradlew -p lucene/backward-codecs test --tests TestBackwardsCompatibility')+ scriptutil.run('./gradlew -p lucene/backward-codecs test --tests TestGenerateBwcIndices')print('ok')def download_from_cdn(version, remotename, localname):@@ -248,7 +248,7 @@ def main():create_and_add_index(source, 'moreterms', c.version, current_version, c.temp_dir)create_and_add_index(source, 'dvupdates', c.version, current_version, c.temp_dir)create_and_add_index(source, 'emptyIndex', c.version, current_version, c.temp_dir)- print ('\nMANUAL UPDATE REQUIRED: edit TestBackwardsCompatibility to enable moreterms, dvupdates, and empty index testing')+ print ('\nMANUAL UPDATE REQUIRED: edit TestGenerateBwcIndices to enable moreterms, dvupdates, and empty index testing')print('\nAdding backwards compatibility tests')update_backcompat_tests(['cfs', 'nocfs'], c.version, current_version)diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.javanew file mode 100644index 00000000000..b3132d6e55f--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java@@ -0,0 +1,253 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_index;++import com.carrotsearch.randomizedtesting.annotations.Name;+import java.io.IOException;+import java.io.InputStream;+import java.lang.reflect.Field;+import java.lang.reflect.Modifier;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.Paths;+import java.text.ParseException;+import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;+import java.util.Locale;+import java.util.Set;+import java.util.regex.Matcher;+import java.util.regex.Pattern;+import org.apache.lucene.codecs.Codec;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.SegmentReader;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.Version;+import org.junit.After;+import org.junit.Before;++public abstract class BackwardsCompatibilityTestBase extends LuceneTestCase {++ protected final Version version;+ private static final Version LATEST_PREVIOUS_MAJOR = getLatestPreviousMajorVersion();+ protected final String indexPattern;+ protected static final Set<Version> BINARY_SUPPORTED_VERSIONS;++ static {+ String[] oldVersions =+ new String[] {+ "8.0.0", "8.0.0", "8.1.0", "8.1.0", "8.1.1", "8.1.1", "8.2.0", "8.2.0", "8.3.0", "8.3.0",+ "8.3.1", "8.3.1", "8.4.0", "8.4.0", "8.4.1", "8.4.1", "8.5.0", "8.5.0", "8.5.1", "8.5.1",+ "8.5.2", "8.5.2", "8.6.0", "8.6.0", "8.6.1", "8.6.1", "8.6.2", "8.6.2", "8.6.3", "8.6.3",+ "8.7.0", "8.7.0", "8.8.0", "8.8.0", "8.8.1", "8.8.1", "8.8.2", "8.8.2", "8.9.0", "8.9.0",+ "8.10.0", "8.10.0", "8.10.1", "8.10.1", "8.11.0", "8.11.0", "8.11.1", "8.11.1", "8.11.2",+ "8.11.2", "9.0.0", "9.1.0", "9.2.0", "9.3.0", "9.4.0", "9.4.1", "9.4.2", "9.5.0", "9.6.0",+ "9.7.0", "9.8.0", "9.9.0", "9.9.1", "9.9.2", "9.10.0", "10.0.0",+ };++ Set<Version> binaryVersions = new HashSet<>();+ for (String version : oldVersions) {+ try {+ Version v = Version.parse(version);+ assertTrue("Unsupported binary version: " + v, v.major >= Version.MIN_SUPPORTED_MAJOR - 1);+ binaryVersions.add(v);+ } catch (ParseException ex) {+ throw new RuntimeException(ex);+ }+ }+ List<Version> allCurrentVersions = getAllCurrentVersions();+ for (Version version : allCurrentVersions) {+ // make sure we never miss a version.+ assertTrue("Version: " + version + " missing", binaryVersions.remove(version));+ }+ BINARY_SUPPORTED_VERSIONS = binaryVersions;+ }++ /**+ * This is a base constructor for parameterized BWC tests. The constructor arguments are provided+ * by {@link com.carrotsearch.randomizedtesting.RandomizedRunner} during test execution. A {@link+ * com.carrotsearch.randomizedtesting.annotations.ParametersFactory} specified in a subclass+ * provides a list lists of arguments for the tests and RandomizedRunner will execute the test for+ * each of the argument list.+ *+ * @param version the version this test should run for+ * @param indexPattern an index pattern in order to open an index of see {@link+ * #createPattern(String, String)}+ */+ protected BackwardsCompatibilityTestBase(+ @Name("version") Version version, @Name("pattern") String indexPattern) {+ this.version = version;+ this.indexPattern = indexPattern;+ }++ Directory directory;++ @Override+ @Before+ public void setUp() throws Exception {+ super.setUp();+ assertNull(+ "Index name " + version + " should not exist found",+ TestAncientIndicesCompatibility.class.getResourceAsStream(+ indexName(LATEST_PREVIOUS_MAJOR)));+ if (supportsVersion(version) == false) {+ assertNull(+ "Index name " + version + " should not exist found",+ TestAncientIndicesCompatibility.class.getResourceAsStream(indexName(version)));+ }+ assumeTrue("This test doesn't support version: " + version, supportsVersion(version));+ if (version.equals(Version.LATEST)) {+ directory = newDirectory();+ createIndex(directory);+ } else {+ Path dir = createTempDir();+ InputStream resource =+ TestAncientIndicesCompatibility.class.getResourceAsStream(indexName(version));+ assertNotNull("Index name " + version + " not found: " + indexName(version), resource);+ TestUtil.unzip(resource, dir);+ directory = newFSDirectory(dir);+ }+ verifyUsesDefaultCodec(directory, indexName(version));+ }++ @Override+ @After+ public void tearDown() throws Exception {+ super.tearDown();+ IOUtils.close(directory);+ directory = null;+ }++ private static Version getLatestPreviousMajorVersion() {+ Version lastPrevMajorVersion = null;+ for (Version v : getAllCurrentVersions()) {+ if (v.major == Version.LATEST.major - 1+ && (lastPrevMajorVersion == null || v.onOrAfter(lastPrevMajorVersion))) {+ lastPrevMajorVersion = v;+ }+ }+ return lastPrevMajorVersion;+ }++ /**+ * Creates an index pattern of the form '$name.$version$suffix.zip' where version is filled in+ * afterward via {@link String#formatted(Object...)} during the test runs.+ *+ * @param name name of the index+ * @param suffix index suffix ie. '-cfs'+ */+ static String createPattern(String name, String suffix) {+ return name + ".%1$s" + suffix + ".zip";+ }++ public static List<Version> getAllCurrentVersions() {+ Pattern constantPattern = Pattern.compile("LUCENE_(\\d+)_(\\d+)_(\\d+)(_ALPHA|_BETA)?");+ List<Version> versions = new ArrayList<>();+ for (Field field : Version.class.getDeclaredFields()) {+ if (Modifier.isStatic(field.getModifiers()) && field.getType() == Version.class) {+ Matcher constant = constantPattern.matcher(field.getName());+ Version v;+ try {+ v = (Version) field.get(Version.class);+ } catch (IllegalAccessException e) {+ throw new RuntimeException(e);+ }+ if (constant.matches() == false) {+ continue;+ }+ versions.add(v);+ }+ }+ return versions;+ }++ public static Iterable<Object[]> allVersion(String name, String... suffixes) {+ List<Object> patterns = new ArrayList<>();+ for (String suffix : suffixes) {+ patterns.add(createPattern(name, suffix));+ }+ List<Object[]> versionAndPatterns = new ArrayList<>();+ List<Version> versionList = getAllCurrentVersions();+ for (Version v : versionList) {+ if (v.equals(LATEST_PREVIOUS_MAJOR)+ == false) { // the latest prev-major has not yet been released+ for (Object p : patterns) {+ versionAndPatterns.add(new Object[] {v, p});+ }+ }+ }+ return versionAndPatterns;+ }++ public String indexName(Version version) {+ return String.format(Locale.ROOT, indexPattern, version);+ }++ protected boolean supportsVersion(Version version) {+ return true;+ }++ protected abstract void createIndex(Directory directory) throws IOException;++ public final void createBWCIndex() throws IOException {+ Path indexDir = getIndexDir().resolve(indexName(Version.LATEST));+ Files.deleteIfExists(indexDir);+ try (Directory dir = newFSDirectory(indexDir)) {+ createIndex(dir);+ }+ }++ private Path getIndexDir() {+ String path = System.getProperty("tests.bwcdir");+ assumeTrue(+ "backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes",+ path != null);+ return Paths.get(path);+ }++ void verifyUsesDefaultCodec(Directory dir, String name) throws IOException {+ DirectoryReader r = DirectoryReader.open(dir);+ for (LeafReaderContext context : r.leaves()) {+ SegmentReader air = (SegmentReader) context.reader();+ Codec codec = air.getSegmentInfo().info.getCodec();+ assertTrue(+ "codec used in "+ + name+ + " ("+ + codec.getName()+ + ") is not a default codec (does not begin with Lucene)",+ codec.getName().startsWith("Lucene"));+ }+ r.close();+ }++ // encodes a long into a BytesRef as VLong so that we get varying number of bytes when we update+ static BytesRef toBytes(long value) {+ BytesRef bytes = new BytesRef(10); // negative longs may take 10 bytes+ while ((value & ~0x7FL) != 0L) {+ bytes.bytes[bytes.length++] = (byte) ((value & 0x7FL) | 0x80L);+ value >>>= 7;+ }+ bytes.bytes[bytes.length++] = (byte) value;+ return bytes;+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.javanew file mode 100644index 00000000000..722d75d775f--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java@@ -0,0 +1,443 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_index;++import static java.nio.charset.StandardCharsets.UTF_8;++import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.io.PrintStream;+import java.nio.file.Path;+import org.apache.lucene.index.CheckIndex;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexFormatTooOldException;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.Term;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.WildcardQuery;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.store.BaseDirectoryWrapper;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;++@SuppressWarnings("deprecation")+public class TestAncientIndicesCompatibility extends LuceneTestCase {++ static final String[] unsupportedNames = {+ "1.9.0-cfs",+ "1.9.0-nocfs",+ "2.0.0-cfs",+ "2.0.0-nocfs",+ "2.1.0-cfs",+ "2.1.0-nocfs",+ "2.2.0-cfs",+ "2.2.0-nocfs",+ "2.3.0-cfs",+ "2.3.0-nocfs",+ "2.4.0-cfs",+ "2.4.0-nocfs",+ "2.4.1-cfs",+ "2.4.1-nocfs",+ "2.9.0-cfs",+ "2.9.0-nocfs",+ "2.9.1-cfs",+ "2.9.1-nocfs",+ "2.9.2-cfs",+ "2.9.2-nocfs",+ "2.9.3-cfs",+ "2.9.3-nocfs",+ "2.9.4-cfs",+ "2.9.4-nocfs",+ "3.0.0-cfs",+ "3.0.0-nocfs",+ "3.0.1-cfs",+ "3.0.1-nocfs",+ "3.0.2-cfs",+ "3.0.2-nocfs",+ "3.0.3-cfs",+ "3.0.3-nocfs",+ "3.1.0-cfs",+ "3.1.0-nocfs",+ "3.2.0-cfs",+ "3.2.0-nocfs",+ "3.3.0-cfs",+ "3.3.0-nocfs",+ "3.4.0-cfs",+ "3.4.0-nocfs",+ "3.5.0-cfs",+ "3.5.0-nocfs",+ "3.6.0-cfs",+ "3.6.0-nocfs",+ "3.6.1-cfs",+ "3.6.1-nocfs",+ "3.6.2-cfs",+ "3.6.2-nocfs",+ "4.0.0-cfs",+ "4.0.0-cfs",+ "4.0.0-nocfs",+ "4.0.0.1-cfs",+ "4.0.0.1-nocfs",+ "4.0.0.2-cfs",+ "4.0.0.2-nocfs",+ "4.1.0-cfs",+ "4.1.0-nocfs",+ "4.2.0-cfs",+ "4.2.0-nocfs",+ "4.2.1-cfs",+ "4.2.1-nocfs",+ "4.3.0-cfs",+ "4.3.0-nocfs",+ "4.3.1-cfs",+ "4.3.1-nocfs",+ "4.4.0-cfs",+ "4.4.0-nocfs",+ "4.5.0-cfs",+ "4.5.0-nocfs",+ "4.5.1-cfs",+ "4.5.1-nocfs",+ "4.6.0-cfs",+ "4.6.0-nocfs",+ "4.6.1-cfs",+ "4.6.1-nocfs",+ "4.7.0-cfs",+ "4.7.0-nocfs",+ "4.7.1-cfs",+ "4.7.1-nocfs",+ "4.7.2-cfs",+ "4.7.2-nocfs",+ "4.8.0-cfs",+ "4.8.0-nocfs",+ "4.8.1-cfs",+ "4.8.1-nocfs",+ "4.9.0-cfs",+ "4.9.0-nocfs",+ "4.9.1-cfs",+ "4.9.1-nocfs",+ "4.10.0-cfs",+ "4.10.0-nocfs",+ "4.10.1-cfs",+ "4.10.1-nocfs",+ "4.10.2-cfs",+ "4.10.2-nocfs",+ "4.10.3-cfs",+ "4.10.3-nocfs",+ "4.10.4-cfs",+ "4.10.4-nocfs",+ "5x-with-4x-segments-cfs",+ "5x-with-4x-segments-nocfs",+ "5.0.0.singlesegment-cfs",+ "5.0.0.singlesegment-nocfs",+ "5.0.0-cfs",+ "5.0.0-nocfs",+ "5.1.0-cfs",+ "5.1.0-nocfs",+ "5.2.0-cfs",+ "5.2.0-nocfs",+ "5.2.1-cfs",+ "5.2.1-nocfs",+ "5.3.0-cfs",+ "5.3.0-nocfs",+ "5.3.1-cfs",+ "5.3.1-nocfs",+ "5.3.2-cfs",+ "5.3.2-nocfs",+ "5.4.0-cfs",+ "5.4.0-nocfs",+ "5.4.1-cfs",+ "5.4.1-nocfs",+ "5.5.0-cfs",+ "5.5.0-nocfs",+ "5.5.1-cfs",+ "5.5.1-nocfs",+ "5.5.2-cfs",+ "5.5.2-nocfs",+ "5.5.3-cfs",+ "5.5.3-nocfs",+ "5.5.4-cfs",+ "5.5.4-nocfs",+ "5.5.5-cfs",+ "5.5.5-nocfs",+ "6.0.0-cfs",+ "6.0.0-nocfs",+ "6.0.1-cfs",+ "6.0.1-nocfs",+ "6.1.0-cfs",+ "6.1.0-nocfs",+ "6.2.0-cfs",+ "6.2.0-nocfs",+ "6.2.1-cfs",+ "6.2.1-nocfs",+ "6.3.0-cfs",+ "6.3.0-nocfs",+ "6.4.0-cfs",+ "6.4.0-nocfs",+ "6.4.1-cfs",+ "6.4.1-nocfs",+ "6.4.2-cfs",+ "6.4.2-nocfs",+ "6.5.0-cfs",+ "6.5.0-nocfs",+ "6.5.1-cfs",+ "6.5.1-nocfs",+ "6.6.0-cfs",+ "6.6.0-nocfs",+ "6.6.1-cfs",+ "6.6.1-nocfs",+ "6.6.2-cfs",+ "6.6.2-nocfs",+ "6.6.3-cfs",+ "6.6.3-nocfs",+ "6.6.4-cfs",+ "6.6.4-nocfs",+ "6.6.5-cfs",+ "6.6.5-nocfs",+ "6.6.6-cfs",+ "6.6.6-nocfs",+ "7.0.0-cfs",+ "7.0.0-nocfs",+ "7.0.1-cfs",+ "7.0.1-nocfs",+ "7.1.0-cfs",+ "7.1.0-nocfs",+ "7.2.0-cfs",+ "7.2.0-nocfs",+ "7.2.1-cfs",+ "7.2.1-nocfs",+ "7.3.0-cfs",+ "7.3.0-nocfs",+ "7.3.1-cfs",+ "7.3.1-nocfs",+ "7.4.0-cfs",+ "7.4.0-nocfs",+ "7.5.0-cfs",+ "7.5.0-nocfs",+ "7.6.0-cfs",+ "7.6.0-nocfs",+ "7.7.0-cfs",+ "7.7.0-nocfs",+ "7.7.1-cfs",+ "7.7.1-nocfs",+ "7.7.2-cfs",+ "7.7.2-nocfs",+ "7.7.3-cfs",+ "7.7.3-nocfs",+ "8.0.0-cfs",+ "8.0.0-nocfs",+ "8.1.0-cfs",+ "8.1.0-nocfs",+ "8.1.1-cfs",+ "8.1.1-nocfs",+ "8.2.0-cfs",+ "8.2.0-nocfs",+ "8.3.0-cfs",+ "8.3.0-nocfs",+ "8.3.1-cfs",+ "8.3.1-nocfs",+ "8.4.0-cfs",+ "8.4.0-nocfs",+ "8.4.1-cfs",+ "8.4.1-nocfs",+ "8.5.0-cfs",+ "8.5.0-nocfs",+ "8.5.1-cfs",+ "8.5.1-nocfs",+ "8.5.2-cfs",+ "8.5.2-nocfs",+ "8.6.0-cfs",+ "8.6.0-nocfs",+ "8.6.1-cfs",+ "8.6.1-nocfs",+ "8.6.2-cfs",+ "8.6.2-nocfs",+ "8.6.3-cfs",+ "8.6.3-nocfs",+ "8.7.0-cfs",+ "8.7.0-nocfs",+ "8.8.0-cfs",+ "8.8.0-nocfs",+ "8.8.1-cfs",+ "8.8.1-nocfs",+ "8.8.2-cfs",+ "8.8.2-nocfs",+ "8.9.0-cfs",+ "8.9.0-nocfs",+ "8.10.0-cfs",+ "8.10.0-nocfs",+ "8.10.1-cfs",+ "8.10.1-nocfs",+ "8.11.0-cfs",+ "8.11.0-nocfs",+ "8.11.1-cfs",+ "8.11.1-nocfs",+ "8.11.2-cfs",+ "8.11.2-nocfs"+ };++ /**+ * This test checks that *only* IndexFormatTooOldExceptions are thrown when you open and operate+ * on too old indexes!+ */+ public void testUnsupportedOldIndexes() throws Exception {+ for (int i = 0; i < unsupportedNames.length; i++) {+ if (VERBOSE) {+ System.out.println("TEST: index " + unsupportedNames[i]);+ }+ Path oldIndexDir = createTempDir(unsupportedNames[i]);+ TestUtil.unzip(+ getDataInputStream("unsupported." + unsupportedNames[i] + ".zip"), oldIndexDir);+ BaseDirectoryWrapper dir = newFSDirectory(oldIndexDir);+ // don't checkindex, these are intentionally not supported+ dir.setCheckIndexOnClose(false);++ IndexReader reader = null;+ IndexWriter writer = null;+ try {+ reader = DirectoryReader.open(dir);+ fail("DirectoryReader.open should not pass for " + unsupportedNames[i]);+ } catch (IndexFormatTooOldException e) {+ if (e.getReason() != null) {+ assertNull(e.getVersion());+ assertNull(e.getMinVersion());+ assertNull(e.getMaxVersion());+ assertEquals(+ e.getMessage(),+ new IndexFormatTooOldException(e.getResourceDescription(), e.getReason())+ .getMessage());+ } else {+ assertNotNull(e.getVersion());+ assertNotNull(e.getMinVersion());+ assertNotNull(e.getMaxVersion());+ assertTrue(e.getMessage(), e.getMaxVersion() >= e.getMinVersion());+ assertTrue(+ e.getMessage(),+ e.getMaxVersion() < e.getVersion() || e.getVersion() < e.getMinVersion());+ assertEquals(+ e.getMessage(),+ new IndexFormatTooOldException(+ e.getResourceDescription(),+ e.getVersion(),+ e.getMinVersion(),+ e.getMaxVersion())+ .getMessage());+ }+ // pass+ if (VERBOSE) {+ System.out.println("TEST: got expected exc:");+ e.printStackTrace(System.out);+ }+ } finally {+ if (reader != null) reader.close();+ reader = null;+ }++ try {+ writer =+ new IndexWriter(+ dir, newIndexWriterConfig(new MockAnalyzer(random())).setCommitOnClose(false));+ fail("IndexWriter creation should not pass for " + unsupportedNames[i]);+ } catch (IndexFormatTooOldException e) {+ if (e.getReason() != null) {+ assertNull(e.getVersion());+ assertNull(e.getMinVersion());+ assertNull(e.getMaxVersion());+ assertEquals(+ e.getMessage(),+ new IndexFormatTooOldException(e.getResourceDescription(), e.getReason())+ .getMessage());+ } else {+ assertNotNull(e.getVersion());+ assertNotNull(e.getMinVersion());+ assertNotNull(e.getMaxVersion());+ assertTrue(e.getMessage(), e.getMaxVersion() >= e.getMinVersion());+ assertTrue(+ e.getMessage(),+ e.getMaxVersion() < e.getVersion() || e.getVersion() < e.getMinVersion());+ assertEquals(+ e.getMessage(),+ new IndexFormatTooOldException(+ e.getResourceDescription(),+ e.getVersion(),+ e.getMinVersion(),+ e.getMaxVersion())+ .getMessage());+ }+ // pass+ if (VERBOSE) {+ System.out.println("TEST: got expected exc:");+ e.printStackTrace(System.out);+ }+ // Make sure exc message includes a path=+ assertTrue("got exc message: " + e.getMessage(), e.getMessage().contains("path=\""));+ } finally {+ // we should fail to open IW, and so it should be null when we get here.+ // However, if the test fails (i.e., IW did not fail on open), we need+ // to close IW. However, if merges are run, IW may throw+ // IndexFormatTooOldException, and we don't want to mask the fail()+ // above, so close without waiting for merges.+ if (writer != null) {+ try {+ writer.commit();+ } finally {+ writer.close();+ }+ }+ }++ ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);+ CheckIndex checker = new CheckIndex(dir);+ checker.setInfoStream(new PrintStream(bos, false, UTF_8));+ CheckIndex.Status indexStatus = checker.checkIndex();+ if (unsupportedNames[i].startsWith("8.")) {+ assertTrue(indexStatus.clean);+ } else {+ assertFalse(indexStatus.clean);+ // CheckIndex doesn't enforce a minimum version, so we either get an+ // IndexFormatTooOldException+ // or an IllegalArgumentException saying that the codec doesn't exist.+ boolean formatTooOld =+ bos.toString(UTF_8).contains(IndexFormatTooOldException.class.getName());+ boolean missingCodec = bos.toString(UTF_8).contains("Could not load codec");+ assertTrue(formatTooOld || missingCodec);+ }+ checker.close();++ dir.close();+ }+ }++ // #12895: test on a carefully crafted 9.8.0 index (from a small contiguous subset+ // of wikibigall unique terms) that shows the read-time exception of+ // IntersectTermsEnum (used by WildcardQuery)+ public void testWildcardQueryExceptions990() throws IOException {+ Path path = createTempDir("12895");++ String name = "index.12895.9.8.0.zip";+ InputStream resource = TestAncientIndicesCompatibility.class.getResourceAsStream(name);+ assertNotNull("missing zip file to reproduce #12895", resource);+ TestUtil.unzip(resource, path);++ try (Directory dir = newFSDirectory(path);+ DirectoryReader reader = DirectoryReader.open(dir)) {+ IndexSearcher searcher = new IndexSearcher(reader);++ searcher.count(new WildcardQuery(new Term("field", "*qx*")));+ }+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.javadeleted file mode 100644index 9788661026f..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java+++ /dev/null@@ -1,2472 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_index;--import static java.nio.charset.StandardCharsets.UTF_8;-import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;-import static org.apache.lucene.util.Version.LUCENE_9_0_0;--import java.io.ByteArrayOutputStream;-import java.io.IOException;-import java.io.InputStream;-import java.io.PrintStream;-import java.lang.reflect.Modifier;-import java.nio.file.Files;-import java.nio.file.Path;-import java.nio.file.Paths;-import java.text.ParsePosition;-import java.text.SimpleDateFormat;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Collections;-import java.util.Date;-import java.util.HashMap;-import java.util.List;-import java.util.Locale;-import java.util.Map;-import java.util.Random;-import java.util.Set;-import java.util.TimeZone;-import java.util.regex.Matcher;-import java.util.regex.Pattern;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.document.BinaryDocValuesField;-import org.apache.lucene.document.BinaryPoint;-import org.apache.lucene.document.Document;-import org.apache.lucene.document.DoubleDocValuesField;-import org.apache.lucene.document.DoublePoint;-import org.apache.lucene.document.Field;-import org.apache.lucene.document.FieldType;-import org.apache.lucene.document.FloatDocValuesField;-import org.apache.lucene.document.FloatPoint;-import org.apache.lucene.document.IntPoint;-import org.apache.lucene.document.KnnFloatVectorField;-import org.apache.lucene.document.LongPoint;-import org.apache.lucene.document.NumericDocValuesField;-import org.apache.lucene.document.SortedDocValuesField;-import org.apache.lucene.document.SortedNumericDocValuesField;-import org.apache.lucene.document.SortedSetDocValuesField;-import org.apache.lucene.document.StringField;-import org.apache.lucene.document.TextField;-import org.apache.lucene.index.BinaryDocValues;-import org.apache.lucene.index.CheckIndex;-import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.Fields;-import org.apache.lucene.index.FloatVectorValues;-import org.apache.lucene.index.IndexCommit;-import org.apache.lucene.index.IndexFormatTooOldException;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.IndexReader;-import org.apache.lucene.index.IndexUpgrader;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.index.IndexWriterConfig.OpenMode;-import org.apache.lucene.index.IndexableField;-import org.apache.lucene.index.LeafReader;-import org.apache.lucene.index.LeafReaderContext;-import org.apache.lucene.index.LogByteSizeMergePolicy;-import org.apache.lucene.index.MergePolicy;-import org.apache.lucene.index.MultiBits;-import org.apache.lucene.index.MultiDocValues;-import org.apache.lucene.index.MultiTerms;-import org.apache.lucene.index.NoMergePolicy;-import org.apache.lucene.index.NumericDocValues;-import org.apache.lucene.index.PostingsEnum;-import org.apache.lucene.index.SegmentCommitInfo;-import org.apache.lucene.index.SegmentInfos;-import org.apache.lucene.index.SegmentReader;-import org.apache.lucene.index.SortedDocValues;-import org.apache.lucene.index.SortedNumericDocValues;-import org.apache.lucene.index.SortedSetDocValues;-import org.apache.lucene.index.StandardDirectoryReader;-import org.apache.lucene.index.StoredFields;-import org.apache.lucene.index.Term;-import org.apache.lucene.index.TermVectors;-import org.apache.lucene.index.Terms;-import org.apache.lucene.index.TermsEnum;-import org.apache.lucene.index.VectorSimilarityFunction;-import org.apache.lucene.search.BooleanClause;-import org.apache.lucene.search.BooleanQuery;-import org.apache.lucene.search.DocIdSetIterator;-import org.apache.lucene.search.FieldDoc;-import org.apache.lucene.search.FieldExistsQuery;-import org.apache.lucene.search.IndexSearcher;-import org.apache.lucene.search.KnnFloatVectorQuery;-import org.apache.lucene.search.MatchAllDocsQuery;-import org.apache.lucene.search.ScoreDoc;-import org.apache.lucene.search.Sort;-import org.apache.lucene.search.SortField;-import org.apache.lucene.search.TermQuery;-import org.apache.lucene.search.TopDocs;-import org.apache.lucene.search.WildcardQuery;-import org.apache.lucene.store.ByteBuffersDirectory;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.FSDirectory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.NIOFSDirectory;-import org.apache.lucene.tests.analysis.MockAnalyzer;-import org.apache.lucene.tests.index.RandomIndexWriter;-import org.apache.lucene.tests.store.BaseDirectoryWrapper;-import org.apache.lucene.tests.util.LineFileDocs;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.Bits;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.InfoStream;-import org.apache.lucene.util.Version;-import org.junit.AfterClass;-import org.junit.BeforeClass;--/*- Verify we can read previous versions' indexes, do searches- against them, and add documents to them.-*/-// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows-// machines occasionally-@SuppressWarnings("deprecation")-public class TestBackwardsCompatibility extends LuceneTestCase {-- // Backcompat index generation, described below, is mostly automated in:- //- // dev-tools/scripts/addBackcompatIndexes.py- //- // For usage information, see:- //- // http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes- //- // ------ //- // To generate backcompat indexes with the current default codec, run the following gradle- // command:- // gradlew test -Ptests.bwcdir=/path/to/store/indexes -Ptests.codec=default- // -Ptests.useSecurityManager=false --tests TestBackwardsCompatibility- // Also add testmethod with one of the index creation methods below, for example:- // -Ptestmethod=testCreateCFS- //- // Zip up the generated indexes:- //- // cd /path/to/store/indexes/index.cfs ; zip index.<VERSION>-cfs.zip *- // cd /path/to/store/indexes/index.nocfs ; zip index.<VERSION>-nocfs.zip *- //- // Then move those 2 zip files to your trunk checkout and add them- // to the oldNames array.-- private static final int DOCS_COUNT = 35;- private static final int DELETED_ID = 7;-- private static final int KNN_VECTOR_MIN_SUPPORTED_VERSION = LUCENE_9_0_0.major;- private static final String KNN_VECTOR_FIELD = "knn_field";- private static final FieldType KNN_VECTOR_FIELD_TYPE =- KnnFloatVectorField.createFieldType(3, VectorSimilarityFunction.COSINE);- private static final float[] KNN_VECTOR = {0.2f, -0.1f, 0.1f};-- public void testCreateCFS() throws IOException {- Path indexDir = getIndexDir().resolve("index.cfs");- Files.deleteIfExists(indexDir);- try (Directory dir = newFSDirectory(indexDir)) {- createIndex(dir, true, false);- }- }-- public void testCreateNoCFS() throws IOException {- Path indexDir = getIndexDir().resolve("index.nocfs");- Files.deleteIfExists(indexDir);- try (Directory dir = newFSDirectory(indexDir)) {- createIndex(dir, false, false);- }- }-- // These are only needed for the special upgrade test to verify- // that also single-segment indexes are correctly upgraded by IndexUpgrader.- // You don't need them to be built for non-4.0 (the test is happy with just one- // "old" segment format, version is unimportant:-- public void testCreateSingleSegmentCFS() throws IOException {- Path indexDir = getIndexDir().resolve("index.singlesegment-cfs");- Files.deleteIfExists(indexDir);- try (Directory dir = newFSDirectory(indexDir)) {- createIndex(dir, true, true);- }- }-- public void testCreateSingleSegmentNoCFS() throws IOException {- Path indexDir = getIndexDir().resolve("index.singlesegment-nocfs");- Files.deleteIfExists(indexDir);- try (Directory dir = newFSDirectory(indexDir)) {- createIndex(dir, false, true);- }- }-- public void testCreateIndexInternal() throws IOException {- try (Directory dir = newDirectory()) {- createIndex(dir, random().nextBoolean(), false);- searchIndex(dir, Version.LATEST.toString(), Version.MIN_SUPPORTED_MAJOR, Version.LATEST);- }- }-- private Path getIndexDir() {- String path = System.getProperty("tests.bwcdir");- assumeTrue(- "backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes",- path != null);- return Paths.get(path);- }-- public void testCreateMoreTermsIndex() throws Exception {- Path indexDir = getIndexDir().resolve("moreterms");- Files.deleteIfExists(indexDir);- try (Directory dir = newFSDirectory(indexDir)) {- createMoreTermsIndex(dir);- }- }-- public void testCreateMoreTermsIndexInternal() throws Exception {- try (Directory dir = newDirectory()) {- createMoreTermsIndex(dir);- }- }-- private void createMoreTermsIndex(Directory dir) throws Exception {- LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();- mp.setNoCFSRatio(1.0);- mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);- MockAnalyzer analyzer = new MockAnalyzer(random());- analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));-- IndexWriterConfig conf =- new IndexWriterConfig(analyzer)- .setMergePolicy(mp)- .setCodec(TestUtil.getDefaultCodec())- .setUseCompoundFile(false);- IndexWriter writer = new IndexWriter(dir, conf);- LineFileDocs docs = new LineFileDocs(new Random(0));- for (int i = 0; i < 50; i++) {- Document doc = TestUtil.cloneDocument(docs.nextDoc());- doc.add(- new NumericDocValuesField(- "docid_intDV", doc.getField("docid_int").numericValue().longValue()));- doc.add(- new SortedDocValuesField("titleDV", new BytesRef(doc.getField("title").stringValue())));- writer.addDocument(doc);- if (i % 10 == 0) { // commit every 10 documents- writer.commit();- }- }- docs.close();- writer.close();- try (DirectoryReader reader = DirectoryReader.open(dir)) {- searchExampleIndex(reader); // make sure we can search it- }- }-- // gradlew test -Ptestmethod=testCreateSortedIndex -Ptests.codec=default- // -Ptests.useSecurityManager=false -Ptests.bwcdir=/tmp/sorted --tests TestBackwardsCompatibility- public void testCreateSortedIndex() throws Exception {- Path indexDir = getIndexDir().resolve("sorted");- Files.deleteIfExists(indexDir);- try (Directory dir = newFSDirectory(indexDir)) {- createSortedIndex(dir);- }- }-- public void testCreateSortedIndexInternal() throws Exception {- // this runs without the -Ptests.bwcdir=/tmp/sorted to make sure we can actually index and- // search the created index- try (Directory dir = newDirectory()) {- createSortedIndex(dir);- }- }-- public void createSortedIndex(Directory dir) throws Exception {- LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();- mp.setNoCFSRatio(1.0);- mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);- MockAnalyzer analyzer = new MockAnalyzer(random());- analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));-- // TODO: remove randomness- IndexWriterConfig conf = new IndexWriterConfig(analyzer);- conf.setMergePolicy(mp);- conf.setUseCompoundFile(false);- conf.setCodec(TestUtil.getDefaultCodec());- conf.setIndexSort(new Sort(new SortField("dateDV", SortField.Type.LONG, true)));- IndexWriter writer = new IndexWriter(dir, conf);- LineFileDocs docs = new LineFileDocs(new Random(0));- SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);- parser.setTimeZone(TimeZone.getTimeZone("UTC"));- ParsePosition position = new ParsePosition(0);- for (int i = 0; i < 50; i++) {- Document doc = TestUtil.cloneDocument(docs.nextDoc());- String dateString = doc.get("date");- position.setIndex(0);- Date date = parser.parse(dateString, position);- if (position.getErrorIndex() != -1) {- throw new AssertionError("failed to parse \"" + dateString + "\" as date");- }- if (position.getIndex() != dateString.length()) {- throw new AssertionError("failed to parse \"" + dateString + "\" as date");- }- doc.add(- new NumericDocValuesField(- "docid_intDV", doc.getField("docid_int").numericValue().longValue()));- doc.add(- new SortedDocValuesField("titleDV", new BytesRef(doc.getField("title").stringValue())));- doc.add(new NumericDocValuesField("dateDV", date.getTime()));- if (i % 10 == 0) { // commit every 10 documents- writer.commit();- }- writer.addDocument(doc);- }- writer.forceMerge(1);- writer.close();-- try (DirectoryReader reader = DirectoryReader.open(dir)) {- searchExampleIndex(reader); // make sure we can search it- }- }-- private void updateNumeric(IndexWriter writer, String id, String f, String cf, long value)- throws IOException {- writer.updateNumericDocValue(new Term("id", id), f, value);- writer.updateNumericDocValue(new Term("id", id), cf, value * 2);- }-- private void updateBinary(IndexWriter writer, String id, String f, String cf, long value)- throws IOException {- writer.updateBinaryDocValue(new Term("id", id), f, toBytes(value));- writer.updateBinaryDocValue(new Term("id", id), cf, toBytes(value * 2));- }-- // Creates an index with DocValues updates- public void testCreateIndexWithDocValuesUpdates() throws IOException {- Path indexDir = getIndexDir().resolve("dvupdates");- Files.deleteIfExists(indexDir);- try (Directory dir = newFSDirectory(indexDir)) {- createIndexWithDocValuesUpdates(dir);- searchDocValuesUpdatesIndex(dir);- }- }-- public void testCreateIndexWithDocValuesUpdatesInternal() throws IOException {- try (Directory dir = newDirectory()) {- createIndexWithDocValuesUpdates(dir);- searchDocValuesUpdatesIndex(dir);- }- }-- private void createIndexWithDocValuesUpdates(Directory dir) throws IOException {- IndexWriterConfig conf =- new IndexWriterConfig(new MockAnalyzer(random()))- .setCodec(TestUtil.getDefaultCodec())- .setUseCompoundFile(false)- .setMergePolicy(NoMergePolicy.INSTANCE);- IndexWriter writer = new IndexWriter(dir, conf);- // create an index w/ few doc-values fields, some with updates and some without- for (int i = 0; i < 30; i++) {- Document doc = new Document();- doc.add(new StringField("id", "" + i, Field.Store.NO));- doc.add(new NumericDocValuesField("ndv1", i));- doc.add(new NumericDocValuesField("ndv1_c", i * 2));- doc.add(new NumericDocValuesField("ndv2", i * 3));- doc.add(new NumericDocValuesField("ndv2_c", i * 6));- doc.add(new BinaryDocValuesField("bdv1", toBytes(i)));- doc.add(new BinaryDocValuesField("bdv1_c", toBytes(i * 2)));- doc.add(new BinaryDocValuesField("bdv2", toBytes(i * 3)));- doc.add(new BinaryDocValuesField("bdv2_c", toBytes(i * 6)));- writer.addDocument(doc);- if ((i + 1) % 10 == 0) {- writer.commit(); // flush every 10 docs- }- }-- // first segment: no updates-- // second segment: update two fields, same gen- updateNumeric(writer, "10", "ndv1", "ndv1_c", 100L);- updateBinary(writer, "11", "bdv1", "bdv1_c", 100L);- writer.commit();-- // third segment: update few fields, different gens, few docs- updateNumeric(writer, "20", "ndv1", "ndv1_c", 100L);- updateBinary(writer, "21", "bdv1", "bdv1_c", 100L);- writer.commit();- updateNumeric(writer, "22", "ndv1", "ndv1_c", 200L); // update the field again- writer.close();- }-- public void testCreateEmptyIndex() throws Exception {- Path indexDir = getIndexDir().resolve("emptyIndex");- Files.deleteIfExists(indexDir);- IndexWriterConfig conf =- new IndexWriterConfig(new MockAnalyzer(random()))- .setUseCompoundFile(false)- .setMergePolicy(NoMergePolicy.INSTANCE);- try (Directory dir = newFSDirectory(indexDir);- IndexWriter writer = new IndexWriter(dir, conf)) {- writer.flush();- }- }-- static final String[] oldNames = {- "9.0.0-cfs", // Force on separate lines- "9.0.0-nocfs",- "9.1.0-cfs",- "9.1.0-nocfs",- "9.2.0-cfs",- "9.2.0-nocfs",- "9.3.0-cfs",- "9.3.0-nocfs",- "9.4.0-cfs",- "9.4.0-nocfs",- "9.4.1-cfs",- "9.4.1-nocfs",- "9.4.2-cfs",- "9.4.2-nocfs",- "9.5.0-cfs",- "9.5.0-nocfs",- "9.6.0-cfs",- "9.6.0-nocfs",- "9.7.0-cfs",- "9.7.0-nocfs",- "9.8.0-cfs",- "9.8.0-nocfs",- "9.9.0-cfs",- "9.9.0-nocfs",- "9.9.1-cfs",- "9.9.1-nocfs",- "9.9.2-cfs",- "9.9.2-nocfs"- };-- public static String[] getOldNames() {- return oldNames;- }-- static final String[] oldSortedNames = {- "sorted.9.0.0", // Force on separate lines- "sorted.9.1.0",- "sorted.9.2.0",- "sorted.9.3.0",- "sorted.9.4.0",- "sorted.9.4.1",- "sorted.9.4.2",- "sorted.9.5.0",- "sorted.9.6.0",- "sorted.9.7.0",- "sorted.9.8.0",- "sorted.9.9.0",- "sorted.9.9.1",- "sorted.9.9.2"- };-- public static String[] getOldSortedNames() {- return oldSortedNames;- }-- static final String[] unsupportedNames = {- "1.9.0-cfs",- "1.9.0-nocfs",- "2.0.0-cfs",- "2.0.0-nocfs",- "2.1.0-cfs",- "2.1.0-nocfs",- "2.2.0-cfs",- "2.2.0-nocfs",- "2.3.0-cfs",- "2.3.0-nocfs",- "2.4.0-cfs",- "2.4.0-nocfs",- "2.4.1-cfs",- "2.4.1-nocfs",- "2.9.0-cfs",- "2.9.0-nocfs",- "2.9.1-cfs",- "2.9.1-nocfs",- "2.9.2-cfs",- "2.9.2-nocfs",- "2.9.3-cfs",- "2.9.3-nocfs",- "2.9.4-cfs",- "2.9.4-nocfs",- "3.0.0-cfs",- "3.0.0-nocfs",- "3.0.1-cfs",- "3.0.1-nocfs",- "3.0.2-cfs",- "3.0.2-nocfs",- "3.0.3-cfs",- "3.0.3-nocfs",- "3.1.0-cfs",- "3.1.0-nocfs",- "3.2.0-cfs",- "3.2.0-nocfs",- "3.3.0-cfs",- "3.3.0-nocfs",- "3.4.0-cfs",- "3.4.0-nocfs",- "3.5.0-cfs",- "3.5.0-nocfs",- "3.6.0-cfs",- "3.6.0-nocfs",- "3.6.1-cfs",- "3.6.1-nocfs",- "3.6.2-cfs",- "3.6.2-nocfs",- "4.0.0-cfs",- "4.0.0-cfs",- "4.0.0-nocfs",- "4.0.0.1-cfs",- "4.0.0.1-nocfs",- "4.0.0.2-cfs",- "4.0.0.2-nocfs",- "4.1.0-cfs",- "4.1.0-nocfs",- "4.2.0-cfs",- "4.2.0-nocfs",- "4.2.1-cfs",- "4.2.1-nocfs",- "4.3.0-cfs",- "4.3.0-nocfs",- "4.3.1-cfs",- "4.3.1-nocfs",- "4.4.0-cfs",- "4.4.0-nocfs",- "4.5.0-cfs",- "4.5.0-nocfs",- "4.5.1-cfs",- "4.5.1-nocfs",- "4.6.0-cfs",- "4.6.0-nocfs",- "4.6.1-cfs",- "4.6.1-nocfs",- "4.7.0-cfs",- "4.7.0-nocfs",- "4.7.1-cfs",- "4.7.1-nocfs",- "4.7.2-cfs",- "4.7.2-nocfs",- "4.8.0-cfs",- "4.8.0-nocfs",- "4.8.1-cfs",- "4.8.1-nocfs",- "4.9.0-cfs",- "4.9.0-nocfs",- "4.9.1-cfs",- "4.9.1-nocfs",- "4.10.0-cfs",- "4.10.0-nocfs",- "4.10.1-cfs",- "4.10.1-nocfs",- "4.10.2-cfs",- "4.10.2-nocfs",- "4.10.3-cfs",- "4.10.3-nocfs",- "4.10.4-cfs",- "4.10.4-nocfs",- "5x-with-4x-segments-cfs",- "5x-with-4x-segments-nocfs",- "5.0.0.singlesegment-cfs",- "5.0.0.singlesegment-nocfs",- "5.0.0-cfs",- "5.0.0-nocfs",- "5.1.0-cfs",- "5.1.0-nocfs",- "5.2.0-cfs",- "5.2.0-nocfs",- "5.2.1-cfs",- "5.2.1-nocfs",- "5.3.0-cfs",- "5.3.0-nocfs",- "5.3.1-cfs",- "5.3.1-nocfs",- "5.3.2-cfs",- "5.3.2-nocfs",- "5.4.0-cfs",- "5.4.0-nocfs",- "5.4.1-cfs",- "5.4.1-nocfs",- "5.5.0-cfs",- "5.5.0-nocfs",- "5.5.1-cfs",- "5.5.1-nocfs",- "5.5.2-cfs",- "5.5.2-nocfs",- "5.5.3-cfs",- "5.5.3-nocfs",- "5.5.4-cfs",- "5.5.4-nocfs",- "5.5.5-cfs",- "5.5.5-nocfs",- "6.0.0-cfs",- "6.0.0-nocfs",- "6.0.1-cfs",- "6.0.1-nocfs",- "6.1.0-cfs",- "6.1.0-nocfs",- "6.2.0-cfs",- "6.2.0-nocfs",- "6.2.1-cfs",- "6.2.1-nocfs",- "6.3.0-cfs",- "6.3.0-nocfs",- "6.4.0-cfs",- "6.4.0-nocfs",- "6.4.1-cfs",- "6.4.1-nocfs",- "6.4.2-cfs",- "6.4.2-nocfs",- "6.5.0-cfs",- "6.5.0-nocfs",- "6.5.1-cfs",- "6.5.1-nocfs",- "6.6.0-cfs",- "6.6.0-nocfs",- "6.6.1-cfs",- "6.6.1-nocfs",- "6.6.2-cfs",- "6.6.2-nocfs",- "6.6.3-cfs",- "6.6.3-nocfs",- "6.6.4-cfs",- "6.6.4-nocfs",- "6.6.5-cfs",- "6.6.5-nocfs",- "6.6.6-cfs",- "6.6.6-nocfs",- "7.0.0-cfs",- "7.0.0-nocfs",- "7.0.1-cfs",- "7.0.1-nocfs",- "7.1.0-cfs",- "7.1.0-nocfs",- "7.2.0-cfs",- "7.2.0-nocfs",- "7.2.1-cfs",- "7.2.1-nocfs",- "7.3.0-cfs",- "7.3.0-nocfs",- "7.3.1-cfs",- "7.3.1-nocfs",- "7.4.0-cfs",- "7.4.0-nocfs",- "7.5.0-cfs",- "7.5.0-nocfs",- "7.6.0-cfs",- "7.6.0-nocfs",- "7.7.0-cfs",- "7.7.0-nocfs",- "7.7.1-cfs",- "7.7.1-nocfs",- "7.7.2-cfs",- "7.7.2-nocfs",- "7.7.3-cfs",- "7.7.3-nocfs",- "8.0.0-cfs",- "8.0.0-nocfs",- "8.1.0-cfs",- "8.1.0-nocfs",- "8.1.1-cfs",- "8.1.1-nocfs",- "8.2.0-cfs",- "8.2.0-nocfs",- "8.3.0-cfs",- "8.3.0-nocfs",- "8.3.1-cfs",- "8.3.1-nocfs",- "8.4.0-cfs",- "8.4.0-nocfs",- "8.4.1-cfs",- "8.4.1-nocfs",- "8.5.0-cfs",- "8.5.0-nocfs",- "8.5.1-cfs",- "8.5.1-nocfs",- "8.5.2-cfs",- "8.5.2-nocfs",- "8.6.0-cfs",- "8.6.0-nocfs",- "8.6.1-cfs",- "8.6.1-nocfs",- "8.6.2-cfs",- "8.6.2-nocfs",- "8.6.3-cfs",- "8.6.3-nocfs",- "8.7.0-cfs",- "8.7.0-nocfs",- "8.8.0-cfs",- "8.8.0-nocfs",- "8.8.1-cfs",- "8.8.1-nocfs",- "8.8.2-cfs",- "8.8.2-nocfs",- "8.9.0-cfs",- "8.9.0-nocfs",- "8.10.0-cfs",- "8.10.0-nocfs",- "8.10.1-cfs",- "8.10.1-nocfs",- "8.11.0-cfs",- "8.11.0-nocfs",- "8.11.1-cfs",- "8.11.1-nocfs",- "8.11.2-cfs",- "8.11.2-nocfs"- };-- static final int MIN_BINARY_SUPPORTED_MAJOR = Version.MIN_SUPPORTED_MAJOR - 1;-- static final String[] binarySupportedNames;-- static {- ArrayList<String> list = new ArrayList<>();- for (String name : unsupportedNames) {- if (name.startsWith(MIN_BINARY_SUPPORTED_MAJOR + ".")) {- list.add(name);- }- }- binarySupportedNames = list.toArray(new String[0]);- }-- // TODO: on 6.0.0 release, gen the single segment indices and add here:- static final String[] oldSingleSegmentNames = {};-- public static String[] getOldSingleSegmentNames() {- return oldSingleSegmentNames;- }-- static Map<String, Directory> oldIndexDirs;-- /** Randomizes the use of some of hte constructor variations */- private static IndexUpgrader newIndexUpgrader(Directory dir) {- final boolean streamType = random().nextBoolean();- final int choice = TestUtil.nextInt(random(), 0, 2);- switch (choice) {- case 0:- return new IndexUpgrader(dir);- case 1:- return new IndexUpgrader(dir, streamType ? null : InfoStream.NO_OUTPUT, false);- case 2:- return new IndexUpgrader(dir, newIndexWriterConfig(null), false);- default:- fail("case statement didn't get updated when random bounds changed");- }- return null; // never get here- }-- @BeforeClass- public static void beforeClass() throws Exception {- List<String> names = new ArrayList<>(oldNames.length + oldSingleSegmentNames.length);- names.addAll(Arrays.asList(oldNames));- names.addAll(Arrays.asList(oldSingleSegmentNames));- oldIndexDirs = new HashMap<>();- for (String name : names) {- Path dir = createTempDir(name);- InputStream resource =- TestBackwardsCompatibility.class.getResourceAsStream("index." + name + ".zip");- assertNotNull("Index name " + name + " not found", resource);- TestUtil.unzip(resource, dir);- oldIndexDirs.put(name, newFSDirectory(dir));- }- }-- @AfterClass- public static void afterClass() throws Exception {- for (Directory d : oldIndexDirs.values()) {- d.close();- }- oldIndexDirs = null;- }-- public void testAllVersionHaveCfsAndNocfs() {- // ensure all tested versions with cfs also have nocfs- String[] files = new String[oldNames.length];- System.arraycopy(oldNames, 0, files, 0, oldNames.length);- Arrays.sort(files);- String prevFile = "";- for (String file : files) {- if (prevFile.endsWith("-cfs")) {- String prefix = prevFile.replace("-cfs", "");- assertEquals("Missing -nocfs for backcompat index " + prefix, prefix + "-nocfs", file);- }- prevFile = file;- }- }-- public void testAllVersionsTested() throws Exception {- Pattern constantPattern = Pattern.compile("LUCENE_(\\d+)_(\\d+)_(\\d+)(_ALPHA|_BETA)?");- // find the unique versions according to Version.java- List<String> expectedVersions = new ArrayList<>();- for (java.lang.reflect.Field field : Version.class.getDeclaredFields()) {- if (Modifier.isStatic(field.getModifiers()) && field.getType() == Version.class) {- Version v = (Version) field.get(Version.class);- if (v.equals(Version.LATEST)) {- continue;- }-- Matcher constant = constantPattern.matcher(field.getName());- if (constant.matches() == false) {- continue;- }-- expectedVersions.add(v + "-cfs");- }- }-- // BEGIN TRUNK ONLY BLOCK- // on trunk, the last release of the prev major release is also untested- Version lastPrevMajorVersion = null;- for (java.lang.reflect.Field field : Version.class.getDeclaredFields()) {- if (Modifier.isStatic(field.getModifiers()) && field.getType() == Version.class) {- Version v = (Version) field.get(Version.class);- Matcher constant = constantPattern.matcher(field.getName());- if (constant.matches() == false) continue;- if (v.major == Version.LATEST.major - 1- && (lastPrevMajorVersion == null || v.onOrAfter(lastPrevMajorVersion))) {- lastPrevMajorVersion = v;- }- }- }- assertNotNull(lastPrevMajorVersion);- expectedVersions.remove(lastPrevMajorVersion + "-cfs");- // END TRUNK ONLY BLOCK-- Collections.sort(expectedVersions);-- // find what versions we are testing- List<String> testedVersions = new ArrayList<>();- for (String testedVersion : oldNames) {- if (testedVersion.endsWith("-cfs") == false) {- continue;- }- testedVersions.add(testedVersion);- }- Collections.sort(testedVersions);-- int i = 0;- int j = 0;- List<String> missingFiles = new ArrayList<>();- List<String> extraFiles = new ArrayList<>();- while (i < expectedVersions.size() && j < testedVersions.size()) {- String expectedVersion = expectedVersions.get(i);- String testedVersion = testedVersions.get(j);- int compare = expectedVersion.compareTo(testedVersion);- if (compare == 0) { // equal, we can move on- ++i;- ++j;- } else if (compare < 0) { // didn't find test for version constant- missingFiles.add(expectedVersion);- ++i;- } else { // extra test file- extraFiles.add(testedVersion);- ++j;- }- }- while (i < expectedVersions.size()) {- missingFiles.add(expectedVersions.get(i));- ++i;- }- while (j < testedVersions.size()) {- missingFiles.add(testedVersions.get(j));- ++j;- }-- // we could be missing up to 1 file, which may be due to a release that is in progress- if (missingFiles.size() <= 1 && extraFiles.isEmpty()) {- // success- return;- }-- StringBuilder msg = new StringBuilder();- if (missingFiles.size() > 1) {- msg.append("Missing backcompat test files:\n");- for (String missingFile : missingFiles) {- msg.append(" ").append(missingFile).append("\n");- }- }- if (extraFiles.isEmpty() == false) {- msg.append("Extra backcompat test files:\n");- for (String extraFile : extraFiles) {- msg.append(" ").append(extraFile).append("\n");- }- }- fail(msg.toString());- }-- /**- * This test checks that *only* IndexFormatTooOldExceptions are thrown when you open and operate- * on too old indexes!- */- public void testUnsupportedOldIndexes() throws Exception {- for (int i = 0; i < unsupportedNames.length; i++) {- if (VERBOSE) {- System.out.println("TEST: index " + unsupportedNames[i]);- }- Path oldIndexDir = createTempDir(unsupportedNames[i]);- TestUtil.unzip(- getDataInputStream("unsupported." + unsupportedNames[i] + ".zip"), oldIndexDir);- BaseDirectoryWrapper dir = newFSDirectory(oldIndexDir);- // don't checkindex, these are intentionally not supported- dir.setCheckIndexOnClose(false);-- IndexReader reader = null;- IndexWriter writer = null;- try {- reader = DirectoryReader.open(dir);- fail("DirectoryReader.open should not pass for " + unsupportedNames[i]);- } catch (IndexFormatTooOldException e) {- if (e.getReason() != null) {- assertNull(e.getVersion());- assertNull(e.getMinVersion());- assertNull(e.getMaxVersion());- assertEquals(- e.getMessage(),- new IndexFormatTooOldException(e.getResourceDescription(), e.getReason())- .getMessage());- } else {- assertNotNull(e.getVersion());- assertNotNull(e.getMinVersion());- assertNotNull(e.getMaxVersion());- assertTrue(e.getMessage(), e.getMaxVersion() >= e.getMinVersion());- assertTrue(- e.getMessage(),- e.getMaxVersion() < e.getVersion() || e.getVersion() < e.getMinVersion());- assertEquals(- e.getMessage(),- new IndexFormatTooOldException(- e.getResourceDescription(),- e.getVersion(),- e.getMinVersion(),- e.getMaxVersion())- .getMessage());- }- // pass- if (VERBOSE) {- System.out.println("TEST: got expected exc:");- e.printStackTrace(System.out);- }- } finally {- if (reader != null) reader.close();- reader = null;- }-- try {- writer =- new IndexWriter(- dir, newIndexWriterConfig(new MockAnalyzer(random())).setCommitOnClose(false));- fail("IndexWriter creation should not pass for " + unsupportedNames[i]);- } catch (IndexFormatTooOldException e) {- if (e.getReason() != null) {- assertNull(e.getVersion());- assertNull(e.getMinVersion());- assertNull(e.getMaxVersion());- assertEquals(- e.getMessage(),- new IndexFormatTooOldException(e.getResourceDescription(), e.getReason())- .getMessage());- } else {- assertNotNull(e.getVersion());- assertNotNull(e.getMinVersion());- assertNotNull(e.getMaxVersion());- assertTrue(e.getMessage(), e.getMaxVersion() >= e.getMinVersion());- assertTrue(- e.getMessage(),- e.getMaxVersion() < e.getVersion() || e.getVersion() < e.getMinVersion());- assertEquals(- e.getMessage(),- new IndexFormatTooOldException(- e.getResourceDescription(),- e.getVersion(),- e.getMinVersion(),- e.getMaxVersion())- .getMessage());- }- // pass- if (VERBOSE) {- System.out.println("TEST: got expected exc:");- e.printStackTrace(System.out);- }- // Make sure exc message includes a path=- assertTrue("got exc message: " + e.getMessage(), e.getMessage().contains("path=\""));- } finally {- // we should fail to open IW, and so it should be null when we get here.- // However, if the test fails (i.e., IW did not fail on open), we need- // to close IW. However, if merges are run, IW may throw- // IndexFormatTooOldException, and we don't want to mask the fail()- // above, so close without waiting for merges.- if (writer != null) {- try {- writer.commit();- } finally {- writer.close();- }- }- }-- ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);- CheckIndex checker = new CheckIndex(dir);- checker.setInfoStream(new PrintStream(bos, false, UTF_8));- CheckIndex.Status indexStatus = checker.checkIndex();- if (unsupportedNames[i].startsWith("8.")) {- assertTrue(indexStatus.clean);- } else {- assertFalse(indexStatus.clean);- // CheckIndex doesn't enforce a minimum version, so we either get an- // IndexFormatTooOldException- // or an IllegalArgumentException saying that the codec doesn't exist.- boolean formatTooOld =- bos.toString(UTF_8).contains(IndexFormatTooOldException.class.getName());- boolean missingCodec = bos.toString(UTF_8).contains("Could not load codec");- assertTrue(formatTooOld || missingCodec);- }- checker.close();-- dir.close();- }- }-- public void testFullyMergeOldIndex() throws Exception {- for (String name : oldNames) {- if (VERBOSE) {- System.out.println("\nTEST: index=" + name);- }- Directory dir = newDirectory(oldIndexDirs.get(name));-- final SegmentInfos oldSegInfos = SegmentInfos.readLatestCommit(dir);-- IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));- w.forceMerge(1);- w.close();-- final SegmentInfos segInfos = SegmentInfos.readLatestCommit(dir);- assertEquals(- oldSegInfos.getIndexCreatedVersionMajor(), segInfos.getIndexCreatedVersionMajor());- assertEquals(Version.LATEST, segInfos.asList().get(0).info.getVersion());- assertEquals(- oldSegInfos.asList().get(0).info.getMinVersion(),- segInfos.asList().get(0).info.getMinVersion());-- dir.close();- }- }-- public void testAddOldIndexes() throws IOException {- for (String name : oldNames) {- if (VERBOSE) {- System.out.println("\nTEST: old index " + name);- }- Directory oldDir = oldIndexDirs.get(name);- SegmentInfos infos = SegmentInfos.readLatestCommit(oldDir);-- Directory targetDir = newDirectory();- if (infos.getCommitLuceneVersion().major != Version.LATEST.major) {- // both indexes are not compatible- Directory targetDir2 = newDirectory();- IndexWriter w =- new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));- IllegalArgumentException e =- expectThrows(IllegalArgumentException.class, () -> w.addIndexes(oldDir));- assertTrue(- e.getMessage(),- e.getMessage()- .startsWith(- "Cannot use addIndexes(Directory) with indexes that have been created by a different Lucene version."));- w.close();- targetDir2.close();-- // for the next test, we simulate writing to an index that was created on the same major- // version- new SegmentInfos(infos.getIndexCreatedVersionMajor()).commit(targetDir);- }-- IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));- w.addIndexes(oldDir);- w.close();-- SegmentInfos si = SegmentInfos.readLatestCommit(targetDir);- assertNull(- "none of the segments should have been upgraded",- si.asList().stream()- .filter( // depending on the MergePolicy we might see these segments merged away- sci ->- sci.getId() != null- && sci.info.getVersion().onOrAfter(Version.fromBits(8, 6, 0)) == false)- .findAny()- .orElse(null));- if (VERBOSE) {- System.out.println("\nTEST: done adding indices; now close");- }-- targetDir.close();- }- }-- public void testAddOldIndexesReader() throws IOException {- for (String name : oldNames) {- Directory oldDir = oldIndexDirs.get(name);- SegmentInfos infos = SegmentInfos.readLatestCommit(oldDir);- DirectoryReader reader = DirectoryReader.open(oldDir);-- Directory targetDir = newDirectory();- if (infos.getCommitLuceneVersion().major != Version.LATEST.major) {- Directory targetDir2 = newDirectory();- IndexWriter w =- new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));- IllegalArgumentException e =- expectThrows(- IllegalArgumentException.class, () -> TestUtil.addIndexesSlowly(w, reader));- assertEquals(- e.getMessage(),- "Cannot merge a segment that has been created with major version 9 into this index which has been created by major version 10");- w.close();- targetDir2.close();-- // for the next test, we simulate writing to an index that was created on the same major- // version- new SegmentInfos(infos.getIndexCreatedVersionMajor()).commit(targetDir);- }- IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));- TestUtil.addIndexesSlowly(w, reader);- w.close();- reader.close();- SegmentInfos si = SegmentInfos.readLatestCommit(targetDir);- assertNull(- "all SCIs should have an id now",- si.asList().stream().filter(sci -> sci.getId() == null).findAny().orElse(null));- targetDir.close();- }- }-- public void testSearchOldIndex() throws Exception {- for (String name : oldNames) {- Version version = Version.parse(name.substring(0, name.indexOf('-')));- searchIndex(oldIndexDirs.get(name), name, Version.MIN_SUPPORTED_MAJOR, version);- }-- if (TEST_NIGHTLY) {- for (String name : binarySupportedNames) {- Path oldIndexDir = createTempDir(name);- TestUtil.unzip(getDataInputStream("unsupported." + name + ".zip"), oldIndexDir);- try (BaseDirectoryWrapper dir = newFSDirectory(oldIndexDir)) {- Version version = Version.parse(name.substring(0, name.indexOf('-')));- searchIndex(dir, name, MIN_BINARY_SUPPORTED_MAJOR, version);- }- }- }- }-- public void testIndexOldIndexNoAdds() throws Exception {- for (String name : oldNames) {- Directory dir = newDirectory(oldIndexDirs.get(name));- Version version = Version.parse(name.substring(0, name.indexOf('-')));- changeIndexNoAdds(random(), dir, version);- dir.close();- }- }-- public void testIndexOldIndex() throws Exception {- for (String name : oldNames) {- if (VERBOSE) {- System.out.println("TEST: oldName=" + name);- }- Directory dir = newDirectory(oldIndexDirs.get(name));- Version v = Version.parse(name.substring(0, name.indexOf('-')));- changeIndexWithAdds(random(), dir, v);- dir.close();- }- }-- private void doTestHits(ScoreDoc[] hits, int expectedCount, IndexReader reader)- throws IOException {- final int hitCount = hits.length;- assertEquals("wrong number of hits", expectedCount, hitCount);- StoredFields storedFields = reader.storedFields();- TermVectors termVectors = reader.termVectors();- for (ScoreDoc hit : hits) {- storedFields.document(hit.doc);- termVectors.get(hit.doc);- }- }-- public void searchIndex(- Directory dir, String oldName, int minIndexMajorVersion, Version nameVersion)- throws IOException {- // QueryParser parser = new QueryParser("contents", new MockAnalyzer(random));- // Query query = parser.parse("handle:1");- IndexCommit indexCommit = DirectoryReader.listCommits(dir).get(0);- IndexReader reader = DirectoryReader.open(indexCommit, minIndexMajorVersion, null);- IndexSearcher searcher = newSearcher(reader);-- TestUtil.checkIndex(dir);-- final Bits liveDocs = MultiBits.getLiveDocs(reader);- assertNotNull(liveDocs);-- StoredFields storedFields = reader.storedFields();- TermVectors termVectors = reader.termVectors();-- for (int i = 0; i < DOCS_COUNT; i++) {- if (liveDocs.get(i)) {- Document d = storedFields.document(i);- List<IndexableField> fields = d.getFields();- boolean isProxDoc = d.getField("content3") == null;- if (isProxDoc) {- assertEquals(7, fields.size());- IndexableField f = d.getField("id");- assertEquals("" + i, f.stringValue());-- f = d.getField("utf8");- assertEquals(- "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());-- f = d.getField("autf8");- assertEquals(- "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());-- f = d.getField("content2");- assertEquals("here is more content with aaa aaa aaa", f.stringValue());-- f = d.getField("fie\u2C77ld");- assertEquals("field with non-ascii name", f.stringValue());- }-- Fields tfvFields = termVectors.get(i);- assertNotNull("i=" + i, tfvFields);- Terms tfv = tfvFields.terms("utf8");- assertNotNull("docID=" + i + " index=" + oldName, tfv);- } else {- assertEquals(DELETED_ID, i);- }- }-- // check docvalues fields- NumericDocValues dvByte = MultiDocValues.getNumericValues(reader, "dvByte");- BinaryDocValues dvBytesDerefFixed = MultiDocValues.getBinaryValues(reader, "dvBytesDerefFixed");- BinaryDocValues dvBytesDerefVar = MultiDocValues.getBinaryValues(reader, "dvBytesDerefVar");- SortedDocValues dvBytesSortedFixed =- MultiDocValues.getSortedValues(reader, "dvBytesSortedFixed");- SortedDocValues dvBytesSortedVar = MultiDocValues.getSortedValues(reader, "dvBytesSortedVar");- BinaryDocValues dvBytesStraightFixed =- MultiDocValues.getBinaryValues(reader, "dvBytesStraightFixed");- BinaryDocValues dvBytesStraightVar =- MultiDocValues.getBinaryValues(reader, "dvBytesStraightVar");- NumericDocValues dvDouble = MultiDocValues.getNumericValues(reader, "dvDouble");- NumericDocValues dvFloat = MultiDocValues.getNumericValues(reader, "dvFloat");- NumericDocValues dvInt = MultiDocValues.getNumericValues(reader, "dvInt");- NumericDocValues dvLong = MultiDocValues.getNumericValues(reader, "dvLong");- NumericDocValues dvPacked = MultiDocValues.getNumericValues(reader, "dvPacked");- NumericDocValues dvShort = MultiDocValues.getNumericValues(reader, "dvShort");-- SortedSetDocValues dvSortedSet = MultiDocValues.getSortedSetValues(reader, "dvSortedSet");- SortedNumericDocValues dvSortedNumeric =- MultiDocValues.getSortedNumericValues(reader, "dvSortedNumeric");-- for (int i = 0; i < DOCS_COUNT; i++) {- int id = Integer.parseInt(storedFields.document(i).get("id"));- assertEquals(i, dvByte.nextDoc());- assertEquals(id, dvByte.longValue());-- byte[] bytes =- new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};- BytesRef expectedRef = new BytesRef(bytes);-- assertEquals(i, dvBytesDerefFixed.nextDoc());- BytesRef term = dvBytesDerefFixed.binaryValue();- assertEquals(expectedRef, term);- assertEquals(i, dvBytesDerefVar.nextDoc());- term = dvBytesDerefVar.binaryValue();- assertEquals(expectedRef, term);- assertEquals(i, dvBytesSortedFixed.nextDoc());- term = dvBytesSortedFixed.lookupOrd(dvBytesSortedFixed.ordValue());- assertEquals(expectedRef, term);- assertEquals(i, dvBytesSortedVar.nextDoc());- term = dvBytesSortedVar.lookupOrd(dvBytesSortedVar.ordValue());- assertEquals(expectedRef, term);- assertEquals(i, dvBytesStraightFixed.nextDoc());- term = dvBytesStraightFixed.binaryValue();- assertEquals(expectedRef, term);- assertEquals(i, dvBytesStraightVar.nextDoc());- term = dvBytesStraightVar.binaryValue();- assertEquals(expectedRef, term);-- assertEquals(i, dvDouble.nextDoc());- assertEquals(id, Double.longBitsToDouble(dvDouble.longValue()), 0D);- assertEquals(i, dvFloat.nextDoc());- assertEquals((float) id, Float.intBitsToFloat((int) dvFloat.longValue()), 0F);- assertEquals(i, dvInt.nextDoc());- assertEquals(id, dvInt.longValue());- assertEquals(i, dvLong.nextDoc());- assertEquals(id, dvLong.longValue());- assertEquals(i, dvPacked.nextDoc());- assertEquals(id, dvPacked.longValue());- assertEquals(i, dvShort.nextDoc());- assertEquals(id, dvShort.longValue());-- assertEquals(i, dvSortedSet.nextDoc());- assertEquals(1, dvSortedSet.docValueCount());- long ord = dvSortedSet.nextOrd();- term = dvSortedSet.lookupOrd(ord);- assertEquals(expectedRef, term);-- assertEquals(i, dvSortedNumeric.nextDoc());- assertEquals(1, dvSortedNumeric.docValueCount());- assertEquals(id, dvSortedNumeric.nextValue());- }-- ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;-- // First document should be #0- Document d = storedFields.document(hits[0].doc);- assertEquals("didn't get the right document first", "0", d.get("id"));-- doTestHits(hits, 34, searcher.getIndexReader());-- hits = searcher.search(new TermQuery(new Term("content5", "aaa")), 1000).scoreDocs;-- doTestHits(hits, 34, searcher.getIndexReader());-- hits = searcher.search(new TermQuery(new Term("content6", "aaa")), 1000).scoreDocs;-- doTestHits(hits, 34, searcher.getIndexReader());-- hits = searcher.search(new TermQuery(new Term("utf8", "\u0000")), 1000).scoreDocs;- assertEquals(34, hits.length);- hits =- searcher.search(new TermQuery(new Term("utf8", "lu\uD834\uDD1Ece\uD834\uDD60ne")), 1000)- .scoreDocs;- assertEquals(34, hits.length);- hits = searcher.search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), 1000).scoreDocs;- assertEquals(34, hits.length);-- doTestHits(- searcher.search(IntPoint.newRangeQuery("intPoint1d", 0, 34), 1000).scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(- IntPoint.newRangeQuery("intPoint2d", new int[] {0, 0}, new int[] {34, 68}), 1000)- .scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(FloatPoint.newRangeQuery("floatPoint1d", 0f, 34f), 1000).scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(- FloatPoint.newRangeQuery(- "floatPoint2d", new float[] {0f, 0f}, new float[] {34f, 68f}),- 1000)- .scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(LongPoint.newRangeQuery("longPoint1d", 0, 34), 1000).scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(- LongPoint.newRangeQuery("longPoint2d", new long[] {0, 0}, new long[] {34, 68}),- 1000)- .scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(DoublePoint.newRangeQuery("doublePoint1d", 0.0, 34.0), 1000).scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(- DoublePoint.newRangeQuery(- "doublePoint2d", new double[] {0.0, 0.0}, new double[] {34.0, 68.0}),- 1000)- .scoreDocs,- 34,- searcher.getIndexReader());-- byte[] bytes1 = new byte[4];- byte[] bytes2 = new byte[] {0, 0, 0, (byte) 34};- doTestHits(- searcher.search(BinaryPoint.newRangeQuery("binaryPoint1d", bytes1, bytes2), 1000).scoreDocs,- 34,- searcher.getIndexReader());- byte[] bytes3 = new byte[] {0, 0, 0, (byte) 68};- doTestHits(- searcher.search(- BinaryPoint.newRangeQuery(- "binaryPoint2d", new byte[][] {bytes1, bytes1}, new byte[][] {bytes2, bytes3}),- 1000)- .scoreDocs,- 34,- searcher.getIndexReader());-- // test vector values and KNN search- if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {- // test vector values- int cnt = 0;- for (LeafReaderContext ctx : reader.leaves()) {- FloatVectorValues values = ctx.reader().getFloatVectorValues(KNN_VECTOR_FIELD);- if (values != null) {- assertEquals(KNN_VECTOR_FIELD_TYPE.vectorDimension(), values.dimension());- for (int doc = values.nextDoc(); doc != NO_MORE_DOCS; doc = values.nextDoc()) {- float[] expectedVector = {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * cnt};- assertArrayEquals(- "vectors do not match for doc=" + cnt, expectedVector, values.vectorValue(), 0);- cnt++;- }- }- }- assertEquals(DOCS_COUNT, cnt);-- // test KNN search- ScoreDoc[] scoreDocs = assertKNNSearch(searcher, KNN_VECTOR, 10, 10, "0");- for (int i = 0; i < scoreDocs.length; i++) {- int id = Integer.parseInt(storedFields.document(scoreDocs[i].doc).get("id"));- int expectedId = i < DELETED_ID ? i : i + 1;- assertEquals(expectedId, id);- }- }-- reader.close();- }-- private static ScoreDoc[] assertKNNSearch(- IndexSearcher searcher,- float[] queryVector,- int k,- int expectedHitsCount,- String expectedFirstDocId)- throws IOException {- ScoreDoc[] hits =- searcher.search(new KnnFloatVectorQuery(KNN_VECTOR_FIELD, queryVector, k), k).scoreDocs;- assertEquals("wrong number of hits", expectedHitsCount, hits.length);- Document d = searcher.storedFields().document(hits[0].doc);- assertEquals("wrong first document", expectedFirstDocId, d.get("id"));- return hits;- }-- public void changeIndexWithAdds(Random random, Directory dir, Version nameVersion)- throws IOException {- SegmentInfos infos = SegmentInfos.readLatestCommit(dir);- assertEquals(nameVersion, infos.getCommitLuceneVersion());- assertEquals(nameVersion, infos.getMinSegmentLuceneVersion());-- // open writer- IndexWriter writer =- new IndexWriter(- dir,- newIndexWriterConfig(new MockAnalyzer(random))- .setOpenMode(OpenMode.APPEND)- .setMergePolicy(newLogMergePolicy()));- // add 10 docs- for (int i = 0; i < 10; i++) {- addDoc(writer, DOCS_COUNT + i);- }-- // make sure writer sees right total -- writer seems not to know about deletes in .del?- final int expected = 45;- assertEquals("wrong doc count", expected, writer.getDocStats().numDocs);- writer.close();-- // make sure searching sees right # hits for term search- IndexReader reader = DirectoryReader.open(dir);- IndexSearcher searcher = newSearcher(reader);- ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;- Document d = searcher.getIndexReader().storedFields().document(hits[0].doc);- assertEquals("wrong first document", "0", d.get("id"));- doTestHits(hits, 44, searcher.getIndexReader());-- if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {- // make sure KNN search sees all hits (graph may not be used if k is big)- assertKNNSearch(searcher, KNN_VECTOR, 1000, 44, "0");- // make sure KNN search using HNSW graph sees newly added docs- assertKNNSearch(- searcher,- new float[] {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * 44},- 10,- 10,- "44");- }- reader.close();-- // fully merge- writer =- new IndexWriter(- dir,- newIndexWriterConfig(new MockAnalyzer(random))- .setOpenMode(OpenMode.APPEND)- .setMergePolicy(newLogMergePolicy()));- writer.forceMerge(1);- writer.close();-- reader = DirectoryReader.open(dir);- searcher = newSearcher(reader);- // make sure searching sees right # hits fot term search- hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;- assertEquals("wrong number of hits", 44, hits.length);- d = searcher.storedFields().document(hits[0].doc);- doTestHits(hits, 44, searcher.getIndexReader());- assertEquals("wrong first document", "0", d.get("id"));-- if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {- // make sure KNN search sees all hits- assertKNNSearch(searcher, KNN_VECTOR, 1000, 44, "0");- // make sure KNN search using HNSW graph sees newly added docs- assertKNNSearch(- searcher,- new float[] {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * 44},- 10,- 10,- "44");- }- reader.close();- }-- public void changeIndexNoAdds(Random random, Directory dir, Version nameVersion)- throws IOException {- // make sure searching sees right # hits for term search- DirectoryReader reader = DirectoryReader.open(dir);- IndexSearcher searcher = newSearcher(reader);- ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;- assertEquals("wrong number of hits", 34, hits.length);- Document d = searcher.storedFields().document(hits[0].doc);- assertEquals("wrong first document", "0", d.get("id"));-- if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {- // make sure KNN search sees all hits- assertKNNSearch(searcher, KNN_VECTOR, 1000, 34, "0");- // make sure KNN search using HNSW graph retrieves correct results- assertKNNSearch(searcher, KNN_VECTOR, 10, 10, "0");- }- reader.close();-- // fully merge- IndexWriter writer =- new IndexWriter(- dir, newIndexWriterConfig(new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));- writer.forceMerge(1);- writer.close();-- reader = DirectoryReader.open(dir);- searcher = newSearcher(reader);- // make sure searching sees right # hits fot term search- hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;- assertEquals("wrong number of hits", 34, hits.length);- doTestHits(hits, 34, searcher.getIndexReader());- // make sure searching sees right # hits for KNN search- if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {- // make sure KNN search sees all hits- assertKNNSearch(searcher, KNN_VECTOR, 1000, 34, "0");- // make sure KNN search using HNSW graph retrieves correct results- assertKNNSearch(searcher, KNN_VECTOR, 10, 10, "0");- }- reader.close();- }-- public void createIndex(Directory dir, boolean doCFS, boolean fullyMerged) throws IOException {- LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();- mp.setNoCFSRatio(doCFS ? 1.0 : 0.0);- mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);- // TODO: remove randomness- IndexWriterConfig conf =- new IndexWriterConfig(new MockAnalyzer(random()))- .setMaxBufferedDocs(10)- .setMergePolicy(NoMergePolicy.INSTANCE);- IndexWriter writer = new IndexWriter(dir, conf);-- for (int i = 0; i < DOCS_COUNT; i++) {- addDoc(writer, i);- }- assertEquals("wrong doc count", DOCS_COUNT, writer.getDocStats().maxDoc);- if (fullyMerged) {- writer.forceMerge(1);- }- writer.close();-- if (!fullyMerged) {- // open fresh writer so we get no prx file in the added segment- mp = new LogByteSizeMergePolicy();- mp.setNoCFSRatio(doCFS ? 1.0 : 0.0);- // TODO: remove randomness- conf =- new IndexWriterConfig(new MockAnalyzer(random()))- .setMaxBufferedDocs(10)- .setMergePolicy(NoMergePolicy.INSTANCE);- writer = new IndexWriter(dir, conf);- addNoProxDoc(writer);- writer.close();-- conf =- new IndexWriterConfig(new MockAnalyzer(random()))- .setMaxBufferedDocs(10)- .setMergePolicy(NoMergePolicy.INSTANCE);- writer = new IndexWriter(dir, conf);- Term searchTerm = new Term("id", String.valueOf(DELETED_ID));- writer.deleteDocuments(searchTerm);- writer.close();- }- }-- private void addDoc(IndexWriter writer, int id) throws IOException {- Document doc = new Document();- doc.add(new TextField("content", "aaa", Field.Store.NO));- doc.add(new StringField("id", Integer.toString(id), Field.Store.YES));- FieldType customType2 = new FieldType(TextField.TYPE_STORED);- customType2.setStoreTermVectors(true);- customType2.setStoreTermVectorPositions(true);- customType2.setStoreTermVectorOffsets(true);- doc.add(- new Field(- "autf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));- doc.add(- new Field(- "utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));- doc.add(new Field("content2", "here is more content with aaa aaa aaa", customType2));- doc.add(new Field("fie\u2C77ld", "field with non-ascii name", customType2));-- // add docvalues fields- doc.add(new NumericDocValuesField("dvByte", (byte) id));- byte[] bytes =- new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};- BytesRef ref = new BytesRef(bytes);- doc.add(new BinaryDocValuesField("dvBytesDerefFixed", ref));- doc.add(new BinaryDocValuesField("dvBytesDerefVar", ref));- doc.add(new SortedDocValuesField("dvBytesSortedFixed", ref));- doc.add(new SortedDocValuesField("dvBytesSortedVar", ref));- doc.add(new BinaryDocValuesField("dvBytesStraightFixed", ref));- doc.add(new BinaryDocValuesField("dvBytesStraightVar", ref));- doc.add(new DoubleDocValuesField("dvDouble", id));- doc.add(new FloatDocValuesField("dvFloat", (float) id));- doc.add(new NumericDocValuesField("dvInt", id));- doc.add(new NumericDocValuesField("dvLong", id));- doc.add(new NumericDocValuesField("dvPacked", id));- doc.add(new NumericDocValuesField("dvShort", (short) id));- doc.add(new SortedSetDocValuesField("dvSortedSet", ref));- doc.add(new SortedNumericDocValuesField("dvSortedNumeric", id));-- doc.add(new IntPoint("intPoint1d", id));- doc.add(new IntPoint("intPoint2d", id, 2 * id));- doc.add(new FloatPoint("floatPoint1d", (float) id));- doc.add(new FloatPoint("floatPoint2d", (float) id, (float) 2 * id));- doc.add(new LongPoint("longPoint1d", id));- doc.add(new LongPoint("longPoint2d", id, 2 * id));- doc.add(new DoublePoint("doublePoint1d", id));- doc.add(new DoublePoint("doublePoint2d", id, (double) 2 * id));- doc.add(new BinaryPoint("binaryPoint1d", bytes));- doc.add(new BinaryPoint("binaryPoint2d", bytes, bytes));-- // a field with both offsets and term vectors for a cross-check- FieldType customType3 = new FieldType(TextField.TYPE_STORED);- customType3.setStoreTermVectors(true);- customType3.setStoreTermVectorPositions(true);- customType3.setStoreTermVectorOffsets(true);- customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);- doc.add(new Field("content5", "here is more content with aaa aaa aaa", customType3));- // a field that omits only positions- FieldType customType4 = new FieldType(TextField.TYPE_STORED);- customType4.setStoreTermVectors(true);- customType4.setStoreTermVectorPositions(false);- customType4.setStoreTermVectorOffsets(true);- customType4.setIndexOptions(IndexOptions.DOCS_AND_FREQS);- doc.add(new Field("content6", "here is more content with aaa aaa aaa", customType4));-- float[] vector = {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * id};- doc.add(new KnnFloatVectorField(KNN_VECTOR_FIELD, vector, KNN_VECTOR_FIELD_TYPE));-- // TODO:- // index different norms types via similarity (we use a random one currently?!)- // remove any analyzer randomness, explicitly add payloads for certain fields.- writer.addDocument(doc);- }-- private void addNoProxDoc(IndexWriter writer) throws IOException {- Document doc = new Document();- FieldType customType = new FieldType(TextField.TYPE_STORED);- customType.setIndexOptions(IndexOptions.DOCS);- Field f = new Field("content3", "aaa", customType);- doc.add(f);- FieldType customType2 = new FieldType();- customType2.setStored(true);- customType2.setIndexOptions(IndexOptions.DOCS);- f = new Field("content4", "aaa", customType2);- doc.add(f);- writer.addDocument(doc);- }-- private int countDocs(PostingsEnum docs) throws IOException {- int count = 0;- while ((docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {- count++;- }- return count;- }-- // flex: test basics of TermsEnum api on non-flex index- public void testNextIntoWrongField() throws Exception {- for (String name : oldNames) {- Directory dir = oldIndexDirs.get(name);- IndexReader r = DirectoryReader.open(dir);- TermsEnum terms = MultiTerms.getTerms(r, "content").iterator();- BytesRef t = terms.next();- assertNotNull(t);-- // content field only has term aaa:- assertEquals("aaa", t.utf8ToString());- assertNull(terms.next());-- BytesRef aaaTerm = new BytesRef("aaa");-- // should be found exactly- assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(aaaTerm));- assertEquals(DOCS_COUNT, countDocs(TestUtil.docs(random(), terms, null, PostingsEnum.NONE)));- assertNull(terms.next());-- // should hit end of field- assertEquals(TermsEnum.SeekStatus.END, terms.seekCeil(new BytesRef("bbb")));- assertNull(terms.next());-- // should seek to aaa- assertEquals(TermsEnum.SeekStatus.NOT_FOUND, terms.seekCeil(new BytesRef("a")));- assertTrue(terms.term().bytesEquals(aaaTerm));- assertEquals(DOCS_COUNT, countDocs(TestUtil.docs(random(), terms, null, PostingsEnum.NONE)));- assertNull(terms.next());-- assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(aaaTerm));- assertEquals(DOCS_COUNT, countDocs(TestUtil.docs(random(), terms, null, PostingsEnum.NONE)));- assertNull(terms.next());-- r.close();- }- }-- /**- * Test that we didn't forget to bump the current Constants.LUCENE_MAIN_VERSION. This is important- * so that we can determine which version of lucene wrote the segment.- */- public void testOldVersions() throws Exception {- // first create a little index with the current code and get the version- Directory currentDir = newDirectory();- RandomIndexWriter riw = new RandomIndexWriter(random(), currentDir);- riw.addDocument(new Document());- riw.close();- DirectoryReader ir = DirectoryReader.open(currentDir);- SegmentReader air = (SegmentReader) ir.leaves().get(0).reader();- Version currentVersion = air.getSegmentInfo().info.getVersion();- assertNotNull(currentVersion); // only 3.0 segments can have a null version- ir.close();- currentDir.close();-- // now check all the old indexes, their version should be < the current version- for (String name : oldNames) {- Directory dir = oldIndexDirs.get(name);- DirectoryReader r = DirectoryReader.open(dir);- for (LeafReaderContext context : r.leaves()) {- air = (SegmentReader) context.reader();- Version oldVersion = air.getSegmentInfo().info.getVersion();- assertNotNull(oldVersion); // only 3.0 segments can have a null version- assertTrue(- "current Version.LATEST is <= an old index: did you forget to bump it?!",- currentVersion.onOrAfter(oldVersion));- }- r.close();- }- }-- public void testIndexCreatedVersion() throws IOException {- for (String name : oldNames) {- Directory dir = oldIndexDirs.get(name);- SegmentInfos infos = SegmentInfos.readLatestCommit(dir);- // those indexes are created by a single version so we can- // compare the commit version with the created version- assertEquals(infos.getCommitLuceneVersion().major, infos.getIndexCreatedVersionMajor());- }- }-- public void testSegmentCommitInfoId() throws IOException {- for (String name : oldNames) {- Directory dir = oldIndexDirs.get(name);- SegmentInfos infos = SegmentInfos.readLatestCommit(dir);- for (SegmentCommitInfo info : infos) {- if (info.info.getVersion().onOrAfter(Version.fromBits(8, 6, 0))) {- assertNotNull(info.toString(), info.getId());- } else {- assertNull(info.toString(), info.getId());- }- }- }- }-- public void verifyUsesDefaultCodec(Directory dir, String name) throws IOException {- DirectoryReader r = DirectoryReader.open(dir);- for (LeafReaderContext context : r.leaves()) {- SegmentReader air = (SegmentReader) context.reader();- Codec codec = air.getSegmentInfo().info.getCodec();- assertTrue(- "codec used in "- + name- + " ("- + codec.getName()- + ") is not a default codec (does not begin with Lucene)",- codec.getName().startsWith("Lucene"));- }- r.close();- }-- public void testAllIndexesUseDefaultCodec() throws Exception {- for (String name : oldNames) {- Directory dir = oldIndexDirs.get(name);- verifyUsesDefaultCodec(dir, name);- }- }-- private int checkAllSegmentsUpgraded(Directory dir, int indexCreatedVersion) throws IOException {- final SegmentInfos infos = SegmentInfos.readLatestCommit(dir);- if (VERBOSE) {- System.out.println("checkAllSegmentsUpgraded: " + infos);- }- for (SegmentCommitInfo si : infos) {- assertEquals(Version.LATEST, si.info.getVersion());- assertNotNull(si.getId());- }- assertEquals(Version.LATEST, infos.getCommitLuceneVersion());- assertEquals(indexCreatedVersion, infos.getIndexCreatedVersionMajor());- return infos.size();- }-- private int getNumberOfSegments(Directory dir) throws IOException {- final SegmentInfos infos = SegmentInfos.readLatestCommit(dir);- return infos.size();- }-- public void testUpgradeOldIndex() throws Exception {- List<String> names = new ArrayList<>(oldNames.length + oldSingleSegmentNames.length);- names.addAll(Arrays.asList(oldNames));- names.addAll(Arrays.asList(oldSingleSegmentNames));- for (String name : names) {- if (VERBOSE) {- System.out.println("testUpgradeOldIndex: index=" + name);- }- Directory dir = newDirectory(oldIndexDirs.get(name));- int indexCreatedVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor();-- newIndexUpgrader(dir).upgrade();-- checkAllSegmentsUpgraded(dir, indexCreatedVersion);-- dir.close();- }- }-- public void testIndexUpgraderCommandLineArgs() throws Exception {-- PrintStream savedSystemOut = System.out;- System.setOut(new PrintStream(new ByteArrayOutputStream(), false, UTF_8));- try {- for (Map.Entry<String, Directory> entry : oldIndexDirs.entrySet()) {- String name = entry.getKey();- Directory origDir = entry.getValue();- int indexCreatedVersion =- SegmentInfos.readLatestCommit(origDir).getIndexCreatedVersionMajor();- Path dir = createTempDir(name);- try (FSDirectory fsDir = FSDirectory.open(dir)) {- // beware that ExtraFS might add extraXXX files- Set<String> extraFiles = Set.of(fsDir.listAll());- for (String file : origDir.listAll()) {- if (extraFiles.contains(file) == false) {- fsDir.copyFrom(origDir, file, file, IOContext.DEFAULT);- }- }- }-- String path = dir.toAbsolutePath().toString();-- List<String> args = new ArrayList<>();- if (random().nextBoolean()) {- args.add("-verbose");- }- if (random().nextBoolean()) {- args.add("-delete-prior-commits");- }- if (random().nextBoolean()) {- // TODO: need to better randomize this, but ...- // - LuceneTestCase.FS_DIRECTORIES is private- // - newFSDirectory returns BaseDirectoryWrapper- // - BaseDirectoryWrapper doesn't expose delegate- Class<? extends FSDirectory> dirImpl = NIOFSDirectory.class;-- args.add("-dir-impl");- args.add(dirImpl.getName());- }- args.add(path);-- IndexUpgrader.main(args.toArray(new String[0]));-- Directory upgradedDir = newFSDirectory(dir);- try (upgradedDir) {- checkAllSegmentsUpgraded(upgradedDir, indexCreatedVersion);- }- }- } finally {- System.setOut(savedSystemOut);- }- }-- public void testUpgradeOldSingleSegmentIndexWithAdditions() throws Exception {- for (String name : oldSingleSegmentNames) {- if (VERBOSE) {- System.out.println("testUpgradeOldSingleSegmentIndexWithAdditions: index=" + name);- }- Directory dir = newDirectory(oldIndexDirs.get(name));- assertEquals("Original index must be single segment", 1, getNumberOfSegments(dir));- int indexCreatedVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor();-- // create a bunch of dummy segments- int id = 40;- Directory ramDir = new ByteBuffersDirectory();- for (int i = 0; i < 3; i++) {- // only use Log- or TieredMergePolicy, to make document addition predictable and not- // suddenly merge:- MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();- IndexWriterConfig iwc =- new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(mp);- IndexWriter w = new IndexWriter(ramDir, iwc);- // add few more docs:- for (int j = 0; j < RANDOM_MULTIPLIER * random().nextInt(30); j++) {- addDoc(w, id++);- }- try {- w.commit();- } finally {- w.close();- }- }-- // add dummy segments (which are all in current- // version) to single segment index- MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();- IndexWriterConfig iwc = new IndexWriterConfig(null).setMergePolicy(mp);- IndexWriter w = new IndexWriter(dir, iwc);- w.addIndexes(ramDir);- try (w) {- w.commit();- }-- // determine count of segments in modified index- final int origSegCount = getNumberOfSegments(dir);-- // ensure there is only one commit- assertEquals(1, DirectoryReader.listCommits(dir).size());- newIndexUpgrader(dir).upgrade();-- final int segCount = checkAllSegmentsUpgraded(dir, indexCreatedVersion);- assertEquals(- "Index must still contain the same number of segments, as only one segment was upgraded and nothing else merged",- origSegCount,- segCount);-- dir.close();- }- }-- public static final String emptyIndex = "empty.9.0.0.zip";-- public void testUpgradeEmptyOldIndex() throws Exception {- Path oldIndexDir = createTempDir("emptyIndex");- TestUtil.unzip(getDataInputStream(emptyIndex), oldIndexDir);- Directory dir = newFSDirectory(oldIndexDir);-- newIndexUpgrader(dir).upgrade();-- checkAllSegmentsUpgraded(dir, 9);-- dir.close();- }-- public static final String moreTermsIndex = "moreterms.9.0.0.zip";-- public void testMoreTerms() throws Exception {- Path oldIndexDir = createTempDir("moreterms");- TestUtil.unzip(getDataInputStream(moreTermsIndex), oldIndexDir);- Directory dir = newFSDirectory(oldIndexDir);- DirectoryReader reader = DirectoryReader.open(dir);-- verifyUsesDefaultCodec(dir, moreTermsIndex);- TestUtil.checkIndex(dir);- searchExampleIndex(reader);-- reader.close();- dir.close();- }-- public static final String dvUpdatesIndex = "dvupdates.9.0.0.zip";-- private void assertNumericDocValues(LeafReader r, String f, String cf) throws IOException {- NumericDocValues ndvf = r.getNumericDocValues(f);- NumericDocValues ndvcf = r.getNumericDocValues(cf);- for (int i = 0; i < r.maxDoc(); i++) {- assertEquals(i, ndvcf.nextDoc());- assertEquals(i, ndvf.nextDoc());- assertEquals(ndvcf.longValue(), ndvf.longValue() * 2);- }- }-- private void assertBinaryDocValues(LeafReader r, String f, String cf) throws IOException {- BinaryDocValues bdvf = r.getBinaryDocValues(f);- BinaryDocValues bdvcf = r.getBinaryDocValues(cf);- for (int i = 0; i < r.maxDoc(); i++) {- assertEquals(i, bdvf.nextDoc());- assertEquals(i, bdvcf.nextDoc());- assertEquals(getValue(bdvcf), getValue(bdvf) * 2);- }- }-- private void verifyDocValues(Directory dir) throws IOException {- DirectoryReader reader = DirectoryReader.open(dir);- for (LeafReaderContext context : reader.leaves()) {- LeafReader r = context.reader();- assertNumericDocValues(r, "ndv1", "ndv1_c");- assertNumericDocValues(r, "ndv2", "ndv2_c");- assertBinaryDocValues(r, "bdv1", "bdv1_c");- assertBinaryDocValues(r, "bdv2", "bdv2_c");- }- reader.close();- }-- public void testDocValuesUpdates() throws Exception {- Path oldIndexDir = createTempDir("dvupdates");- TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);- try (Directory dir = newFSDirectory(oldIndexDir)) {- searchDocValuesUpdatesIndex(dir);- }- }-- private void searchDocValuesUpdatesIndex(Directory dir) throws IOException {- verifyUsesDefaultCodec(dir, dvUpdatesIndex);- verifyDocValues(dir);-- // update fields and verify index- IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));- IndexWriter writer = new IndexWriter(dir, conf);- updateNumeric(writer, "1", "ndv1", "ndv1_c", 300L);- updateNumeric(writer, "1", "ndv2", "ndv2_c", 300L);- updateBinary(writer, "1", "bdv1", "bdv1_c", 300L);- updateBinary(writer, "1", "bdv2", "bdv2_c", 300L);-- writer.commit();- verifyDocValues(dir);-- // merge all segments- writer.forceMerge(1);- writer.commit();- verifyDocValues(dir);-- writer.close();- }-- public void testDeletes() throws Exception {- Path oldIndexDir = createTempDir("dvupdates");- TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);- Directory dir = newFSDirectory(oldIndexDir);- verifyUsesDefaultCodec(dir, dvUpdatesIndex);-- IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));- IndexWriter writer = new IndexWriter(dir, conf);-- int maxDoc = writer.getDocStats().maxDoc;- writer.deleteDocuments(new Term("id", "1"));- if (random().nextBoolean()) {- writer.commit();- }-- writer.forceMerge(1);- writer.commit();- assertEquals(maxDoc - 1, writer.getDocStats().maxDoc);-- writer.close();- dir.close();- }-- public void testSoftDeletes() throws Exception {- Path oldIndexDir = createTempDir("dvupdates");- TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);- Directory dir = newFSDirectory(oldIndexDir);- verifyUsesDefaultCodec(dir, dvUpdatesIndex);- IndexWriterConfig conf =- new IndexWriterConfig(new MockAnalyzer(random())).setSoftDeletesField("__soft_delete");- IndexWriter writer = new IndexWriter(dir, conf);- int maxDoc = writer.getDocStats().maxDoc;- writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("__soft_delete", 1));-- if (random().nextBoolean()) {- writer.commit();- }- writer.forceMerge(1);- writer.commit();- assertEquals(maxDoc - 1, writer.getDocStats().maxDoc);- writer.close();- dir.close();- }-- public void testDocValuesUpdatesWithNewField() throws Exception {- Path oldIndexDir = createTempDir("dvupdates");- TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);- Directory dir = newFSDirectory(oldIndexDir);- verifyUsesDefaultCodec(dir, dvUpdatesIndex);-- // update fields and verify index- IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));- IndexWriter writer = new IndexWriter(dir, conf);- // introduce a new field that we later update- writer.addDocument(- Arrays.asList(- new StringField("id", "" + Integer.MAX_VALUE, Field.Store.NO),- new NumericDocValuesField("new_numeric", 1),- new BinaryDocValuesField("new_binary", toBytes(1))));- writer.updateNumericDocValue(new Term("id", "1"), "new_numeric", 1);- writer.updateBinaryDocValue(new Term("id", "1"), "new_binary", toBytes(1));-- writer.commit();- Runnable assertDV =- () -> {- boolean found = false;- try (DirectoryReader reader = DirectoryReader.open(dir)) {- for (LeafReaderContext ctx : reader.leaves()) {- LeafReader leafReader = ctx.reader();- TermsEnum id = leafReader.terms("id").iterator();- if (id.seekExact(new BytesRef("1"))) {- PostingsEnum postings = id.postings(null, PostingsEnum.NONE);- NumericDocValues numericDocValues = leafReader.getNumericDocValues("new_numeric");- BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("new_binary");- int doc;- while ((doc = postings.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {- found = true;- assertTrue(binaryDocValues.advanceExact(doc));- assertTrue(numericDocValues.advanceExact(doc));- assertEquals(1, numericDocValues.longValue());- assertEquals(toBytes(1), binaryDocValues.binaryValue());- }- }- }- } catch (IOException e) {- throw new AssertionError(e);- }- assertTrue(found);- };- assertDV.run();- // merge all segments- writer.forceMerge(1);- writer.commit();- assertDV.run();- writer.close();- dir.close();- }-- // LUCENE-5907- public void testUpgradeWithNRTReader() throws Exception {- for (String name : oldNames) {- Directory dir = newDirectory(oldIndexDirs.get(name));-- IndexWriter writer =- new IndexWriter(- dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));- writer.addDocument(new Document());- DirectoryReader r = DirectoryReader.open(writer);- writer.commit();- r.close();- writer.forceMerge(1);- writer.commit();- writer.rollback();- SegmentInfos.readLatestCommit(dir);- dir.close();- }- }-- // LUCENE-5907- public void testUpgradeThenMultipleCommits() throws Exception {- for (String name : oldNames) {- Directory dir = newDirectory(oldIndexDirs.get(name));-- IndexWriter writer =- new IndexWriter(- dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));- writer.addDocument(new Document());- writer.commit();- writer.addDocument(new Document());- writer.commit();- writer.close();- dir.close();- }- }-- public void testSortedIndex() throws Exception {- for (String name : oldSortedNames) {- Path path = createTempDir("sorted");- InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name + ".zip");- assertNotNull("Sorted index index " + name + " not found", resource);- TestUtil.unzip(resource, path);-- Directory dir = newFSDirectory(path);- DirectoryReader reader = DirectoryReader.open(dir);-- assertEquals(1, reader.leaves().size());- Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();- assertNotNull(sort);- assertEquals("<long: \"dateDV\">!", sort.toString());-- // This will confirm the docs are really sorted- TestUtil.checkIndex(dir);-- searchExampleIndex(reader);-- reader.close();- dir.close();- }- }-- public void testSortedIndexAddDocBlocks() throws Exception {- for (String name : oldSortedNames) {- Path path = createTempDir("sorted");- InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name + ".zip");- assertNotNull("Sorted index index " + name + " not found", resource);- TestUtil.unzip(resource, path);-- try (Directory dir = newFSDirectory(path)) {- final Sort sort;- try (DirectoryReader reader = DirectoryReader.open(dir)) {- assertEquals(1, reader.leaves().size());- sort = reader.leaves().get(0).reader().getMetaData().getSort();- assertNotNull(sort);- searchExampleIndex(reader);- }- // open writer- try (IndexWriter writer =- new IndexWriter(- dir,- newIndexWriterConfig(new MockAnalyzer(random()))- .setOpenMode(OpenMode.APPEND)- .setIndexSort(sort)- .setMergePolicy(newLogMergePolicy()))) {- // add 10 docs- for (int i = 0; i < 10; i++) {- Document child = new Document();- child.add(new StringField("relation", "child", Field.Store.NO));- child.add(new StringField("bid", "" + i, Field.Store.NO));- child.add(new NumericDocValuesField("dateDV", i));- Document parent = new Document();- parent.add(new StringField("relation", "parent", Field.Store.NO));- parent.add(new StringField("bid", "" + i, Field.Store.NO));- parent.add(new NumericDocValuesField("dateDV", i));- writer.addDocuments(Arrays.asList(child, child, parent));- if (random().nextBoolean()) {- writer.flush();- }- }- if (random().nextBoolean()) {- writer.forceMerge(1);- }- writer.commit();- try (IndexReader reader = DirectoryReader.open(dir)) {- IndexSearcher searcher = new IndexSearcher(reader);- for (int i = 0; i < 10; i++) {- TopDocs children =- searcher.search(- new BooleanQuery.Builder()- .add(- new TermQuery(new Term("relation", "child")),- BooleanClause.Occur.MUST)- .add(new TermQuery(new Term("bid", "" + i)), BooleanClause.Occur.MUST)- .build(),- 2);- TopDocs parents =- searcher.search(- new BooleanQuery.Builder()- .add(- new TermQuery(new Term("relation", "parent")),- BooleanClause.Occur.MUST)- .add(new TermQuery(new Term("bid", "" + i)), BooleanClause.Occur.MUST)- .build(),- 2);- assertEquals(2, children.totalHits.value);- assertEquals(1, parents.totalHits.value);- // make sure it's sorted- assertEquals(children.scoreDocs[0].doc + 1, children.scoreDocs[1].doc);- assertEquals(children.scoreDocs[1].doc + 1, parents.scoreDocs[0].doc);- }- }- }- // This will confirm the docs are really sorted- TestUtil.checkIndex(dir);- }- }- }-- private void searchExampleIndex(DirectoryReader reader) throws IOException {- IndexSearcher searcher = newSearcher(reader);-- TopDocs topDocs = searcher.search(new FieldExistsQuery("titleTokenized"), 10);- assertEquals(50, topDocs.totalHits.value);-- topDocs = searcher.search(new FieldExistsQuery("titleDV"), 10);- assertEquals(50, topDocs.totalHits.value);-- topDocs = searcher.search(new TermQuery(new Term("body", "ja")), 10);- assertTrue(topDocs.totalHits.value > 0);-- topDocs =- searcher.search(- IntPoint.newRangeQuery("docid_int", 42, 44),- 10,- new Sort(new SortField("docid_intDV", SortField.Type.INT)));- assertEquals(3, topDocs.totalHits.value);- assertEquals(3, topDocs.scoreDocs.length);- assertEquals(42, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]);- assertEquals(43, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]);- assertEquals(44, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]);-- topDocs = searcher.search(new TermQuery(new Term("body", "the")), 5);- assertTrue(topDocs.totalHits.value > 0);-- topDocs =- searcher.search(- new MatchAllDocsQuery(), 5, new Sort(new SortField("dateDV", SortField.Type.LONG)));- assertEquals(50, topDocs.totalHits.value);- assertEquals(5, topDocs.scoreDocs.length);- long firstDate = (Long) ((FieldDoc) topDocs.scoreDocs[0]).fields[0];- long lastDate = (Long) ((FieldDoc) topDocs.scoreDocs[4]).fields[0];- assertTrue(firstDate <= lastDate);- }-- static long getValue(BinaryDocValues bdv) throws IOException {- BytesRef term = bdv.binaryValue();- int idx = term.offset;- byte b = term.bytes[idx++];- long value = b & 0x7FL;- for (int shift = 7; (b & 0x80L) != 0; shift += 7) {- b = term.bytes[idx++];- value |= (b & 0x7FL) << shift;- }- return value;- }-- // encodes a long into a BytesRef as VLong so that we get varying number of bytes when we update- static BytesRef toBytes(long value) {- BytesRef bytes = new BytesRef(10); // negative longs may take 10 bytes- while ((value & ~0x7FL) != 0L) {- bytes.bytes[bytes.length++] = (byte) ((value & 0x7FL) | 0x80L);- value >>>= 7;- }- bytes.bytes[bytes.length++] = (byte) value;- return bytes;- }-- public void testFailOpenOldIndex() throws IOException {- for (String name : oldNames) {- Directory directory = oldIndexDirs.get(name);- IndexCommit commit = DirectoryReader.listCommits(directory).get(0);- IndexFormatTooOldException ex =- expectThrows(- IndexFormatTooOldException.class,- () -> StandardDirectoryReader.open(commit, Version.LATEST.major, null));- assertTrue(- ex.getMessage()- .contains(- "only supports reading from version " + Version.LATEST.major + " upwards."));- // now open with allowed min version- StandardDirectoryReader.open(commit, Version.MIN_SUPPORTED_MAJOR, null).close();- }- }-- // #12895: test on a carefully crafted 9.8.0 index (from a small contiguous subset- // of wikibigall unique terms) that shows the read-time exception of- // IntersectTermsEnum (used by WildcardQuery)- public void testWildcardQueryExceptions990() throws IOException {- Path path = createTempDir("12895");-- String name = "index.12895.9.8.0.zip";- InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name);- assertNotNull("missing zip file to reproduce #12895", resource);- TestUtil.unzip(resource, path);-- try (Directory dir = newFSDirectory(path);- DirectoryReader reader = DirectoryReader.open(dir)) {- IndexSearcher searcher = new IndexSearcher(reader);-- searcher.count(new WildcardQuery(new Term("field", "*qx*")));- }- }-- @Nightly- public void testReadNMinusTwoCommit() throws IOException {- for (String name : binarySupportedNames) {- Path oldIndexDir = createTempDir(name);- TestUtil.unzip(getDataInputStream("unsupported." + name + ".zip"), oldIndexDir);- try (BaseDirectoryWrapper dir = newFSDirectory(oldIndexDir)) {- IndexCommit commit = DirectoryReader.listCommits(dir).get(0);- StandardDirectoryReader.open(commit, MIN_BINARY_SUPPORTED_MAJOR, null).close();- }- }- }-- @Nightly- public void testReadNMinusTwoSegmentInfos() throws IOException {- for (String name : binarySupportedNames) {- Path oldIndexDir = createTempDir(name);- TestUtil.unzip(getDataInputStream("unsupported." + name + ".zip"), oldIndexDir);- try (BaseDirectoryWrapper dir = newFSDirectory(oldIndexDir)) {- expectThrows(- IndexFormatTooOldException.class,- () -> SegmentInfos.readLatestCommit(dir, Version.MIN_SUPPORTED_MAJOR));- SegmentInfos.readLatestCommit(dir, MIN_BINARY_SUPPORTED_MAJOR);- }- }- }-- public void testOpenModeAndCreatedVersion() throws IOException {- for (String name : oldNames) {- Directory dir = newDirectory(oldIndexDirs.get(name));- int majorVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor();- if (majorVersion != Version.MIN_SUPPORTED_MAJOR && majorVersion != Version.LATEST.major) {- fail(- "expected one of: ["- + Version.MIN_SUPPORTED_MAJOR- + ", "- + Version.LATEST.major- + "] but got: "- + majorVersion);- }- for (OpenMode openMode : OpenMode.values()) {- Directory tmpDir = newDirectory(dir);- IndexWriter w = new IndexWriter(tmpDir, newIndexWriterConfig().setOpenMode(openMode));- w.commit();- w.close();- switch (openMode) {- case CREATE:- assertEquals(- Version.LATEST.major,- SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());- break;- case APPEND:- case CREATE_OR_APPEND:- default:- assertEquals(- majorVersion, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());- }- tmpDir.close();- }- dir.close();- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.javanew file mode 100644index 00000000000..078741543f2--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java@@ -0,0 +1,899 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_index;++import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;+import static org.apache.lucene.util.Version.LUCENE_9_0_0;++import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;+import java.io.IOException;+import java.util.List;+import java.util.Random;+import org.apache.lucene.document.BinaryDocValuesField;+import org.apache.lucene.document.BinaryPoint;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.DoubleDocValuesField;+import org.apache.lucene.document.DoublePoint;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.FieldType;+import org.apache.lucene.document.FloatDocValuesField;+import org.apache.lucene.document.FloatPoint;+import org.apache.lucene.document.IntPoint;+import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.document.LongPoint;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.document.SortedDocValuesField;+import org.apache.lucene.document.SortedNumericDocValuesField;+import org.apache.lucene.document.SortedSetDocValuesField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.document.TextField;+import org.apache.lucene.index.BinaryDocValues;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.Fields;+import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.IndexCommit;+import org.apache.lucene.index.IndexFormatTooOldException;+import org.apache.lucene.index.IndexOptions;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.IndexableField;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.LogByteSizeMergePolicy;+import org.apache.lucene.index.MultiBits;+import org.apache.lucene.index.MultiDocValues;+import org.apache.lucene.index.MultiTerms;+import org.apache.lucene.index.NoMergePolicy;+import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.PostingsEnum;+import org.apache.lucene.index.SegmentCommitInfo;+import org.apache.lucene.index.SegmentInfos;+import org.apache.lucene.index.SegmentReader;+import org.apache.lucene.index.SortedDocValues;+import org.apache.lucene.index.SortedNumericDocValues;+import org.apache.lucene.index.SortedSetDocValues;+import org.apache.lucene.index.StandardDirectoryReader;+import org.apache.lucene.index.StoredFields;+import org.apache.lucene.index.Term;+import org.apache.lucene.index.TermVectors;+import org.apache.lucene.index.Terms;+import org.apache.lucene.index.TermsEnum;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.KnnFloatVectorQuery;+import org.apache.lucene.search.ScoreDoc;+import org.apache.lucene.search.TermQuery;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.index.RandomIndexWriter;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.Bits;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.Version;++public class TestBasicBackwardsCompatibility extends BackwardsCompatibilityTestBase {++ static final String INDEX_NAME = "index";+ static final String SUFFIX_CFS = "-cfs";+ static final String SUFFIX_NO_CFS = "-nocfs";++ private static final int DOCS_COUNT = 35;+ private static final int DELETED_ID = 7;++ private static final int KNN_VECTOR_MIN_SUPPORTED_VERSION = LUCENE_9_0_0.major;+ private static final String KNN_VECTOR_FIELD = "knn_field";+ private static final FieldType KNN_VECTOR_FIELD_TYPE =+ KnnFloatVectorField.createFieldType(3, VectorSimilarityFunction.COSINE);+ private static final float[] KNN_VECTOR = {0.2f, -0.1f, 0.1f};++ static final int MIN_BINARY_SUPPORTED_MAJOR = Version.MIN_SUPPORTED_MAJOR - 1;++ /**+ * A parameter constructor for {@link com.carrotsearch.randomizedtesting.RandomizedRunner}. See+ * {@link #testVersionsFactory()} for details on the values provided to the framework.+ */+ public TestBasicBackwardsCompatibility(Version version, String pattern) {+ super(version, pattern);+ }++ /** Provides all current version to the test-framework for each of the index suffixes. */+ @ParametersFactory(argumentFormatting = "Lucene-Version:%1$s; Pattern: %2$s")+ public static Iterable<Object[]> testVersionsFactory() throws IllegalAccessException {+ return allVersion(INDEX_NAME, SUFFIX_CFS, SUFFIX_NO_CFS);+ }++ @Override+ protected void createIndex(Directory directory) throws IOException {+ if (indexPattern.equals(createPattern(INDEX_NAME, SUFFIX_CFS))) {+ createIndex(directory, true, false);+ } else {+ createIndex(directory, false, false);+ }+ }++ static void createIndex(Directory dir, boolean doCFS, boolean fullyMerged) throws IOException {+ LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();+ mp.setNoCFSRatio(doCFS ? 1.0 : 0.0);+ mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);+ // TODO: remove randomness+ IndexWriterConfig conf =+ new IndexWriterConfig(new MockAnalyzer(random()))+ .setMaxBufferedDocs(10)+ .setCodec(TestUtil.getDefaultCodec())+ .setMergePolicy(NoMergePolicy.INSTANCE);+ IndexWriter writer = new IndexWriter(dir, conf);++ for (int i = 0; i < DOCS_COUNT; i++) {+ addDoc(writer, i);+ }+ assertEquals("wrong doc count", DOCS_COUNT, writer.getDocStats().maxDoc);+ if (fullyMerged) {+ writer.forceMerge(1);+ }+ writer.close();++ if (!fullyMerged) {+ // open fresh writer so we get no prx file in the added segment+ mp = new LogByteSizeMergePolicy();+ mp.setNoCFSRatio(doCFS ? 1.0 : 0.0);+ // TODO: remove randomness+ conf =+ new IndexWriterConfig(new MockAnalyzer(random()))+ .setMaxBufferedDocs(10)+ .setCodec(TestUtil.getDefaultCodec())+ .setMergePolicy(NoMergePolicy.INSTANCE);+ writer = new IndexWriter(dir, conf);+ addNoProxDoc(writer);+ writer.close();++ conf =+ new IndexWriterConfig(new MockAnalyzer(random()))+ .setMaxBufferedDocs(10)+ .setCodec(TestUtil.getDefaultCodec())+ .setMergePolicy(NoMergePolicy.INSTANCE);+ writer = new IndexWriter(dir, conf);+ Term searchTerm = new Term("id", String.valueOf(DELETED_ID));+ writer.deleteDocuments(searchTerm);+ writer.close();+ }+ }++ static void addDoc(IndexWriter writer, int id) throws IOException {+ Document doc = new Document();+ doc.add(new TextField("content", "aaa", Field.Store.NO));+ doc.add(new StringField("id", Integer.toString(id), Field.Store.YES));+ FieldType customType2 = new FieldType(TextField.TYPE_STORED);+ customType2.setStoreTermVectors(true);+ customType2.setStoreTermVectorPositions(true);+ customType2.setStoreTermVectorOffsets(true);+ doc.add(+ new Field(+ "autf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));+ doc.add(+ new Field(+ "utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));+ doc.add(new Field("content2", "here is more content with aaa aaa aaa", customType2));+ doc.add(new Field("fie\u2C77ld", "field with non-ascii name", customType2));++ // add docvalues fields+ doc.add(new NumericDocValuesField("dvByte", (byte) id));+ byte[] bytes =+ new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};+ BytesRef ref = new BytesRef(bytes);+ doc.add(new BinaryDocValuesField("dvBytesDerefFixed", ref));+ doc.add(new BinaryDocValuesField("dvBytesDerefVar", ref));+ doc.add(new SortedDocValuesField("dvBytesSortedFixed", ref));+ doc.add(new SortedDocValuesField("dvBytesSortedVar", ref));+ doc.add(new BinaryDocValuesField("dvBytesStraightFixed", ref));+ doc.add(new BinaryDocValuesField("dvBytesStraightVar", ref));+ doc.add(new DoubleDocValuesField("dvDouble", id));+ doc.add(new FloatDocValuesField("dvFloat", (float) id));+ doc.add(new NumericDocValuesField("dvInt", id));+ doc.add(new NumericDocValuesField("dvLong", id));+ doc.add(new NumericDocValuesField("dvPacked", id));+ doc.add(new NumericDocValuesField("dvShort", (short) id));+ doc.add(new SortedSetDocValuesField("dvSortedSet", ref));+ doc.add(new SortedNumericDocValuesField("dvSortedNumeric", id));++ doc.add(new IntPoint("intPoint1d", id));+ doc.add(new IntPoint("intPoint2d", id, 2 * id));+ doc.add(new FloatPoint("floatPoint1d", (float) id));+ doc.add(new FloatPoint("floatPoint2d", (float) id, (float) 2 * id));+ doc.add(new LongPoint("longPoint1d", id));+ doc.add(new LongPoint("longPoint2d", id, 2 * id));+ doc.add(new DoublePoint("doublePoint1d", id));+ doc.add(new DoublePoint("doublePoint2d", id, (double) 2 * id));+ doc.add(new BinaryPoint("binaryPoint1d", bytes));+ doc.add(new BinaryPoint("binaryPoint2d", bytes, bytes));++ // a field with both offsets and term vectors for a cross-check+ FieldType customType3 = new FieldType(TextField.TYPE_STORED);+ customType3.setStoreTermVectors(true);+ customType3.setStoreTermVectorPositions(true);+ customType3.setStoreTermVectorOffsets(true);+ customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);+ doc.add(new Field("content5", "here is more content with aaa aaa aaa", customType3));+ // a field that omits only positions+ FieldType customType4 = new FieldType(TextField.TYPE_STORED);+ customType4.setStoreTermVectors(true);+ customType4.setStoreTermVectorPositions(false);+ customType4.setStoreTermVectorOffsets(true);+ customType4.setIndexOptions(IndexOptions.DOCS_AND_FREQS);+ doc.add(new Field("content6", "here is more content with aaa aaa aaa", customType4));++ float[] vector = {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * id};+ doc.add(new KnnFloatVectorField(KNN_VECTOR_FIELD, vector, KNN_VECTOR_FIELD_TYPE));++ // TODO:+ // index different norms types via similarity (we use a random one currently?!)+ // remove any analyzer randomness, explicitly add payloads for certain fields.+ writer.addDocument(doc);+ }++ static void addNoProxDoc(IndexWriter writer) throws IOException {+ Document doc = new Document();+ FieldType customType = new FieldType(TextField.TYPE_STORED);+ customType.setIndexOptions(IndexOptions.DOCS);+ Field f = new Field("content3", "aaa", customType);+ doc.add(f);+ FieldType customType2 = new FieldType();+ customType2.setStored(true);+ customType2.setIndexOptions(IndexOptions.DOCS);+ f = new Field("content4", "aaa", customType2);+ doc.add(f);+ writer.addDocument(doc);+ }++ public static void searchIndex(+ Directory dir, String oldName, int minIndexMajorVersion, Version nameVersion)+ throws IOException {+ // QueryParser parser = new QueryParser("contents", new MockAnalyzer(random));+ // Query query = parser.parse("handle:1");+ IndexCommit indexCommit = DirectoryReader.listCommits(dir).get(0);+ IndexReader reader = DirectoryReader.open(indexCommit, minIndexMajorVersion, null);+ IndexSearcher searcher = newSearcher(reader);++ TestUtil.checkIndex(dir);++ final Bits liveDocs = MultiBits.getLiveDocs(reader);+ assertNotNull(liveDocs);++ StoredFields storedFields = reader.storedFields();+ TermVectors termVectors = reader.termVectors();++ for (int i = 0; i < DOCS_COUNT; i++) {+ if (liveDocs.get(i)) {+ Document d = storedFields.document(i);+ List<IndexableField> fields = d.getFields();+ boolean isProxDoc = d.getField("content3") == null;+ if (isProxDoc) {+ assertEquals(7, fields.size());+ IndexableField f = d.getField("id");+ assertEquals("" + i, f.stringValue());++ f = d.getField("utf8");+ assertEquals(+ "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());++ f = d.getField("autf8");+ assertEquals(+ "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());++ f = d.getField("content2");+ assertEquals("here is more content with aaa aaa aaa", f.stringValue());++ f = d.getField("fie\u2C77ld");+ assertEquals("field with non-ascii name", f.stringValue());+ }++ Fields tfvFields = termVectors.get(i);+ assertNotNull("i=" + i, tfvFields);+ Terms tfv = tfvFields.terms("utf8");+ assertNotNull("docID=" + i + " index=" + oldName, tfv);+ } else {+ assertEquals(DELETED_ID, i);+ }+ }++ // check docvalues fields+ NumericDocValues dvByte = MultiDocValues.getNumericValues(reader, "dvByte");+ BinaryDocValues dvBytesDerefFixed = MultiDocValues.getBinaryValues(reader, "dvBytesDerefFixed");+ BinaryDocValues dvBytesDerefVar = MultiDocValues.getBinaryValues(reader, "dvBytesDerefVar");+ SortedDocValues dvBytesSortedFixed =+ MultiDocValues.getSortedValues(reader, "dvBytesSortedFixed");+ SortedDocValues dvBytesSortedVar = MultiDocValues.getSortedValues(reader, "dvBytesSortedVar");+ BinaryDocValues dvBytesStraightFixed =+ MultiDocValues.getBinaryValues(reader, "dvBytesStraightFixed");+ BinaryDocValues dvBytesStraightVar =+ MultiDocValues.getBinaryValues(reader, "dvBytesStraightVar");+ NumericDocValues dvDouble = MultiDocValues.getNumericValues(reader, "dvDouble");+ NumericDocValues dvFloat = MultiDocValues.getNumericValues(reader, "dvFloat");+ NumericDocValues dvInt = MultiDocValues.getNumericValues(reader, "dvInt");+ NumericDocValues dvLong = MultiDocValues.getNumericValues(reader, "dvLong");+ NumericDocValues dvPacked = MultiDocValues.getNumericValues(reader, "dvPacked");+ NumericDocValues dvShort = MultiDocValues.getNumericValues(reader, "dvShort");++ SortedSetDocValues dvSortedSet = MultiDocValues.getSortedSetValues(reader, "dvSortedSet");+ SortedNumericDocValues dvSortedNumeric =+ MultiDocValues.getSortedNumericValues(reader, "dvSortedNumeric");++ for (int i = 0; i < DOCS_COUNT; i++) {+ int id = Integer.parseInt(storedFields.document(i).get("id"));+ assertEquals(i, dvByte.nextDoc());+ assertEquals(id, dvByte.longValue());++ byte[] bytes =+ new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};+ BytesRef expectedRef = new BytesRef(bytes);++ assertEquals(i, dvBytesDerefFixed.nextDoc());+ BytesRef term = dvBytesDerefFixed.binaryValue();+ assertEquals(expectedRef, term);+ assertEquals(i, dvBytesDerefVar.nextDoc());+ term = dvBytesDerefVar.binaryValue();+ assertEquals(expectedRef, term);+ assertEquals(i, dvBytesSortedFixed.nextDoc());+ term = dvBytesSortedFixed.lookupOrd(dvBytesSortedFixed.ordValue());+ assertEquals(expectedRef, term);+ assertEquals(i, dvBytesSortedVar.nextDoc());+ term = dvBytesSortedVar.lookupOrd(dvBytesSortedVar.ordValue());+ assertEquals(expectedRef, term);+ assertEquals(i, dvBytesStraightFixed.nextDoc());+ term = dvBytesStraightFixed.binaryValue();+ assertEquals(expectedRef, term);+ assertEquals(i, dvBytesStraightVar.nextDoc());+ term = dvBytesStraightVar.binaryValue();+ assertEquals(expectedRef, term);++ assertEquals(i, dvDouble.nextDoc());+ assertEquals(id, Double.longBitsToDouble(dvDouble.longValue()), 0D);+ assertEquals(i, dvFloat.nextDoc());+ assertEquals((float) id, Float.intBitsToFloat((int) dvFloat.longValue()), 0F);+ assertEquals(i, dvInt.nextDoc());+ assertEquals(id, dvInt.longValue());+ assertEquals(i, dvLong.nextDoc());+ assertEquals(id, dvLong.longValue());+ assertEquals(i, dvPacked.nextDoc());+ assertEquals(id, dvPacked.longValue());+ assertEquals(i, dvShort.nextDoc());+ assertEquals(id, dvShort.longValue());++ assertEquals(i, dvSortedSet.nextDoc());+ assertEquals(1, dvSortedSet.docValueCount());+ long ord = dvSortedSet.nextOrd();+ term = dvSortedSet.lookupOrd(ord);+ assertEquals(expectedRef, term);++ assertEquals(i, dvSortedNumeric.nextDoc());+ assertEquals(1, dvSortedNumeric.docValueCount());+ assertEquals(id, dvSortedNumeric.nextValue());+ }++ ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;++ // First document should be #0+ Document d = storedFields.document(hits[0].doc);+ assertEquals("didn't get the right document first", "0", d.get("id"));++ doTestHits(hits, 34, searcher.getIndexReader());++ hits = searcher.search(new TermQuery(new Term("content5", "aaa")), 1000).scoreDocs;++ doTestHits(hits, 34, searcher.getIndexReader());++ hits = searcher.search(new TermQuery(new Term("content6", "aaa")), 1000).scoreDocs;++ doTestHits(hits, 34, searcher.getIndexReader());++ hits = searcher.search(new TermQuery(new Term("utf8", "\u0000")), 1000).scoreDocs;+ assertEquals(34, hits.length);+ hits =+ searcher.search(new TermQuery(new Term("utf8", "lu\uD834\uDD1Ece\uD834\uDD60ne")), 1000)+ .scoreDocs;+ assertEquals(34, hits.length);+ hits = searcher.search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), 1000).scoreDocs;+ assertEquals(34, hits.length);++ doTestHits(+ searcher.search(IntPoint.newRangeQuery("intPoint1d", 0, 34), 1000).scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(+ IntPoint.newRangeQuery("intPoint2d", new int[] {0, 0}, new int[] {34, 68}), 1000)+ .scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(FloatPoint.newRangeQuery("floatPoint1d", 0f, 34f), 1000).scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(+ FloatPoint.newRangeQuery(+ "floatPoint2d", new float[] {0f, 0f}, new float[] {34f, 68f}),+ 1000)+ .scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(LongPoint.newRangeQuery("longPoint1d", 0, 34), 1000).scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(+ LongPoint.newRangeQuery("longPoint2d", new long[] {0, 0}, new long[] {34, 68}),+ 1000)+ .scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(DoublePoint.newRangeQuery("doublePoint1d", 0.0, 34.0), 1000).scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(+ DoublePoint.newRangeQuery(+ "doublePoint2d", new double[] {0.0, 0.0}, new double[] {34.0, 68.0}),+ 1000)+ .scoreDocs,+ 34,+ searcher.getIndexReader());++ byte[] bytes1 = new byte[4];+ byte[] bytes2 = new byte[] {0, 0, 0, (byte) 34};+ doTestHits(+ searcher.search(BinaryPoint.newRangeQuery("binaryPoint1d", bytes1, bytes2), 1000).scoreDocs,+ 34,+ searcher.getIndexReader());+ byte[] bytes3 = new byte[] {0, 0, 0, (byte) 68};+ doTestHits(+ searcher.search(+ BinaryPoint.newRangeQuery(+ "binaryPoint2d", new byte[][] {bytes1, bytes1}, new byte[][] {bytes2, bytes3}),+ 1000)+ .scoreDocs,+ 34,+ searcher.getIndexReader());++ // test vector values and KNN search+ if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {+ // test vector values+ int cnt = 0;+ for (LeafReaderContext ctx : reader.leaves()) {+ FloatVectorValues values = ctx.reader().getFloatVectorValues(KNN_VECTOR_FIELD);+ if (values != null) {+ assertEquals(KNN_VECTOR_FIELD_TYPE.vectorDimension(), values.dimension());+ for (int doc = values.nextDoc(); doc != NO_MORE_DOCS; doc = values.nextDoc()) {+ float[] expectedVector = {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * cnt};+ assertArrayEquals(+ "vectors do not match for doc=" + cnt, expectedVector, values.vectorValue(), 0);+ cnt++;+ }+ }+ }+ assertEquals(DOCS_COUNT, cnt);++ // test KNN search+ ScoreDoc[] scoreDocs = assertKNNSearch(searcher, KNN_VECTOR, 10, 10, "0");+ for (int i = 0; i < scoreDocs.length; i++) {+ int id = Integer.parseInt(storedFields.document(scoreDocs[i].doc).get("id"));+ int expectedId = i < DELETED_ID ? i : i + 1;+ assertEquals(expectedId, id);+ }+ }++ reader.close();+ }++ private static void doTestHits(ScoreDoc[] hits, int expectedCount, IndexReader reader)+ throws IOException {+ final int hitCount = hits.length;+ assertEquals("wrong number of hits", expectedCount, hitCount);+ StoredFields storedFields = reader.storedFields();+ TermVectors termVectors = reader.termVectors();+ for (ScoreDoc hit : hits) {+ storedFields.document(hit.doc);+ termVectors.get(hit.doc);+ }+ }++ private static ScoreDoc[] assertKNNSearch(+ IndexSearcher searcher,+ float[] queryVector,+ int k,+ int expectedHitsCount,+ String expectedFirstDocId)+ throws IOException {+ ScoreDoc[] hits =+ searcher.search(new KnnFloatVectorQuery(KNN_VECTOR_FIELD, queryVector, k), k).scoreDocs;+ assertEquals("wrong number of hits", expectedHitsCount, hits.length);+ Document d = searcher.storedFields().document(hits[0].doc);+ assertEquals("wrong first document", expectedFirstDocId, d.get("id"));+ return hits;+ }++ public void changeIndexWithAdds(Random random, Directory dir, Version nameVersion)+ throws IOException {+ SegmentInfos infos = SegmentInfos.readLatestCommit(dir);+ assertEquals(nameVersion, infos.getCommitLuceneVersion());+ assertEquals(nameVersion, infos.getMinSegmentLuceneVersion());++ // open writer+ IndexWriter writer =+ new IndexWriter(+ dir,+ newIndexWriterConfig(new MockAnalyzer(random))+ .setOpenMode(IndexWriterConfig.OpenMode.APPEND)+ .setMergePolicy(newLogMergePolicy()));+ // add 10 docs+ for (int i = 0; i < 10; i++) {+ addDoc(writer, DOCS_COUNT + i);+ }++ // make sure writer sees right total -- writer seems not to know about deletes in .del?+ final int expected = 45;+ assertEquals("wrong doc count", expected, writer.getDocStats().numDocs);+ writer.close();++ // make sure searching sees right # hits for term search+ IndexReader reader = DirectoryReader.open(dir);+ IndexSearcher searcher = newSearcher(reader);+ ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;+ Document d = searcher.getIndexReader().storedFields().document(hits[0].doc);+ assertEquals("wrong first document", "0", d.get("id"));+ doTestHits(hits, 44, searcher.getIndexReader());++ if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {+ // make sure KNN search sees all hits (graph may not be used if k is big)+ assertKNNSearch(searcher, KNN_VECTOR, 1000, 44, "0");+ // make sure KNN search using HNSW graph sees newly added docs+ assertKNNSearch(+ searcher,+ new float[] {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * 44},+ 10,+ 10,+ "44");+ }+ reader.close();++ // fully merge+ writer =+ new IndexWriter(+ dir,+ newIndexWriterConfig(new MockAnalyzer(random))+ .setOpenMode(IndexWriterConfig.OpenMode.APPEND)+ .setMergePolicy(newLogMergePolicy()));+ writer.forceMerge(1);+ writer.close();++ reader = DirectoryReader.open(dir);+ searcher = newSearcher(reader);+ // make sure searching sees right # hits fot term search+ hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;+ assertEquals("wrong number of hits", 44, hits.length);+ d = searcher.storedFields().document(hits[0].doc);+ doTestHits(hits, 44, searcher.getIndexReader());+ assertEquals("wrong first document", "0", d.get("id"));++ if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {+ // make sure KNN search sees all hits+ assertKNNSearch(searcher, KNN_VECTOR, 1000, 44, "0");+ // make sure KNN search using HNSW graph sees newly added docs+ assertKNNSearch(+ searcher,+ new float[] {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * 44},+ 10,+ 10,+ "44");+ }+ reader.close();+ }++ public void changeIndexNoAdds(Random random, Directory dir, Version nameVersion)+ throws IOException {+ // make sure searching sees right # hits for term search+ DirectoryReader reader = DirectoryReader.open(dir);+ IndexSearcher searcher = newSearcher(reader);+ ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;+ assertEquals("wrong number of hits", 34, hits.length);+ Document d = searcher.storedFields().document(hits[0].doc);+ assertEquals("wrong first document", "0", d.get("id"));++ if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {+ // make sure KNN search sees all hits+ assertKNNSearch(searcher, KNN_VECTOR, 1000, 34, "0");+ // make sure KNN search using HNSW graph retrieves correct results+ assertKNNSearch(searcher, KNN_VECTOR, 10, 10, "0");+ }+ reader.close();++ // fully merge+ IndexWriter writer =+ new IndexWriter(+ dir,+ newIndexWriterConfig(new MockAnalyzer(random))+ .setOpenMode(IndexWriterConfig.OpenMode.APPEND));+ writer.forceMerge(1);+ writer.close();++ reader = DirectoryReader.open(dir);+ searcher = newSearcher(reader);+ // make sure searching sees right # hits fot term search+ hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;+ assertEquals("wrong number of hits", 34, hits.length);+ doTestHits(hits, 34, searcher.getIndexReader());+ // make sure searching sees right # hits for KNN search+ if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {+ // make sure KNN search sees all hits+ assertKNNSearch(searcher, KNN_VECTOR, 1000, 34, "0");+ // make sure KNN search using HNSW graph retrieves correct results+ assertKNNSearch(searcher, KNN_VECTOR, 10, 10, "0");+ }+ reader.close();+ }++ // flex: test basics of TermsEnum api on non-flex index+ public void testNextIntoWrongField() throws Exception {+ IndexReader r = DirectoryReader.open(directory);+ TermsEnum terms = MultiTerms.getTerms(r, "content").iterator();+ BytesRef t = terms.next();+ assertNotNull(t);++ // content field only has term aaa:+ assertEquals("aaa", t.utf8ToString());+ assertNull(terms.next());++ BytesRef aaaTerm = new BytesRef("aaa");++ // should be found exactly+ assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(aaaTerm));+ assertEquals(DOCS_COUNT, countDocs(TestUtil.docs(random(), terms, null, PostingsEnum.NONE)));+ assertNull(terms.next());++ // should hit end of field+ assertEquals(TermsEnum.SeekStatus.END, terms.seekCeil(new BytesRef("bbb")));+ assertNull(terms.next());++ // should seek to aaa+ assertEquals(TermsEnum.SeekStatus.NOT_FOUND, terms.seekCeil(new BytesRef("a")));+ assertTrue(terms.term().bytesEquals(aaaTerm));+ assertEquals(DOCS_COUNT, countDocs(TestUtil.docs(random(), terms, null, PostingsEnum.NONE)));+ assertNull(terms.next());++ assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(aaaTerm));+ assertEquals(DOCS_COUNT, countDocs(TestUtil.docs(random(), terms, null, PostingsEnum.NONE)));+ assertNull(terms.next());++ r.close();+ }++ /**+ * Test that we didn't forget to bump the current Constants.LUCENE_MAIN_VERSION. This is important+ * so that we can determine which version of lucene wrote the segment.+ */+ public void testOldVersions() throws Exception {+ // first create a little index with the current code and get the version+ Directory currentDir = newDirectory();+ RandomIndexWriter riw = new RandomIndexWriter(random(), currentDir);+ riw.addDocument(new Document());+ riw.close();+ DirectoryReader ir = DirectoryReader.open(currentDir);+ SegmentReader air = (SegmentReader) ir.leaves().get(0).reader();+ Version currentVersion = air.getSegmentInfo().info.getVersion();+ assertNotNull(currentVersion); // only 3.0 segments can have a null version+ ir.close();+ currentDir.close();++ // now check all the old indexes, their version should be < the current version+ DirectoryReader r = DirectoryReader.open(directory);+ for (LeafReaderContext context : r.leaves()) {+ air = (SegmentReader) context.reader();+ Version oldVersion = air.getSegmentInfo().info.getVersion();+ assertNotNull(oldVersion); // only 3.0 segments can have a null version+ assertTrue(+ "current Version.LATEST is <= an old index: did you forget to bump it?!",+ currentVersion.onOrAfter(oldVersion));+ }+ r.close();+ }++ public void testIndexCreatedVersion() throws IOException {+ SegmentInfos infos = SegmentInfos.readLatestCommit(directory);+ // those indexes are created by a single version so we can+ // compare the commit version with the created version+ assertEquals(infos.getCommitLuceneVersion().major, infos.getIndexCreatedVersionMajor());+ }++ public void testSegmentCommitInfoId() throws IOException {+ Directory dir = this.directory;+ SegmentInfos infos = SegmentInfos.readLatestCommit(dir);+ for (SegmentCommitInfo info : infos) {+ if (info.info.getVersion().onOrAfter(Version.fromBits(8, 6, 0))) {+ assertNotNull(info.toString(), info.getId());+ } else {+ assertNull(info.toString(), info.getId());+ }+ }+ }++ private int countDocs(PostingsEnum docs) throws IOException {+ int count = 0;+ while ((docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {+ count++;+ }+ return count;+ }++ public void testIndexOldIndexNoAdds() throws Exception {+ try (Directory dir = newDirectory(directory)) {+ changeIndexNoAdds(random(), dir, version);+ }+ }++ public void testIndexOldIndex() throws Exception {+ try (Directory dir = newDirectory(directory)) {+ changeIndexWithAdds(random(), dir, version);+ }+ }++ public void testSearchOldIndex() throws Exception {+ searchIndex(directory, indexPattern, Version.MIN_SUPPORTED_MAJOR, version);+ }++ public void testFullyMergeOldIndex() throws Exception {+ try (Directory dir = newDirectory(this.directory)) {+ final SegmentInfos oldSegInfos = SegmentInfos.readLatestCommit(dir);++ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));+ w.forceMerge(1);+ w.close();++ final SegmentInfos segInfos = SegmentInfos.readLatestCommit(dir);+ assertEquals(+ oldSegInfos.getIndexCreatedVersionMajor(), segInfos.getIndexCreatedVersionMajor());+ assertEquals(Version.LATEST, segInfos.asList().get(0).info.getVersion());+ assertEquals(+ oldSegInfos.asList().get(0).info.getMinVersion(),+ segInfos.asList().get(0).info.getMinVersion());+ }+ }++ public void testAddOldIndexes() throws IOException {+ SegmentInfos infos = SegmentInfos.readLatestCommit(directory);++ Directory targetDir = newDirectory();+ if (infos.getCommitLuceneVersion().major != Version.LATEST.major) {+ // both indexes are not compatible+ Directory targetDir2 = newDirectory();+ IndexWriter w = new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));+ IllegalArgumentException e =+ expectThrows(IllegalArgumentException.class, () -> w.addIndexes(directory));+ assertTrue(+ e.getMessage(),+ e.getMessage()+ .startsWith(+ "Cannot use addIndexes(Directory) with indexes that have been created by a different Lucene version."));+ w.close();+ targetDir2.close();++ // for the next test, we simulate writing to an index that was created on the same major+ // version+ new SegmentInfos(infos.getIndexCreatedVersionMajor()).commit(targetDir);+ }++ IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));+ w.addIndexes(directory);+ w.close();++ SegmentInfos si = SegmentInfos.readLatestCommit(targetDir);+ assertNull(+ "none of the segments should have been upgraded",+ si.asList().stream()+ .filter( // depending on the MergePolicy we might see these segments merged away+ sci ->+ sci.getId() != null+ && sci.info.getVersion().onOrAfter(Version.fromBits(8, 6, 0)) == false)+ .findAny()+ .orElse(null));+ if (VERBOSE) {+ System.out.println("\nTEST: done adding indices; now close");+ }++ targetDir.close();+ }++ public void testAddOldIndexesReader() throws IOException {+ SegmentInfos infos = SegmentInfos.readLatestCommit(directory);+ DirectoryReader reader = DirectoryReader.open(directory);++ Directory targetDir = newDirectory();+ if (infos.getCommitLuceneVersion().major != Version.LATEST.major) {+ Directory targetDir2 = newDirectory();+ IndexWriter w = new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));+ IllegalArgumentException e =+ expectThrows(IllegalArgumentException.class, () -> TestUtil.addIndexesSlowly(w, reader));+ assertEquals(+ e.getMessage(),+ "Cannot merge a segment that has been created with major version 9 into this index which has been created by major version 10");+ w.close();+ targetDir2.close();++ // for the next test, we simulate writing to an index that was created on the same major+ // version+ new SegmentInfos(infos.getIndexCreatedVersionMajor()).commit(targetDir);+ }+ IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));+ TestUtil.addIndexesSlowly(w, reader);+ w.close();+ reader.close();+ SegmentInfos si = SegmentInfos.readLatestCommit(targetDir);+ assertNull(+ "all SCIs should have an id now",+ si.asList().stream().filter(sci -> sci.getId() == null).findAny().orElse(null));+ targetDir.close();+ }++ public void testFailOpenOldIndex() throws IOException {+ assumeFalse("doesn't work on current index", version.equals(Version.LATEST));+ IndexCommit commit = DirectoryReader.listCommits(directory).get(0);+ IndexFormatTooOldException ex =+ expectThrows(+ IndexFormatTooOldException.class,+ () -> StandardDirectoryReader.open(commit, Version.LATEST.major, null));+ assertTrue(+ ex.getMessage()+ .contains("only supports reading from version " + Version.LATEST.major + " upwards."));+ // now open with allowed min version+ StandardDirectoryReader.open(commit, Version.MIN_SUPPORTED_MAJOR, null).close();+ }++ public void testOpenModeAndCreatedVersion() throws IOException {+ Directory dir = newDirectory(directory);+ int majorVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor();+ if (majorVersion != Version.MIN_SUPPORTED_MAJOR && majorVersion != Version.LATEST.major) {+ fail(+ "expected one of: ["+ + Version.MIN_SUPPORTED_MAJOR+ + ", "+ + Version.LATEST.major+ + "] but got: "+ + majorVersion);+ }+ for (IndexWriterConfig.OpenMode openMode : IndexWriterConfig.OpenMode.values()) {+ Directory tmpDir = newDirectory(dir);+ IndexWriter w = new IndexWriter(tmpDir, newIndexWriterConfig().setOpenMode(openMode));+ w.commit();+ w.close();+ switch (openMode) {+ case CREATE:+ assertEquals(+ Version.LATEST.major,+ SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());+ break;+ case APPEND:+ case CREATE_OR_APPEND:+ default:+ assertEquals(+ majorVersion, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());+ }+ tmpDir.close();+ }+ dir.close();+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBinaryBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBinaryBackwardsCompatibility.javanew file mode 100644index 00000000000..82594e775c7--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBinaryBackwardsCompatibility.java@@ -0,0 +1,87 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_index;++import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;+import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexCommit;+import org.apache.lucene.index.IndexFormatTooOldException;+import org.apache.lucene.index.SegmentInfos;+import org.apache.lucene.index.StandardDirectoryReader;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.store.BaseDirectoryWrapper;+import org.apache.lucene.util.Version;++public class TestBinaryBackwardsCompatibility extends BackwardsCompatibilityTestBase {++ static final int MIN_BINARY_SUPPORTED_MAJOR = Version.MIN_SUPPORTED_MAJOR - 1;+ static final String INDEX_NAME = "unsupported";+ static final String SUFFIX_CFS = "-cfs";+ static final String SUFFIX_NO_CFS = "-nocfs";++ public TestBinaryBackwardsCompatibility(Version version, String pattern) {+ super(version, pattern);+ }++ @ParametersFactory(argumentFormatting = "Lucene-Version:%1$s; Pattern: %2$s")+ public static Iterable<Object[]> testVersionsFactory() {+ List<Object[]> params = new ArrayList<>();+ for (Version version : BINARY_SUPPORTED_VERSIONS) {+ params.add(new Object[] {version, createPattern(INDEX_NAME, SUFFIX_CFS)});+ params.add(new Object[] {version, createPattern(INDEX_NAME, SUFFIX_NO_CFS)});+ }+ return params;+ }++ @Override+ void verifyUsesDefaultCodec(Directory dir, String name) throws IOException {+ // don't this will fail since the indices are not supported+ }++ @Override+ protected void createIndex(Directory directory) throws IOException {+ fail("not supported");+ }++ @Nightly+ public void testReadNMinusTwoCommit() throws IOException {++ try (BaseDirectoryWrapper dir = newDirectory(directory)) {+ IndexCommit commit = DirectoryReader.listCommits(dir).get(0);+ StandardDirectoryReader.open(commit, MIN_BINARY_SUPPORTED_MAJOR, null).close();+ }+ }++ @Nightly+ public void testReadNMinusTwoSegmentInfos() throws IOException {+ try (BaseDirectoryWrapper dir = newDirectory(directory)) {+ expectThrows(+ IndexFormatTooOldException.class,+ () -> SegmentInfos.readLatestCommit(dir, Version.MIN_SUPPORTED_MAJOR));+ SegmentInfos.readLatestCommit(dir, MIN_BINARY_SUPPORTED_MAJOR);+ }+ }++ @Nightly+ public void testSearchOldIndex() throws Exception {+ TestBasicBackwardsCompatibility.searchIndex(+ directory, indexPattern, MIN_BINARY_SUPPORTED_MAJOR, version);+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestDVUpdateBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestDVUpdateBackwardsCompatibility.javanew file mode 100644index 00000000000..332daa621ed--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestDVUpdateBackwardsCompatibility.java@@ -0,0 +1,269 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_index;++import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;+import java.io.IOException;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.List;+import org.apache.lucene.document.BinaryDocValuesField;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.index.BinaryDocValues;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.LeafReader;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.NoMergePolicy;+import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.PostingsEnum;+import org.apache.lucene.index.Term;+import org.apache.lucene.index.TermsEnum;+import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.Version;++public class TestDVUpdateBackwardsCompatibility extends BackwardsCompatibilityTestBase {++ static final String INDEX_NAME = "dvupdates";+ static final String SUFFIX = "";++ public TestDVUpdateBackwardsCompatibility(Version version, String pattern) {+ super(version, pattern);+ }++ /** Provides the initial release of the previous major to the test-framework */+ @ParametersFactory(argumentFormatting = "Lucene-Version:%1$s; Pattern: %2$s")+ public static Iterable<Object[]> testVersionsFactory() {+ List<Object[]> params = new ArrayList<>();+ // TODO - WHY ONLY on the first major version?+ params.add(new Object[] {Version.LUCENE_9_0_0, createPattern(INDEX_NAME, SUFFIX)});+ return params;+ }++ @Override+ protected void createIndex(Directory directory) throws IOException {+ IndexWriterConfig conf =+ new IndexWriterConfig(new MockAnalyzer(random()))+ .setCodec(TestUtil.getDefaultCodec())+ .setUseCompoundFile(false)+ .setMergePolicy(NoMergePolicy.INSTANCE);+ IndexWriter writer = new IndexWriter(directory, conf);+ // create an index w/ few doc-values fields, some with updates and some without+ for (int i = 0; i < 30; i++) {+ Document doc = new Document();+ doc.add(new StringField("id", "" + i, Field.Store.NO));+ doc.add(new NumericDocValuesField("ndv1", i));+ doc.add(new NumericDocValuesField("ndv1_c", i * 2));+ doc.add(new NumericDocValuesField("ndv2", i * 3));+ doc.add(new NumericDocValuesField("ndv2_c", i * 6));+ doc.add(new BinaryDocValuesField("bdv1", toBytes(i)));+ doc.add(new BinaryDocValuesField("bdv1_c", toBytes(i * 2)));+ doc.add(new BinaryDocValuesField("bdv2", toBytes(i * 3)));+ doc.add(new BinaryDocValuesField("bdv2_c", toBytes(i * 6)));+ writer.addDocument(doc);+ if ((i + 1) % 10 == 0) {+ writer.commit(); // flush every 10 docs+ }+ }++ // first segment: no updates++ // second segment: update two fields, same gen+ updateNumeric(writer, "10", "ndv1", "ndv1_c", 100L);+ updateBinary(writer, "11", "bdv1", "bdv1_c", 100L);+ writer.commit();++ // third segment: update few fields, different gens, few docs+ updateNumeric(writer, "20", "ndv1", "ndv1_c", 100L);+ updateBinary(writer, "21", "bdv1", "bdv1_c", 100L);+ writer.commit();+ updateNumeric(writer, "22", "ndv1", "ndv1_c", 200L); // update the field again+ writer.close();+ }++ public void testDocValuesUpdates() throws Exception {+ searchDocValuesUpdatesIndex(directory);+ }++ public void testDeletes() throws Exception {+ IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));+ IndexWriter writer = new IndexWriter(directory, conf);++ int maxDoc = writer.getDocStats().maxDoc;+ writer.deleteDocuments(new Term("id", "1"));+ if (random().nextBoolean()) {+ writer.commit();+ }++ writer.forceMerge(1);+ writer.commit();+ assertEquals(maxDoc - 1, writer.getDocStats().maxDoc);++ writer.close();+ }++ public void testSoftDeletes() throws Exception {+ IndexWriterConfig conf =+ new IndexWriterConfig(new MockAnalyzer(random())).setSoftDeletesField("__soft_delete");+ IndexWriter writer = new IndexWriter(directory, conf);+ int maxDoc = writer.getDocStats().maxDoc;+ writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("__soft_delete", 1));++ if (random().nextBoolean()) {+ writer.commit();+ }+ writer.forceMerge(1);+ writer.commit();+ assertEquals(maxDoc - 1, writer.getDocStats().maxDoc);+ writer.close();+ }++ public void testDocValuesUpdatesWithNewField() throws Exception {+ // update fields and verify index+ IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));+ IndexWriter writer = new IndexWriter(directory, conf);+ // introduce a new field that we later update+ writer.addDocument(+ Arrays.asList(+ new StringField("id", "" + Integer.MAX_VALUE, Field.Store.NO),+ new NumericDocValuesField("new_numeric", 1),+ new BinaryDocValuesField("new_binary", toBytes(1))));+ writer.updateNumericDocValue(new Term("id", "1"), "new_numeric", 1);+ writer.updateBinaryDocValue(new Term("id", "1"), "new_binary", toBytes(1));++ writer.commit();+ Runnable assertDV =+ () -> {+ boolean found = false;+ try (DirectoryReader reader = DirectoryReader.open(directory)) {+ for (LeafReaderContext ctx : reader.leaves()) {+ LeafReader leafReader = ctx.reader();+ TermsEnum id = leafReader.terms("id").iterator();+ if (id.seekExact(new BytesRef("1"))) {+ PostingsEnum postings = id.postings(null, PostingsEnum.NONE);+ NumericDocValues numericDocValues = leafReader.getNumericDocValues("new_numeric");+ BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("new_binary");+ int doc;+ while ((doc = postings.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {+ found = true;+ assertTrue(binaryDocValues.advanceExact(doc));+ assertTrue(numericDocValues.advanceExact(doc));+ assertEquals(1, numericDocValues.longValue());+ assertEquals(toBytes(1), binaryDocValues.binaryValue());+ }+ }+ }+ } catch (IOException e) {+ throw new AssertionError(e);+ }+ assertTrue(found);+ };+ assertDV.run();+ // merge all segments+ writer.forceMerge(1);+ writer.commit();+ assertDV.run();+ writer.close();+ }++ private void assertNumericDocValues(LeafReader r, String f, String cf) throws IOException {+ NumericDocValues ndvf = r.getNumericDocValues(f);+ NumericDocValues ndvcf = r.getNumericDocValues(cf);+ for (int i = 0; i < r.maxDoc(); i++) {+ assertEquals(i, ndvcf.nextDoc());+ assertEquals(i, ndvf.nextDoc());+ assertEquals(ndvcf.longValue(), ndvf.longValue() * 2);+ }+ }++ private void assertBinaryDocValues(LeafReader r, String f, String cf) throws IOException {+ BinaryDocValues bdvf = r.getBinaryDocValues(f);+ BinaryDocValues bdvcf = r.getBinaryDocValues(cf);+ for (int i = 0; i < r.maxDoc(); i++) {+ assertEquals(i, bdvf.nextDoc());+ assertEquals(i, bdvcf.nextDoc());+ assertEquals(getValue(bdvcf), getValue(bdvf) * 2);+ }+ }++ static long getValue(BinaryDocValues bdv) throws IOException {+ BytesRef term = bdv.binaryValue();+ int idx = term.offset;+ byte b = term.bytes[idx++];+ long value = b & 0x7FL;+ for (int shift = 7; (b & 0x80L) != 0; shift += 7) {+ b = term.bytes[idx++];+ value |= (b & 0x7FL) << shift;+ }+ return value;+ }++ private void verifyDocValues(Directory dir) throws IOException {+ DirectoryReader reader = DirectoryReader.open(dir);+ for (LeafReaderContext context : reader.leaves()) {+ LeafReader r = context.reader();+ assertNumericDocValues(r, "ndv1", "ndv1_c");+ assertNumericDocValues(r, "ndv2", "ndv2_c");+ assertBinaryDocValues(r, "bdv1", "bdv1_c");+ assertBinaryDocValues(r, "bdv2", "bdv2_c");+ }+ reader.close();+ }++ private void searchDocValuesUpdatesIndex(Directory dir) throws IOException {+ verifyUsesDefaultCodec(dir, indexName(version));+ verifyDocValues(dir);++ // update fields and verify index+ IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));+ IndexWriter writer = new IndexWriter(dir, conf);+ updateNumeric(writer, "1", "ndv1", "ndv1_c", 300L);+ updateNumeric(writer, "1", "ndv2", "ndv2_c", 300L);+ updateBinary(writer, "1", "bdv1", "bdv1_c", 300L);+ updateBinary(writer, "1", "bdv2", "bdv2_c", 300L);++ writer.commit();+ verifyDocValues(dir);++ // merge all segments+ writer.forceMerge(1);+ writer.commit();+ verifyDocValues(dir);++ writer.close();+ }++ private void updateNumeric(IndexWriter writer, String id, String f, String cf, long value)+ throws IOException {+ writer.updateNumericDocValue(new Term("id", id), f, value);+ writer.updateNumericDocValue(new Term("id", id), cf, value * 2);+ }++ private void updateBinary(IndexWriter writer, String id, String f, String cf, long value)+ throws IOException {+ writer.updateBinaryDocValue(new Term("id", id), f, toBytes(value));+ writer.updateBinaryDocValue(new Term("id", id), cf, toBytes(value * 2));+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestEmptyIndexBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestEmptyIndexBackwardsCompatibility.javanew file mode 100644index 00000000000..40fcd4c59bf--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestEmptyIndexBackwardsCompatibility.java@@ -0,0 +1,66 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_index;++import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;+import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.NoMergePolicy;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.Version;++public class TestEmptyIndexBackwardsCompatibility extends BackwardsCompatibilityTestBase {+ static final String INDEX_NAME = "empty";+ static final String SUFFIX = "";++ public TestEmptyIndexBackwardsCompatibility(Version version, String pattern) {+ super(version, pattern);+ }++ @Override+ protected void createIndex(Directory directory) throws IOException {+ IndexWriterConfig conf =+ new IndexWriterConfig(new MockAnalyzer(random()))+ .setUseCompoundFile(false)+ .setCodec(TestUtil.getDefaultCodec())+ .setMergePolicy(NoMergePolicy.INSTANCE);+ try (IndexWriter writer = new IndexWriter(directory, conf)) {+ writer.flush();+ }+ }++ /** Provides the initial release of the previous major to the test-framework */+ @ParametersFactory(argumentFormatting = "Lucene-Version:%1$s; Pattern: %2$s")+ public static Iterable<Object[]> testVersionsFactory() {+ List<Object[]> params = new ArrayList<>();+ // TODO - WHY ONLY on the first major version?+ params.add(new Object[] {Version.LUCENE_9_0_0, createPattern(INDEX_NAME, SUFFIX)});+ return params;+ }++ public void testUpgradeEmptyOldIndex() throws Exception {+ try (Directory dir = newDirectory(directory)) {+ TestIndexUpgradeBackwardsCompatibility.newIndexUpgrader(dir).upgrade();+ TestIndexUpgradeBackwardsCompatibility.checkAllSegmentsUpgraded(dir, 9);+ }+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.javanew file mode 100644index 00000000000..0cd9f37d5c3--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java@@ -0,0 +1,125 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_index;++import static org.apache.lucene.backward_index.BackwardsCompatibilityTestBase.createPattern;++import java.io.IOException;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.util.Version;++public class TestGenerateBwcIndices extends LuceneTestCase {++ // Backcompat index generation, described below, is mostly automated in:+ //+ // dev-tools/scripts/addBackcompatIndexes.py+ //+ // For usage information, see:+ //+ // http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes+ //+ // -----+ //+ // To generate backcompat indexes with the current default codec, run the following gradle+ // command:+ // gradlew test -Ptests.bwcdir=/path/to/store/indexes -Ptests.codec=default+ // -Ptests.useSecurityManager=false --tests TestGenerateBwcIndices+ //+ // Also add testmethod with one of the index creation methods below, for example:+ // -Ptestmethod=testCreateCFS+ //+ // Zip up the generated indexes:+ //+ // cd /path/to/store/indexes/index.cfs ; zip index.<VERSION>-cfs.zip *+ // cd /path/to/store/indexes/index.nocfs ; zip index.<VERSION>-nocfs.zip *+ //+ // Then move those 2 zip files to your trunk checkout and add them+ // to the oldNames array.++ public void testCreateCFS() throws IOException {+ TestBasicBackwardsCompatibility basicTest =+ new TestBasicBackwardsCompatibility(+ Version.LATEST,+ createPattern(+ TestBasicBackwardsCompatibility.INDEX_NAME,+ TestBasicBackwardsCompatibility.SUFFIX_CFS));+ basicTest.createBWCIndex();+ }++ public void testCreateNoCFS() throws IOException {+ TestBasicBackwardsCompatibility basicTest =+ new TestBasicBackwardsCompatibility(+ Version.LATEST,+ createPattern(+ TestBasicBackwardsCompatibility.INDEX_NAME,+ TestBasicBackwardsCompatibility.SUFFIX_NO_CFS));+ basicTest.createBWCIndex();+ }++ public void testCreateSortedIndex() throws IOException {+ TestIndexSortBackwardsCompatibility sortedTest =+ new TestIndexSortBackwardsCompatibility(+ Version.LATEST,+ createPattern(+ TestIndexSortBackwardsCompatibility.INDEX_NAME,+ TestIndexSortBackwardsCompatibility.SUFFIX));+ sortedTest.createBWCIndex();+ }++ private boolean isInitialMajorVersionRelease() {+ return Version.LATEST.equals(Version.fromBits(Version.LATEST.major, 0, 0));+ }++ public void testCreateMoreTermsIndex() throws IOException {+ if (isInitialMajorVersionRelease()) {+ // TODO - WHY ONLY on the first major version?+ TestMoreTermsBackwardsCompatibility moreTermsTest =+ new TestMoreTermsBackwardsCompatibility(+ Version.LATEST,+ createPattern(+ TestMoreTermsBackwardsCompatibility.INDEX_NAME,+ TestMoreTermsBackwardsCompatibility.SUFFIX));+ moreTermsTest.createBWCIndex();+ }+ }++ public void testCreateIndexWithDocValuesUpdates() throws IOException {+ if (isInitialMajorVersionRelease()) {+ // TODO - WHY ONLY on the first major version?+ TestDVUpdateBackwardsCompatibility dvUpdatesTest =+ new TestDVUpdateBackwardsCompatibility(+ Version.LATEST,+ createPattern(+ TestDVUpdateBackwardsCompatibility.INDEX_NAME,+ TestDVUpdateBackwardsCompatibility.SUFFIX));+ dvUpdatesTest.createBWCIndex();+ }+ }++ public void testCreateEmptyIndex() throws IOException {+ if (isInitialMajorVersionRelease()) {+ // TODO - WHY ONLY on the first major version?+ TestEmptyIndexBackwardsCompatibility emptyIndex =+ new TestEmptyIndexBackwardsCompatibility(+ Version.LATEST,+ createPattern(+ TestEmptyIndexBackwardsCompatibility.INDEX_NAME,+ TestEmptyIndexBackwardsCompatibility.SUFFIX));+ emptyIndex.createBWCIndex();+ }+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestIndexSortBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestIndexSortBackwardsCompatibility.javanew file mode 100644index 00000000000..b8708c3b7cb--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestIndexSortBackwardsCompatibility.java@@ -0,0 +1,234 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_index;++import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;+import java.io.IOException;+import java.text.ParsePosition;+import java.text.SimpleDateFormat;+import java.util.Arrays;+import java.util.Date;+import java.util.Locale;+import java.util.Random;+import java.util.TimeZone;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.IntPoint;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.document.SortedDocValuesField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.LogByteSizeMergePolicy;+import org.apache.lucene.index.Term;+import org.apache.lucene.search.BooleanClause;+import org.apache.lucene.search.BooleanQuery;+import org.apache.lucene.search.FieldDoc;+import org.apache.lucene.search.FieldExistsQuery;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.MatchAllDocsQuery;+import org.apache.lucene.search.Sort;+import org.apache.lucene.search.SortField;+import org.apache.lucene.search.TermQuery;+import org.apache.lucene.search.TopDocs;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.util.LineFileDocs;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.Version;++public class TestIndexSortBackwardsCompatibility extends BackwardsCompatibilityTestBase {++ static final String INDEX_NAME = "sorted";+ static final String SUFFIX = "";++ public TestIndexSortBackwardsCompatibility(Version version, String pattern) {+ super(version, pattern);+ }++ /** Provides all sorted versions to the test-framework */+ @ParametersFactory(argumentFormatting = "Lucene-Version:%1$s; Pattern: %2$s")+ public static Iterable<Object[]> testVersionsFactory() throws IllegalAccessException {+ return allVersion(INDEX_NAME, SUFFIX);+ }++ public void testSortedIndexAddDocBlocks() throws Exception {+ final Sort sort;+ try (DirectoryReader reader = DirectoryReader.open(directory)) {+ assertEquals(1, reader.leaves().size());+ sort = reader.leaves().get(0).reader().getMetaData().getSort();+ assertNotNull(sort);+ searchExampleIndex(reader);+ }+ IndexWriterConfig indexWriterConfig =+ newIndexWriterConfig(new MockAnalyzer(random()))+ .setOpenMode(IndexWriterConfig.OpenMode.APPEND)+ .setIndexSort(sort)+ .setMergePolicy(newLogMergePolicy());+ if (this.version.onOrAfter(Version.LUCENE_10_0_0)) {+ indexWriterConfig.setParentField("___parent");+ }+ // open writer+ try (IndexWriter writer = new IndexWriter(directory, indexWriterConfig)) {+ // add 10 docs+ for (int i = 0; i < 10; i++) {+ Document child = new Document();+ child.add(new StringField("relation", "child", Field.Store.NO));+ child.add(new StringField("bid", "" + i, Field.Store.NO));+ child.add(new NumericDocValuesField("dateDV", i));+ Document parent = new Document();+ parent.add(new StringField("relation", "parent", Field.Store.NO));+ parent.add(new StringField("bid", "" + i, Field.Store.NO));+ parent.add(new NumericDocValuesField("dateDV", i));+ writer.addDocuments(Arrays.asList(child, child, parent));+ if (random().nextBoolean()) {+ writer.flush();+ }+ }+ if (random().nextBoolean()) {+ writer.forceMerge(1);+ }+ writer.commit();+ try (IndexReader reader = DirectoryReader.open(directory)) {+ IndexSearcher searcher = new IndexSearcher(reader);+ for (int i = 0; i < 10; i++) {+ TopDocs children =+ searcher.search(+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("relation", "child")), BooleanClause.Occur.MUST)+ .add(new TermQuery(new Term("bid", "" + i)), BooleanClause.Occur.MUST)+ .build(),+ 2);+ TopDocs parents =+ searcher.search(+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("relation", "parent")), BooleanClause.Occur.MUST)+ .add(new TermQuery(new Term("bid", "" + i)), BooleanClause.Occur.MUST)+ .build(),+ 2);+ assertEquals(2, children.totalHits.value);+ assertEquals(1, parents.totalHits.value);+ // make sure it's sorted+ assertEquals(children.scoreDocs[0].doc + 1, children.scoreDocs[1].doc);+ assertEquals(children.scoreDocs[1].doc + 1, parents.scoreDocs[0].doc);+ }+ }+ }+ // This will confirm the docs are really sorted+ TestUtil.checkIndex(directory);+ }++ public void testSortedIndex() throws Exception {+ try (DirectoryReader reader = DirectoryReader.open(directory)) {+ assertEquals(1, reader.leaves().size());+ Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();+ assertNotNull(sort);+ assertEquals("<long: \"dateDV\">!", sort.toString());+ // This will confirm the docs are really sorted+ TestUtil.checkIndex(directory);+ searchExampleIndex(reader);+ }+ }++ @Override+ protected void createIndex(Directory directory) throws IOException {+ LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();+ mp.setNoCFSRatio(1.0);+ mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);+ MockAnalyzer analyzer = new MockAnalyzer(random());+ analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));++ // TODO: remove randomness+ IndexWriterConfig conf = new IndexWriterConfig(analyzer);+ conf.setMergePolicy(mp);+ conf.setUseCompoundFile(false);+ conf.setCodec(TestUtil.getDefaultCodec());+ conf.setParentField("___parent");+ conf.setIndexSort(new Sort(new SortField("dateDV", SortField.Type.LONG, true)));+ IndexWriter writer = new IndexWriter(directory, conf);+ LineFileDocs docs = new LineFileDocs(new Random(0));+ SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);+ parser.setTimeZone(TimeZone.getTimeZone("UTC"));+ ParsePosition position = new ParsePosition(0);+ for (int i = 0; i < 50; i++) {+ Document doc = TestUtil.cloneDocument(docs.nextDoc());+ String dateString = doc.get("date");+ position.setIndex(0);+ Date date = parser.parse(dateString, position);+ if (position.getErrorIndex() != -1) {+ throw new AssertionError("failed to parse \"" + dateString + "\" as date");+ }+ if (position.getIndex() != dateString.length()) {+ throw new AssertionError("failed to parse \"" + dateString + "\" as date");+ }+ doc.add(+ new NumericDocValuesField(+ "docid_intDV", doc.getField("docid_int").numericValue().longValue()));+ doc.add(+ new SortedDocValuesField("titleDV", new BytesRef(doc.getField("title").stringValue())));+ doc.add(new NumericDocValuesField("dateDV", date.getTime()));+ if (i % 10 == 0) { // commit every 10 documents+ writer.commit();+ }+ writer.addDocument(doc);+ }+ writer.forceMerge(1);+ writer.close();++ try (DirectoryReader reader = DirectoryReader.open(directory)) {+ searchExampleIndex(reader); // make sure we can search it+ }+ }++ public static void searchExampleIndex(DirectoryReader reader) throws IOException {+ IndexSearcher searcher = newSearcher(reader);++ TopDocs topDocs = searcher.search(new FieldExistsQuery("titleTokenized"), 10);+ assertEquals(50, topDocs.totalHits.value);++ topDocs = searcher.search(new FieldExistsQuery("titleDV"), 10);+ assertEquals(50, topDocs.totalHits.value);++ topDocs = searcher.search(new TermQuery(new Term("body", "ja")), 10);+ assertTrue(topDocs.totalHits.value > 0);++ topDocs =+ searcher.search(+ IntPoint.newRangeQuery("docid_int", 42, 44),+ 10,+ new Sort(new SortField("docid_intDV", SortField.Type.INT)));+ assertEquals(3, topDocs.totalHits.value);+ assertEquals(3, topDocs.scoreDocs.length);+ assertEquals(42, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]);+ assertEquals(43, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]);+ assertEquals(44, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]);++ topDocs = searcher.search(new TermQuery(new Term("body", "the")), 5);+ assertTrue(topDocs.totalHits.value > 0);+ topDocs =+ searcher.search(+ new MatchAllDocsQuery(), 5, new Sort(new SortField("dateDV", SortField.Type.LONG)));+ assertEquals(50, topDocs.totalHits.value);+ assertEquals(5, topDocs.scoreDocs.length);+ long firstDate = (Long) ((FieldDoc) topDocs.scoreDocs[0]).fields[0];+ long lastDate = (Long) ((FieldDoc) topDocs.scoreDocs[4]).fields[0];+ assertTrue(firstDate <= lastDate);+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestIndexUpgradeBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestIndexUpgradeBackwardsCompatibility.javanew file mode 100644index 00000000000..10099a5b5b5--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestIndexUpgradeBackwardsCompatibility.java@@ -0,0 +1,255 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_index;++import static java.nio.charset.StandardCharsets.UTF_8;++import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;+import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.PrintStream;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;+import java.util.Set;+import org.apache.lucene.document.Document;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexUpgrader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.MergePolicy;+import org.apache.lucene.index.SegmentCommitInfo;+import org.apache.lucene.index.SegmentInfos;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FSDirectory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.NIOFSDirectory;+import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.InfoStream;+import org.apache.lucene.util.Version;++public class TestIndexUpgradeBackwardsCompatibility extends BackwardsCompatibilityTestBase {++ /**+ * A parameter constructor for {@link com.carrotsearch.randomizedtesting.RandomizedRunner}. See+ * {@link #testVersionsFactory()} for details on the values provided to the framework.+ */+ public TestIndexUpgradeBackwardsCompatibility(Version version, String pattern) {+ super(version, pattern);+ }++ /** Provides all current version to the test-framework for each of the index suffixes. */+ @ParametersFactory(argumentFormatting = "Lucene-Version:%1$s; Pattern: %2$s")+ public static Iterable<Object[]> testVersionsFactory() throws IllegalAccessException {+ Iterable<Object[]> allSupportedVersions =+ allVersion(+ TestBasicBackwardsCompatibility.INDEX_NAME,+ TestBasicBackwardsCompatibility.SUFFIX_CFS,+ TestBasicBackwardsCompatibility.SUFFIX_NO_CFS);+ return allSupportedVersions;+ }++ /** Randomizes the use of some of the constructor variations */+ static IndexUpgrader newIndexUpgrader(Directory dir) {+ final boolean streamType = random().nextBoolean();+ final int choice = TestUtil.nextInt(random(), 0, 2);+ switch (choice) {+ case 0:+ return new IndexUpgrader(dir);+ case 1:+ return new IndexUpgrader(dir, streamType ? null : InfoStream.NO_OUTPUT, false);+ case 2:+ return new IndexUpgrader(dir, newIndexWriterConfig(null), false);+ default:+ fail("case statement didn't get updated when random bounds changed");+ }+ return null; // never get here+ }++ public void testUpgradeOldIndex() throws Exception {+ int indexCreatedVersion =+ SegmentInfos.readLatestCommit(directory).getIndexCreatedVersionMajor();+ newIndexUpgrader(directory).upgrade();+ checkAllSegmentsUpgraded(directory, indexCreatedVersion);+ }++ @Override+ protected void createIndex(Directory directory) throws IOException {+ if (indexPattern.equals(+ createPattern(+ TestBasicBackwardsCompatibility.INDEX_NAME,+ TestBasicBackwardsCompatibility.SUFFIX_CFS))) {+ TestBasicBackwardsCompatibility.createIndex(directory, true, false);+ } else {+ TestBasicBackwardsCompatibility.createIndex(directory, false, false);+ }+ }++ public void testUpgradeOldSingleSegmentIndexWithAdditions() throws Exception {+ // TODO we use to have single segment indices but we stopped creating them at some point+ // either delete the test or recreate the indices+ assumeTrue("Original index must be single segment", 1 == getNumberOfSegments(directory));+ int indexCreatedVersion =+ SegmentInfos.readLatestCommit(directory).getIndexCreatedVersionMajor();++ // create a bunch of dummy segments+ int id = 40;+ Directory ramDir = new ByteBuffersDirectory();+ for (int i = 0; i < 3; i++) {+ // only use Log- or TieredMergePolicy, to make document addition predictable and not+ // suddenly merge:+ MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(mp);+ IndexWriter w = new IndexWriter(ramDir, iwc);+ // add few more docs:+ for (int j = 0; j < RANDOM_MULTIPLIER * random().nextInt(30); j++) {+ TestBasicBackwardsCompatibility.addDoc(w, id++);+ }+ try {+ w.commit();+ } finally {+ w.close();+ }+ }++ // add dummy segments (which are all in current+ // version) to single segment index+ MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();+ IndexWriterConfig iwc = new IndexWriterConfig(null).setMergePolicy(mp);+ IndexWriter w = new IndexWriter(directory, iwc);+ w.addIndexes(ramDir);+ try (w) {+ w.commit();+ }++ // determine count of segments in modified index+ final int origSegCount = getNumberOfSegments(directory);++ // ensure there is only one commit+ assertEquals(1, DirectoryReader.listCommits(directory).size());+ newIndexUpgrader(directory).upgrade();++ final int segCount = checkAllSegmentsUpgraded(directory, indexCreatedVersion);+ assertEquals(+ "Index must still contain the same number of segments, as only one segment was upgraded and nothing else merged",+ origSegCount,+ segCount);+ }++ // LUCENE-5907+ public void testUpgradeWithNRTReader() throws Exception {+ IndexWriter writer =+ new IndexWriter(+ directory,+ newIndexWriterConfig(new MockAnalyzer(random()))+ .setOpenMode(IndexWriterConfig.OpenMode.APPEND));+ writer.addDocument(new Document());+ DirectoryReader r = DirectoryReader.open(writer);+ writer.commit();+ r.close();+ writer.forceMerge(1);+ writer.commit();+ writer.rollback();+ SegmentInfos.readLatestCommit(directory);+ }++ // LUCENE-5907+ public void testUpgradeThenMultipleCommits() throws Exception {+ IndexWriter writer =+ new IndexWriter(+ directory,+ newIndexWriterConfig(new MockAnalyzer(random()))+ .setOpenMode(IndexWriterConfig.OpenMode.APPEND));+ writer.addDocument(new Document());+ writer.commit();+ writer.addDocument(new Document());+ writer.commit();+ writer.close();+ }++ public void testIndexUpgraderCommandLineArgs() throws Exception {+ PrintStream savedSystemOut = System.out;+ System.setOut(new PrintStream(new ByteArrayOutputStream(), false, UTF_8));+ try {+ String name = indexName(this.version);+ Directory origDir = directory;+ int indexCreatedVersion =+ SegmentInfos.readLatestCommit(origDir).getIndexCreatedVersionMajor();+ Path dir = createTempDir(name);+ try (FSDirectory fsDir = FSDirectory.open(dir)) {+ // beware that ExtraFS might add extraXXX files+ Set<String> extraFiles = Set.of(fsDir.listAll());+ for (String file : origDir.listAll()) {+ if (extraFiles.contains(file) == false) {+ fsDir.copyFrom(origDir, file, file, IOContext.DEFAULT);+ }+ }+ }++ String path = dir.toAbsolutePath().toString();++ List<String> args = new ArrayList<>();+ if (random().nextBoolean()) {+ args.add("-verbose");+ }+ if (random().nextBoolean()) {+ args.add("-delete-prior-commits");+ }+ if (random().nextBoolean()) {+ // TODO: need to better randomize this, but ...+ // - LuceneTestCase.FS_DIRECTORIES is private+ // - newFSDirectory returns BaseDirectoryWrapper+ // - BaseDirectoryWrapper doesn't expose delegate+ Class<? extends FSDirectory> dirImpl = NIOFSDirectory.class;++ args.add("-dir-impl");+ args.add(dirImpl.getName());+ }+ args.add(path);++ IndexUpgrader.main(args.toArray(new String[0]));++ try (Directory upgradedDir = newFSDirectory(dir)) {+ checkAllSegmentsUpgraded(upgradedDir, indexCreatedVersion);+ }++ } finally {+ System.setOut(savedSystemOut);+ }+ }++ static int checkAllSegmentsUpgraded(Directory dir, int indexCreatedVersion) throws IOException {+ final SegmentInfos infos = SegmentInfos.readLatestCommit(dir);+ if (VERBOSE) {+ System.out.println("checkAllSegmentsUpgraded: " + infos);+ }+ for (SegmentCommitInfo si : infos) {+ assertEquals(Version.LATEST, si.info.getVersion());+ assertNotNull(si.getId());+ }+ assertEquals(Version.LATEST, infos.getCommitLuceneVersion());+ assertEquals(indexCreatedVersion, infos.getIndexCreatedVersionMajor());+ return infos.size();+ }++ static int getNumberOfSegments(Directory dir) throws IOException {+ final SegmentInfos infos = SegmentInfos.readLatestCommit(dir);+ return infos.size();+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestMoreTermsBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestMoreTermsBackwardsCompatibility.javanew file mode 100644index 00000000000..6bacb49dd65--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestMoreTermsBackwardsCompatibility.java@@ -0,0 +1,96 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_index;++import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;+import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import java.util.Random;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.document.SortedDocValuesField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.LogByteSizeMergePolicy;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.util.LineFileDocs;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.Version;++public class TestMoreTermsBackwardsCompatibility extends BackwardsCompatibilityTestBase {++ static final String INDEX_NAME = "moreterms";++ static final String SUFFIX = "";++ public TestMoreTermsBackwardsCompatibility(Version version, String pattern) {+ super(version, pattern);+ }++ @ParametersFactory(argumentFormatting = "Lucene-Version:%1$s; Pattern: %2$s")+ public static Iterable<Object[]> testVersionsFactory() {+ List<Object[]> params = new ArrayList<>();+ params.add(new Object[] {Version.LUCENE_9_0_0, createPattern(INDEX_NAME, SUFFIX)});+ return params;+ }++ @Override+ protected void createIndex(Directory directory) throws IOException {+ LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();+ mp.setNoCFSRatio(1.0);+ mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);+ MockAnalyzer analyzer = new MockAnalyzer(random());+ analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));++ IndexWriterConfig conf =+ new IndexWriterConfig(analyzer)+ .setMergePolicy(mp)+ .setCodec(TestUtil.getDefaultCodec())+ .setUseCompoundFile(false);+ IndexWriter writer = new IndexWriter(directory, conf);+ LineFileDocs docs = new LineFileDocs(new Random(0));+ for (int i = 0; i < 50; i++) {+ Document doc = TestUtil.cloneDocument(docs.nextDoc());+ doc.add(+ new NumericDocValuesField(+ "docid_intDV", doc.getField("docid_int").numericValue().longValue()));+ doc.add(+ new SortedDocValuesField("titleDV", new BytesRef(doc.getField("title").stringValue())));+ writer.addDocument(doc);+ if (i % 10 == 0) { // commit every 10 documents+ writer.commit();+ }+ }+ docs.close();+ writer.close();+ try (DirectoryReader reader = DirectoryReader.open(directory)) {+ TestIndexSortBackwardsCompatibility.searchExampleIndex(reader); // make sure we can search it+ }+ }++ public void testMoreTerms() throws Exception {+ try (DirectoryReader reader = DirectoryReader.open(directory)) {++ TestUtil.checkIndex(directory);+ TestIndexSortBackwardsCompatibility.searchExampleIndex(reader);+ }+ }+}