Skip to content

Apache Lucene open-source search software

Adding support for passing an ExecutorService into DirectoryReader.open() to enable concurrent segment reader initialization (#15428)

Bryce Kane7d4e22454b53
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2babac26f56..28d338b5869 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -156,6 +156,8 @@ API Changes
* GITHUB#15422: Add a new method in DirectWriter to compute how many bytes are written for encoding a number of values using a number of
bits per value. (Ignacio Vera)
+* GITHUB##15428: Add support for ExecutorServices to be passed into DirectoryReader.open() and DirectoryReader.openIfChanged() APIs (Bryce Kane)
+
New Features
---------------------
* GITHUB#15328: VectorSimilarityFunction.getValues() now implements doubleVal allowing its
diff --git a/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
index 7f4a118ad02..2077e29419d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.ExecutorService;
import org.apache.lucene.search.SearcherManager; // javadocs
import org.apache.lucene.store.Directory;
@@ -60,6 +61,18 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
return StandardDirectoryReader.open(directory, null, null);
}
+ /**
+ * Returns a IndexReader reading the index in the given Directory
+ *
+ * @param directory the index directory
+ * @param executorService used to open segment readers in parallel
+ * @throws IOException if there is a low-level IO error
+ */
+ public static DirectoryReader open(final Directory directory, ExecutorService executorService)
+ throws IOException {
+ return StandardDirectoryReader.open(directory, null, null, executorService);
+ }
+
/**
* Returns a IndexReader for the index in the given Directory
*
@@ -74,7 +87,26 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
*/
public static DirectoryReader open(final Directory directory, Comparator<LeafReader> leafSorter)
throws IOException {
- return StandardDirectoryReader.open(directory, null, leafSorter);
+ return StandardDirectoryReader.open(directory, null, leafSorter, null);
+ }
+
+ /**
+ * Returns a IndexReader for the index in the given Directory
+ *
+ * @param directory the index directory
+ * @param leafSorter a comparator for sorting leaf readers. Providing leafSorter is useful for
+ * indices on which it is expected to run many queries with particular sort criteria (e.g. for
+ * time-based indices this is usually a descending sort on timestamp). In this case {@code
+ * leafSorter} should sort leaves according to this sort criteria. Providing leafSorter allows
+ * to speed up this particular type of sort queries by early terminating while iterating
+ * through segments and segments' documents.
+ * @param executorService used to open segment readers in parallel
+ * @throws IOException if there is a low-level IO error
+ */
+ public static DirectoryReader open(
+ final Directory directory, Comparator<LeafReader> leafSorter, ExecutorService executorService)
+ throws IOException {
+ return StandardDirectoryReader.open(directory, null, leafSorter, executorService);
}
/**
@@ -119,7 +151,19 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
* @throws IOException if there is a low-level IO error
*/
public static DirectoryReader open(final IndexCommit commit) throws IOException {
- return StandardDirectoryReader.open(commit.getDirectory(), commit, null);
+ return StandardDirectoryReader.open(commit.getDirectory(), commit, null, null);
+ }
+
+ /**
+ * Expert: returns an IndexReader reading the index in the given {@link IndexCommit}.
+ *
+ * @param commit the commit point to open
+ * @param executorService used to open segment readers in parallel
+ * @throws IOException if there is a low-level IO error
+ */
+ public static DirectoryReader open(final IndexCommit commit, ExecutorService executorService)
+ throws IOException {
+ return StandardDirectoryReader.open(commit.getDirectory(), commit, null, executorService);
}
/**
@@ -144,7 +188,36 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
final IndexCommit commit, int minSupportedMajorVersion, Comparator<LeafReader> leafSorter)
throws IOException {
return StandardDirectoryReader.open(
- commit.getDirectory(), minSupportedMajorVersion, commit, leafSorter);
+ commit.getDirectory(), minSupportedMajorVersion, commit, leafSorter, null);
+ }
+
+ /**
+ * Expert: returns an IndexReader reading the index on the given {@link IndexCommit}. This method
+ * allows to open indices that were created with a Lucene version older than N-1 provided that all
+ * codecs for this index are available in the classpath and the segment file format used was
+ * created with Lucene 7 or newer. Users of this API must be aware that Lucene doesn't guarantee
+ * semantic compatibility for indices created with versions older than N-1. All backwards
+ * compatibility aside from the file format is optional and applied on a best effort basis.
+ *
+ * @param commit the commit point to open
+ * @param minSupportedMajorVersion the minimum supported major index version
+ * @param leafSorter a comparator for sorting leaf readers. Providing leafSorter is useful for
+ * indices on which it is expected to run many queries with particular sort criteria (e.g. for
+ * time-based indices, this is usually a descending sort on timestamp). In this case {@code
+ * leafSorter} should sort leaves according to this sort criteria. Providing leafSorter allows
+ * to speed up this particular type of sort queries by early terminating while iterating
+ * through segments and segments' documents
+ * @param executorService used to open segment readers in parallel
+ * @throws IOException if there is a low-level IO error
+ */
+ public static DirectoryReader open(
+ final IndexCommit commit,
+ int minSupportedMajorVersion,
+ Comparator<LeafReader> leafSorter,
+ ExecutorService executorService)
+ throws IOException {
+ return StandardDirectoryReader.open(
+ commit.getDirectory(), minSupportedMajorVersion, commit, leafSorter, executorService);
}
/**
@@ -171,6 +244,31 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
return newReader;
}
+ /**
+ * If the index has changed since the provided reader was opened, open and return a new reader;
+ * else, return null. The new reader, if not null, will be the same type of reader as the previous
+ * one, ie an NRT reader will open a new NRT reader etc.
+ *
+ * <p>This method is typically far less costly than opening a fully new <code>DirectoryReader
+ * </code> as it shares resources (for example sub-readers) with the provided <code>
+ * DirectoryReader</code>, when possible.
+ *
+ * <p>The provided reader is not closed (you are responsible for doing so); if a new reader is
+ * returned you also must eventually close it. Be sure to never close a reader while other threads
+ * are still using it; see {@link SearcherManager} to simplify managing this.
+ *
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ * @return null if there are no changes; else, a new DirectoryReader instance which you must
+ * eventually close
+ */
+ public static DirectoryReader openIfChanged(
+ DirectoryReader oldReader, ExecutorService executorService) throws IOException {
+ final DirectoryReader newReader = oldReader.doOpenIfChanged(executorService);
+ assert newReader != oldReader;
+ return newReader;
+ }
+
/**
* If the IndexCommit differs from what the provided reader is searching, open and return a new
* reader; else, return null.
@@ -184,6 +282,20 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
return newReader;
}
+ /**
+ * If the IndexCommit differs from what the provided reader is searching, open and return a new
+ * reader; else, return null. Executor service used to open segment readers in parallel.
+ *
+ * @see #openIfChanged(DirectoryReader)
+ */
+ public static DirectoryReader openIfChanged(
+ DirectoryReader oldReader, IndexCommit commit, ExecutorService executorService)
+ throws IOException {
+ final DirectoryReader newReader = oldReader.doOpenIfChanged(commit, executorService);
+ assert newReader != oldReader;
+ return newReader;
+ }
+
/**
* Expert: If there changes (committed or not) in the {@link IndexWriter} versus what the provided
* reader is searching, then open and return a new IndexReader searching both committed and
@@ -226,6 +338,50 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
return openIfChanged(oldReader, writer, true);
}
+ /**
+ * Expert: If there changes (committed or not) in the {@link IndexWriter} versus what the provided
+ * reader is searching, then open and return a new IndexReader searching both committed and
+ * uncommitted changes from the writer; else, return null (though, the current implementation
+ * never returns null).
+ *
+ * <p>This provides "near real-time" searching, in that changes made during an {@link IndexWriter}
+ * session can be quickly made available for searching without closing the writer or calling
+ * {@link IndexWriter#commit}.
+ *
+ * <p>It's <i>near</i> real-time because there is no hard guarantee on how quickly you can get a
+ * new reader after making changes with IndexWriter. You'll have to experiment in your situation
+ * to determine if it's fast enough. As this is a new and experimental feature, please report back
+ * on your findings so we can learn, improve and iterate.
+ *
+ * <p>The very first time this method is called, this writer instance will make every effort to
+ * pool the readers that it opens for doing merges, applying deletes, etc. This means additional
+ * resources (RAM, file descriptors, CPU time) will be consumed.
+ *
+ * <p>For lower latency on reopening a reader, you should call {@link
+ * IndexWriterConfig#setMergedSegmentWarmer} to pre-warm a newly merged segment before it's
+ * committed to the index. This is important for minimizing index-to-search delay after a large
+ * merge.
+ *
+ * <p>If an addIndexes* call is running in another thread, then this reader will only search those
+ * segments from the foreign index that have been successfully copied over, so far.
+ *
+ * <p><b>NOTE</b>: Once the writer is closed, any outstanding readers may continue to be used.
+ * However, if you attempt to reopen any of those readers, you'll hit an {@link
+ * org.apache.lucene.store.AlreadyClosedException}.
+ *
+ * @return DirectoryReader that covers entire index plus all changes made so far by this
+ * IndexWriter instance, or null if there are no new changes
+ * @param writer The IndexWriter to open from
+ * @param executorService used to open segment readers in parallel
+ * @throws IOException if there is a low-level IO error
+ * @lucene.experimental
+ */
+ public static DirectoryReader openIfChanged(
+ DirectoryReader oldReader, IndexWriter writer, ExecutorService executorService)
+ throws IOException {
+ return openIfChanged(oldReader, writer, true, executorService);
+ }
+
/**
* Expert: Opens a new reader, if there are any changes, controlling whether past deletions should
* be applied.
@@ -247,6 +403,33 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
return newReader;
}
+ /**
+ * Expert: Opens a new reader, if there are any changes, controlling whether past deletions should
+ * be applied.
+ *
+ * @see #openIfChanged(DirectoryReader,IndexWriter)
+ * @param writer The IndexWriter to open from
+ * @param applyAllDeletes If true, all buffered deletes will be applied (made visible) in the
+ * returned reader. If false, the deletes are not applied but remain buffered (in IndexWriter)
+ * so that they will be applied in the future. Applying deletes can be costly, so if your app
+ * can tolerate deleted documents being returned you might gain some performance by passing
+ * false.
+ * @param executorService used to open segment readers in parallel
+ * @throws IOException if there is a low-level IO error
+ * @lucene.experimental
+ */
+ public static DirectoryReader openIfChanged(
+ DirectoryReader oldReader,
+ IndexWriter writer,
+ boolean applyAllDeletes,
+ ExecutorService executorService)
+ throws IOException {
+ final DirectoryReader newReader =
+ oldReader.doOpenIfChanged(writer, applyAllDeletes, executorService);
+ assert newReader != oldReader;
+ return newReader;
+ }
+
/**
* Returns all commit points that exist in the Directory. Normally, because the default is {@link
* KeepOnlyLastCommitDeletionPolicy}, there would be only one commit point. But if you're using a
@@ -371,6 +554,18 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
*/
protected abstract DirectoryReader doOpenIfChanged() throws IOException;
+ /**
+ * Implement this method to support {@link #openIfChanged(DirectoryReader)}. If this reader does
+ * not support reopen, return {@code null}, so client code is happy. This should be consistent
+ * with {@link #isCurrent} (should always return {@code true}) if reopen is not supported.
+ * ExecutorService provides intra-opening concurrency.
+ *
+ * @throws IOException if there is a low-level IO error
+ * @return null if there are no changes; else, a new DirectoryReader instance.
+ */
+ protected abstract DirectoryReader doOpenIfChanged(ExecutorService executorService)
+ throws IOException;
+
/**
* Implement this method to support {@link #openIfChanged(DirectoryReader,IndexCommit)}. If this
* reader does not support reopen from a specific {@link IndexCommit}, throw {@link
@@ -381,6 +576,17 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
*/
protected abstract DirectoryReader doOpenIfChanged(final IndexCommit commit) throws IOException;
+ /**
+ * Implement this method to support {@link #openIfChanged(DirectoryReader,IndexCommit)}. If this
+ * reader does not support reopen from a specific {@link IndexCommit}, throw {@link
+ * UnsupportedOperationException}. ExecutorService provides intra-opening concurrency.
+ *
+ * @throws IOException if there is a low-level IO error
+ * @return null if there are no changes; else, a new DirectoryReader instance.
+ */
+ protected abstract DirectoryReader doOpenIfChanged(
+ final IndexCommit commit, ExecutorService executorService) throws IOException;
+
/**
* Implement this method to support {@link #openIfChanged(DirectoryReader,IndexWriter,boolean)}.
* If this reader does not support reopen from {@link IndexWriter}, throw {@link
@@ -392,6 +598,18 @@ public abstract class DirectoryReader extends BaseCompositeReader<LeafReader> {
protected abstract DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes)
throws IOException;
+ /**
+ * Implement this method to support {@link #openIfChanged(DirectoryReader,IndexWriter,boolean)}.
+ * If this reader does not support reopen from {@link IndexWriter}, throw {@link
+ * UnsupportedOperationException}. ExecutorService used to open segment readers in parallel.
+ *
+ * @throws IOException if there is a low-level IO error
+ * @return null if there are no changes; else, a new DirectoryReader instance.
+ */
+ protected abstract DirectoryReader doOpenIfChanged(
+ IndexWriter writer, boolean applyAllDeletes, ExecutorService executorService)
+ throws IOException;
+
/**
* Version number when this IndexReader was opened.
*
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
index 4e2fe24be65..c286dfc34e3 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.List;
+import java.util.concurrent.ExecutorService;
/**
* A FilterDirectoryReader wraps another DirectoryReader, allowing implementations to transform or
@@ -112,17 +113,36 @@ public abstract class FilterDirectoryReader extends DirectoryReader {
return wrapDirectoryReader(in.doOpenIfChanged());
}
+ @Override
+ protected final DirectoryReader doOpenIfChanged(ExecutorService executorService)
+ throws IOException {
+ return wrapDirectoryReader(in.doOpenIfChanged(executorService));
+ }
+
@Override
protected final DirectoryReader doOpenIfChanged(IndexCommit commit) throws IOException {
return wrapDirectoryReader(in.doOpenIfChanged(commit));
}
+ @Override
+ protected final DirectoryReader doOpenIfChanged(
+ IndexCommit commit, ExecutorService executorService) throws IOException {
+ return wrapDirectoryReader(in.doOpenIfChanged(commit, executorService));
+ }
+
@Override
protected final DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes)
throws IOException {
return wrapDirectoryReader(in.doOpenIfChanged(writer, applyAllDeletes));
}
+ @Override
+ protected final DirectoryReader doOpenIfChanged(
+ IndexWriter writer, boolean applyAllDeletes, ExecutorService executorService)
+ throws IOException {
+ return wrapDirectoryReader(in.doOpenIfChanged(writer, applyAllDeletes, executorService));
+ }
+
@Override
public long getVersion() {
return in.getVersion();
diff --git a/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
index fbdf8895747..16969ecefdd 100644
--- a/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
@@ -27,6 +27,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -62,9 +65,12 @@ public final class StandardDirectoryReader extends DirectoryReader {
}
static DirectoryReader open(
- final Directory directory, final IndexCommit commit, Comparator<LeafReader> leafSorter)
+ final Directory directory,
+ final IndexCommit commit,
+ Comparator<LeafReader> leafSorter,
+ ExecutorService executor)
throws IOException {
- return open(directory, Version.MIN_SUPPORTED_MAJOR, commit, leafSorter);
+ return open(directory, Version.MIN_SUPPORTED_MAJOR, commit, leafSorter, executor);
}
/** called from DirectoryReader.open(...) methods */
@@ -72,7 +78,8 @@ public final class StandardDirectoryReader extends DirectoryReader {
final Directory directory,
int minSupportedMajorVersion,
final IndexCommit commit,
- Comparator<LeafReader> leafSorter)
+ Comparator<LeafReader> leafSorter,
+ ExecutorService executor)
throws IOException {
return new SegmentInfos.FindSegmentsFile<DirectoryReader>(directory) {
@Override
@@ -86,13 +93,8 @@ public final class StandardDirectoryReader extends DirectoryReader {
}
SegmentInfos sis =
SegmentInfos.readCommit(directory, segmentFileName, minSupportedMajorVersion);
- final SegmentReader[] readers = new SegmentReader[sis.size()];
+ SegmentReader[] readers = createSegmentReaders(sis, null, executor);
try {
- for (int i = sis.size() - 1; i >= 0; i--) {
- readers[i] =
- new SegmentReader(
- sis.info(i), sis.getIndexCreatedVersionMajor(), IOContext.DEFAULT);
- }
// This may throw CorruptIndexException if there are too many docs, so
// it must be inside try clause so we close readers in that case:
return new StandardDirectoryReader(
@@ -166,8 +168,8 @@ public final class StandardDirectoryReader extends DirectoryReader {
}
/**
- * This constructor is only used for {@link #doOpenIfChanged(SegmentInfos)}, as well as NRT
- * replication.
+ * This constructor is only used for {@link #doOpenIfChanged(SegmentInfos, ExecutorService)} and
+ * NRT replication
*
* @lucene.internal
*/
@@ -175,60 +177,168 @@ public final class StandardDirectoryReader extends DirectoryReader {
Directory directory,
SegmentInfos infos,
List<? extends LeafReader> oldReaders,
- Comparator<LeafReader> leafSorter)
+ Comparator<LeafReader> leafSorter,
+ ExecutorService executor)
throws IOException {
+ SegmentReader[] newReaders = createSegmentReaders(infos, oldReaders, executor);
+ return new StandardDirectoryReader(
+ directory, newReaders, null, infos, leafSorter, false, false);
+ }
+ /** Creates segment readers, will prefer to reuse existing segment readers if possible * */
+ private static SegmentReader[] createSegmentReaders(
+ SegmentInfos sis, List<? extends LeafReader> oldReaders, ExecutorService executor)
+ throws IOException {
// we put the old SegmentReaders in a map, that allows us
// to lookup a reader using its segment name
- Map<String, Integer> segmentReaders = Collections.emptyMap();
+ Map<String, Integer> previousSegmentReaders = mapPreviousReaders(oldReaders);
+
+ final SegmentReader[] readers = new SegmentReader[sis.size()];
+ if (executor != null) {
+ List<Future<SegmentReader>> futures = new ArrayList<>();
+ for (int i = sis.size() - 1; i >= 0; i--) {
+ SegmentCommitInfo commitInfo = sis.info(i);
+ SegmentReader oldReader =
+ getOldSegmentReader(
+ oldReaders, previousSegmentReaders.get(commitInfo.info.name), commitInfo);
+ futures.add(
+ executor.submit(
+ () ->
+ createOrReuseSegmentReader(
+ commitInfo, oldReader, sis.getIndexCreatedVersionMajor())));
+ }
+ RuntimeException firstException = null;
+ for (int i = 0; i < futures.size(); i++) {
+ try {
+ readers[sis.size() - 1 - i] = futures.get(i).get();
+ } catch (ExecutionException | InterruptedException e) {
+ // If there is an exception creating the reader we still process
+ // the rest of the completed futures to allow us to close created readers
+ if (firstException == null) firstException = new RuntimeException(e);
+ }
+ }
+ if (firstException != null) {
+ // If its a new reader decRef should close... otherwise old readers we will remove the new
+ // refs that were created
+ decRefWhileSuppressingException(firstException, readers);
+ // Unwrap IOExceptions
+ Throwable cause = firstException.getCause();
+ if (cause instanceof ExecutionException && (cause).getCause() instanceof IOException) {
+ throw (IOException) (cause).getCause();
+ }
+ throw firstException;
+ }
+ } else {
+ // No Executor Service has been passed in fallback to sequentially processing on the caller
+ // thread
+ try {
+ for (int i = sis.size() - 1; i >= 0; i--) {
+ SegmentCommitInfo commitInfo = sis.info(i);
+ SegmentReader oldReader =
+ getOldSegmentReader(
+ oldReaders, previousSegmentReaders.get(commitInfo.info.name), commitInfo);
+ readers[i] =
+ createOrReuseSegmentReader(commitInfo, oldReader, sis.getIndexCreatedVersionMajor());
+ }
+ } catch (Throwable t) {
+ decRefWhileSuppressingException(t, readers);
+ throw t;
+ }
+ }
+ return readers;
+ }
+
+ private static Map<String, Integer> mapPreviousReaders(List<? extends LeafReader> oldReaders) {
+ Map<String, Integer> previousSegmentReaders = Collections.emptyMap();
if (oldReaders != null) {
- segmentReaders = CollectionUtil.newHashMap(oldReaders.size());
+ previousSegmentReaders = CollectionUtil.newHashMap(oldReaders.size());
// create a Map SegmentName->SegmentReader
for (int i = 0, c = oldReaders.size(); i < c; i++) {
final SegmentReader sr = (SegmentReader) oldReaders.get(i);
- segmentReaders.put(sr.getSegmentName(), Integer.valueOf(i));
+ previousSegmentReaders.put(sr.getSegmentName(), Integer.valueOf(i));
}
}
+ return previousSegmentReaders;
+ }
- SegmentReader[] newReaders = new SegmentReader[infos.size()];
- for (int i = infos.size() - 1; i >= 0; i--) {
- SegmentCommitInfo commitInfo = infos.info(i);
+ private static SegmentReader getOldSegmentReader(
+ List<? extends LeafReader> oldReaders, Integer oldReaderIndex, SegmentCommitInfo commitInfo) {
+ SegmentReader oldReader;
+ if (oldReaderIndex == null) {
+ // this is a new segment, no old SegmentReader can be reused
+ oldReader = null;
+ } else {
+ // there is an old reader for this segment - we'll try to reopen it
+ oldReader = (SegmentReader) oldReaders.get(oldReaderIndex.intValue());
+ }
- // find SegmentReader for this segment
- Integer oldReaderIndex = segmentReaders.get(commitInfo.info.name);
- SegmentReader oldReader;
- if (oldReaderIndex == null) {
- // this is a new segment, no old SegmentReader can be reused
- oldReader = null;
- } else {
- // there is an old reader for this segment - we'll try to reopen it
- oldReader = (SegmentReader) oldReaders.get(oldReaderIndex.intValue());
- }
+ // Make a best effort to detect when the app illegally "rm -rf" their
+ // index while a reader was open, and then called openIfChanged:
+ if (oldReader != null
+ && Arrays.equals(commitInfo.info.getId(), oldReader.getSegmentInfo().info.getId())
+ == false) {
+ throw new IllegalStateException(
+ "same segment "
+ + commitInfo.info.name
+ + " has invalid doc count change; likely you are re-opening a reader after illegally removing index files yourself and building a new index in their place. Use IndexWriter.deleteAll or open a new IndexWriter using OpenMode.CREATE instead");
+ }
+ return oldReader;
+ }
- // Make a best effort to detect when the app illegally "rm -rf" their
- // index while a reader was open, and then called openIfChanged:
- if (oldReader != null
- && Arrays.equals(commitInfo.info.getId(), oldReader.getSegmentInfo().info.getId())
- == false) {
- throw new IllegalStateException(
- "same segment "
- + commitInfo.info.name
- + " has invalid doc count change; likely you are re-opening a reader after illegally removing index files yourself and building a new index in their place. Use IndexWriter.deleteAll or open a new IndexWriter using OpenMode.CREATE instead");
- }
+ private static SegmentReader createOrReuseSegmentReader(
+ SegmentCommitInfo commitInfo, SegmentReader oldReader, int indexCreatedVersionMajor)
+ throws IOException {
- try {
- SegmentReader newReader;
- if (oldReader == null
- || commitInfo.info.getUseCompoundFile()
- != oldReader.getSegmentInfo().info.getUseCompoundFile()) {
- // this is a new reader; in case we hit an exception we can decRef it safely
- newReader =
- new SegmentReader(commitInfo, infos.getIndexCreatedVersionMajor(), IOContext.DEFAULT);
- newReaders[i] = newReader;
+ SegmentReader newReader;
+ if (oldReader == null
+ || commitInfo.info.getUseCompoundFile()
+ != oldReader.getSegmentInfo().info.getUseCompoundFile()) {
+ // this is a new reader; in case we hit an exception we can decRef it safely
+ newReader = new SegmentReader(commitInfo, indexCreatedVersionMajor, IOContext.DEFAULT);
+ } else {
+ if (oldReader.isNRT) {
+ // We must load liveDocs/DV updates from disk:
+ Bits liveDocs =
+ commitInfo.hasDeletions()
+ ? commitInfo
+ .info
+ .getCodec()
+ .liveDocsFormat()
+ .readLiveDocs(commitInfo.info.dir, commitInfo, IOContext.READONCE)
+ : null;
+ newReader =
+ new SegmentReader(
+ commitInfo,
+ oldReader,
+ liveDocs,
+ liveDocs,
+ commitInfo.info.maxDoc() - commitInfo.getDelCount(),
+ false);
+ } else {
+ if (oldReader.getSegmentInfo().getDelGen() == commitInfo.getDelGen()
+ && oldReader.getSegmentInfo().getFieldInfosGen() == commitInfo.getFieldInfosGen()) {
+ // No change; this reader will be shared between
+ // the old and the new one, so we must incRef
+ // it:
+ oldReader.incRef();
+ newReader = oldReader;
} else {
- if (oldReader.isNRT) {
- // We must load liveDocs/DV updates from disk:
+ // Steal the ref returned by SegmentReader ctor:
+ assert commitInfo.info.dir == oldReader.getSegmentInfo().info.dir;
+
+ if (oldReader.getSegmentInfo().getDelGen() == commitInfo.getDelGen()) {
+ // only DV updates
+ newReader =
+ new SegmentReader(
+ commitInfo,
+ oldReader,
+ oldReader.getLiveDocs(),
+ oldReader.getHardLiveDocs(),
+ oldReader.numDocs(),
+ false); // this is not an NRT reader!
+ } else {
+ // both DV and liveDocs have changed
Bits liveDocs =
commitInfo.hasDeletions()
? commitInfo
@@ -237,7 +347,7 @@ public final class StandardDirectoryReader extends DirectoryReader {
.liveDocsFormat()
.readLiveDocs(commitInfo.info.dir, commitInfo, IOContext.READONCE)
: null;
- newReaders[i] =
+ newReader =
new SegmentReader(
commitInfo,
oldReader,
@@ -245,57 +355,11 @@ public final class StandardDirectoryReader extends DirectoryReader {
liveDocs,
commitInfo.info.maxDoc() - commitInfo.getDelCount(),
false);
- } else {
- if (oldReader.getSegmentInfo().getDelGen() == commitInfo.getDelGen()
- && oldReader.getSegmentInfo().getFieldInfosGen() == commitInfo.getFieldInfosGen()) {
- // No change; this reader will be shared between
- // the old and the new one, so we must incRef
- // it:
- oldReader.incRef();
- newReaders[i] = oldReader;
- } else {
- // Steal the ref returned by SegmentReader ctor:
- assert commitInfo.info.dir == oldReader.getSegmentInfo().info.dir;
-
- if (oldReader.getSegmentInfo().getDelGen() == commitInfo.getDelGen()) {
- // only DV updates
- newReaders[i] =
- new SegmentReader(
- commitInfo,
- oldReader,
- oldReader.getLiveDocs(),
- oldReader.getHardLiveDocs(),
- oldReader.numDocs(),
- false); // this is not an NRT reader!
- } else {
- // both DV and liveDocs have changed
- Bits liveDocs =
- commitInfo.hasDeletions()
- ? commitInfo
- .info
- .getCodec()
- .liveDocsFormat()
- .readLiveDocs(commitInfo.info.dir, commitInfo, IOContext.READONCE)
- : null;
- newReaders[i] =
- new SegmentReader(
- commitInfo,
- oldReader,
- liveDocs,
- liveDocs,
- commitInfo.info.maxDoc() - commitInfo.getDelCount(),
- false);
- }
- }
}
}
- } catch (Throwable t) {
- decRefWhileSuppressingException(t, newReaders);
- throw t;
}
}
- return new StandardDirectoryReader(
- directory, newReaders, null, infos, leafSorter, false, false);
+ return newReader;
}
// TODO: move somewhere shared if it's useful elsewhere
@@ -333,7 +397,12 @@ public final class StandardDirectoryReader extends DirectoryReader {
@Override
protected DirectoryReader doOpenIfChanged() throws IOException {
- return doOpenIfChanged((IndexCommit) null);
+ return doOpenIfChanged((IndexCommit) null, null);
+ }
+
+ @Override
+ protected DirectoryReader doOpenIfChanged(ExecutorService executorService) throws IOException {
+ return doOpenIfChanged((IndexCommit) null, executorService);
}
@Override
@@ -343,9 +412,23 @@ public final class StandardDirectoryReader extends DirectoryReader {
// If we were obtained by writer.getReader(), re-ask the
// writer to get a new reader.
if (writer != null) {
- return doOpenFromWriter(commit);
+ return doOpenFromWriter(commit, null);
+ } else {
+ return doOpenNoWriter(commit, null);
+ }
+ }
+
+ @Override
+ protected DirectoryReader doOpenIfChanged(
+ final IndexCommit commit, ExecutorService executorService) throws IOException {
+ ensureOpen();
+
+ // If we were obtained by writer.getReader(), re-ask the
+ // writer to get a new reader.
+ if (writer != null) {
+ return doOpenFromWriter(commit, executorService);
} else {
- return doOpenNoWriter(commit);
+ return doOpenNoWriter(commit, executorService);
}
}
@@ -354,15 +437,28 @@ public final class StandardDirectoryReader extends DirectoryReader {
throws IOException {
ensureOpen();
if (writer == this.writer && applyAllDeletes == this.applyAllDeletes) {
- return doOpenFromWriter(null);
+ return doOpenFromWriter(null, null);
} else {
return writer.getReader(applyAllDeletes, writeAllDeletes);
}
}
- private DirectoryReader doOpenFromWriter(IndexCommit commit) throws IOException {
+ @Override
+ protected DirectoryReader doOpenIfChanged(
+ IndexWriter writer, boolean applyAllDeletes, ExecutorService executorService)
+ throws IOException {
+ ensureOpen();
+ if (writer == this.writer && applyAllDeletes == this.applyAllDeletes) {
+ return doOpenFromWriter(null, executorService);
+ } else {
+ return writer.getReader(applyAllDeletes, writeAllDeletes);
+ }
+ }
+
+ private DirectoryReader doOpenFromWriter(IndexCommit commit, ExecutorService executorService)
+ throws IOException {
if (commit != null) {
- return doOpenFromCommit(commit);
+ return doOpenFromCommit(commit, executorService);
}
if (writer.nrtIsCurrent(segmentInfos)) {
@@ -380,8 +476,8 @@ public final class StandardDirectoryReader extends DirectoryReader {
return reader;
}
- private DirectoryReader doOpenNoWriter(IndexCommit commit) throws IOException {
-
+ private DirectoryReader doOpenNoWriter(IndexCommit commit, ExecutorService executorService)
+ throws IOException {
if (commit == null) {
if (isCurrent()) {
return null;
@@ -396,22 +492,24 @@ public final class StandardDirectoryReader extends DirectoryReader {
}
}
- return doOpenFromCommit(commit);
+ return doOpenFromCommit(commit, executorService);
}
- private DirectoryReader doOpenFromCommit(IndexCommit commit) throws IOException {
+ private DirectoryReader doOpenFromCommit(IndexCommit commit, ExecutorService executorService)
+ throws IOException {
return new SegmentInfos.FindSegmentsFile<DirectoryReader>(directory) {
@Override
protected DirectoryReader doBody(String segmentFileName) throws IOException {
final SegmentInfos infos = SegmentInfos.readCommit(directory, segmentFileName);
- return doOpenIfChanged(infos);
+ return doOpenIfChanged(infos, executorService);
}
}.run(commit);
}
- DirectoryReader doOpenIfChanged(SegmentInfos infos) throws IOException {
+ DirectoryReader doOpenIfChanged(SegmentInfos infos, ExecutorService executorService)
+ throws IOException {
return StandardDirectoryReader.open(
- directory, infos, getSequentialSubReaders(), subReadersSorter);
+ directory, infos, getSequentialSubReaders(), subReadersSorter, executorService);
}
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
index 0e0bae7a1a0..5879ee83719 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
@@ -24,12 +24,21 @@ import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -49,6 +58,7 @@ import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util.SuppressForbidden;
import org.apache.lucene.util.Version;
import org.junit.Assume;
@@ -755,6 +765,67 @@ public class TestDirectoryReader extends LuceneTestCase {
d.close();
}
+ public void testGetIndexCommitWithExecutorService() throws IOException {
+
+ Directory d = newDirectory();
+ ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReader"));
+
+ try {
+ // set up writer
+ IndexWriter writer =
+ new IndexWriter(
+ d,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMaxBufferedDocs(2)
+ .setMergePolicy(newLogMergePolicy(10)));
+ for (int i = 0; i < 27; i++) addDocumentWithFields(writer);
+ writer.close();
+
+ SegmentInfos sis = SegmentInfos.readLatestCommit(d);
+ DirectoryReader r = DirectoryReader.open(d);
+ IndexCommit c = r.getIndexCommit();
+
+ assertEquals(sis.getSegmentsFileName(), c.getSegmentsFileName());
+
+ assertEquals(c, r.getIndexCommit());
+
+ // Change the index
+ writer =
+ new IndexWriter(
+ d,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.APPEND)
+ .setMaxBufferedDocs(2)
+ .setMergePolicy(newLogMergePolicy(10)));
+ for (int i = 0; i < 7; i++) addDocumentWithFields(writer);
+ writer.close();
+
+ DirectoryReader r2 = DirectoryReader.openIfChanged(r, executorService);
+ assertNotNull(r2);
+ assertNotEquals(c, r2.getIndexCommit());
+ assertNotEquals(1, r2.getIndexCommit().getSegmentCount());
+ r2.close();
+
+ writer =
+ new IndexWriter(
+ d, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
+ writer.forceMerge(1);
+ writer.close();
+
+ r2 = DirectoryReader.openIfChanged(r, executorService);
+ assertNotNull(r2);
+ assertNull(DirectoryReader.openIfChanged(r2, executorService));
+ assertEquals(1, r2.getIndexCommit().getSegmentCount());
+
+ r.close();
+ r2.close();
+ } finally {
+ executorService.shutdown();
+ d.close();
+ }
+ }
+
static Document createDocument(String id) {
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
@@ -775,6 +846,19 @@ public class TestDirectoryReader extends LuceneTestCase {
dir.close();
}
+ public void testNoDirWithExecutor() throws Throwable {
+ Path tempDir = createTempDir("doesnotexist");
+ Directory dir = newFSDirectory(tempDir);
+ ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReader"));
+ try {
+ expectThrows(IndexNotFoundException.class, () -> DirectoryReader.open(dir, executorService));
+ } finally {
+ dir.close();
+ executorService.shutdown();
+ }
+ }
+
// LUCENE-1509
public void testNoDupCommitFileNames() throws Throwable {
@@ -1121,4 +1205,232 @@ public class TestDirectoryReader extends LuceneTestCase {
DirectoryReader.open(commit, random().nextInt(Version.LATEST.major + 1), null).close();
}
}
+
+ public void testOpenWithExecutorService() throws IOException {
+ Directory dir = newDirectory();
+ createMultiSegmentIndex(dir, 5);
+
+ ExecutorService executor =
+ Executors.newFixedThreadPool(1, new NamedThreadFactory("TestDirectoryReader"));
+ try {
+ DirectoryReader reader = DirectoryReader.open(dir, executor);
+ assertNotNull(reader);
+ assertTrue(reader.numDocs() > 0);
+ reader.close();
+ } finally {
+ executor.shutdown();
+ dir.close();
+ }
+ }
+
+ public void testOpenWithLeafSorterAndExecutor() throws IOException {
+ Directory dir = newDirectory();
+ createMultiSegmentIndex(dir, 3);
+
+ ExecutorService executor =
+ Executors.newFixedThreadPool(1, new NamedThreadFactory("TestDirectoryReader"));
+ Comparator<LeafReader> sorter = (r1, r2) -> Integer.compare(r1.numDocs(), r2.numDocs());
+
+ try {
+ DirectoryReader reader = DirectoryReader.open(dir, sorter, executor);
+ assertNotNull(reader);
+ reader.close();
+ } finally {
+ executor.shutdown();
+ dir.close();
+ }
+ }
+
+ public void testOpenCommitWithExecutor() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+ Document doc = new Document();
+ doc.add(newStringField("id", "1", Field.Store.YES));
+ writer.addDocument(doc);
+ writer.commit();
+ writer.close();
+
+ IndexCommit commit = DirectoryReader.listCommits(dir).get(0);
+ ExecutorService executor =
+ Executors.newFixedThreadPool(1, new NamedThreadFactory("TestDirectoryReader"));
+ try {
+ DirectoryReader reader = DirectoryReader.open(commit, executor);
+ assertNotNull(reader);
+ assertEquals(1, reader.numDocs());
+ reader.close();
+ } finally {
+ executor.shutdown();
+ dir.close();
+ }
+ }
+
+ public void testOpenCommitWithMinVersionAndExecutor() throws IOException {
+ Directory dir = newDirectory();
+ createMultiSegmentIndex(dir, 3);
+
+ ExecutorService executor =
+ Executors.newFixedThreadPool(1, new NamedThreadFactory("TestDirectoryReader"));
+ Comparator<LeafReader> sorter = (r1, r2) -> Integer.compare(r1.numDocs(), r2.numDocs());
+
+ try (DirectoryReader baseReader = DirectoryReader.open(dir)) {
+ IndexCommit commit = baseReader.getIndexCommit();
+ DirectoryReader reader =
+ DirectoryReader.open(commit, Version.MIN_SUPPORTED_MAJOR, sorter, executor);
+ assertNotNull(reader);
+ assertEquals(baseReader.numDocs(), reader.numDocs());
+ reader.close();
+ } finally {
+ executor.shutdown();
+ dir.close();
+ }
+ }
+
+ public void testNullExecutorService() throws IOException {
+ Directory dir = newDirectory();
+ createMultiSegmentIndex(dir, 3);
+
+ // Should work fine with null executor (fallback to sequential)
+ DirectoryReader reader = DirectoryReader.open(dir, (ExecutorService) null);
+ assertNotNull(reader);
+ assertTrue(reader.numDocs() > 0);
+ reader.close();
+ dir.close();
+ }
+
+ public void testExecutorServiceExceptionHandling() throws IOException {
+ Directory dir = newDirectory();
+ createMultiSegmentIndex(dir, 3);
+
+ // Test with shutdown executor
+ ExecutorService executor =
+ Executors.newFixedThreadPool(1, new NamedThreadFactory("TestDirectoryReader"));
+ executor.shutdown();
+
+ expectThrows(
+ RuntimeException.class,
+ () -> {
+ DirectoryReader.open(dir, executor);
+ });
+
+ dir.close();
+ }
+
+ public void testExecutorServicePartialFailure() throws IOException {
+ Directory dir = newDirectory();
+ createMultiSegmentIndex(dir, 3);
+
+ // Create an executor that fails on the second task but succeeds on others
+ ExecutorService executor = createExecutorServiceThatFailsOneTask();
+
+ try {
+ expectThrows(
+ RuntimeException.class,
+ () -> {
+ DirectoryReader.open(dir, executor);
+ });
+ } finally {
+ executor.shutdown();
+ dir.close();
+ }
+ }
+
+ private ExecutorService createExecutorServiceThatFailsOneTask() {
+ return new ExecutorService() {
+ private int taskCount = 0;
+ private final ExecutorService delegate =
+ Executors.newFixedThreadPool(2, new NamedThreadFactory("TestDirectoryReader"));
+
+ @Override
+ public <T> Future<T> submit(Callable<T> task) {
+ taskCount++;
+ if (taskCount == 2) {
+ // Second task fails
+ CompletableFuture<T> future = new CompletableFuture<>();
+ future.completeExceptionally(
+ new RuntimeException(new IOException("Simulated failure on segment")));
+ return future;
+ } else {
+ // Other tasks succeed normally
+ return delegate.submit(task);
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ delegate.shutdown();
+ }
+
+ @Override
+ public List<Runnable> shutdownNow() {
+ return delegate.shutdownNow();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return delegate.isShutdown();
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return delegate.isTerminated();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return delegate.awaitTermination(timeout, unit);
+ }
+
+ @Override
+ public Future<?> submit(Runnable task) {
+ return delegate.submit(task);
+ }
+
+ @Override
+ public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks)
+ throws InterruptedException {
+ return delegate.invokeAll(tasks);
+ }
+
+ @Override
+ public <T> List<Future<T>> invokeAll(
+ Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit)
+ throws InterruptedException {
+ return delegate.invokeAll(tasks, timeout, unit);
+ }
+
+ @Override
+ public <T> T invokeAny(Collection<? extends Callable<T>> tasks)
+ throws InterruptedException, ExecutionException {
+ return delegate.invokeAny(tasks);
+ }
+
+ @Override
+ public <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ return delegate.invokeAny(tasks, timeout, unit);
+ }
+
+ @Override
+ public <T> Future<T> submit(Runnable task, T result) {
+ return delegate.submit(task, result);
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ delegate.execute(command);
+ }
+ };
+ }
+
+ private void createMultiSegmentIndex(Directory dir, int numSegments) throws IOException {
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig().setMaxBufferedDocs(2));
+ for (int i = 0; i < numSegments; i++) {
+ Document doc = new Document();
+ doc.add(newStringField("id", String.valueOf(i), Field.Store.YES));
+ doc.add(newTextField("content", "segment " + i + " content", Field.Store.YES));
+ writer.addDocument(doc);
+ writer.commit(); // Force new segment
+ }
+ writer.close();
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
index 3acd59cfc5b..ebd099d0b94 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
@@ -26,6 +26,8 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.document.Field;
@@ -45,6 +47,7 @@ import org.apache.lucene.tests.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.NamedThreadFactory;
public class TestDirectoryReaderReopen extends LuceneTestCase {
@@ -1138,4 +1141,881 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
assertEquals(0, reader.numDeletedDocs());
IOUtils.close(reader, writer, dir);
}
+
+ public void testReopenWithExecutor() throws Exception {
+ final Directory dir1 = newDirectory();
+ final ExecutorService executorService1 =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ createIndex(random(), dir1, false);
+ performDefaultTests(
+ new TestReopen() {
+
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ TestDirectoryReaderReopen.modifyIndex(i, dir1);
+ }
+
+ @Override
+ protected DirectoryReader openReader() throws IOException {
+ return DirectoryReader.open(dir1, executorService1);
+ }
+ });
+ } finally {
+ executorService1.shutdown();
+ }
+ dir1.close();
+
+ final Directory dir2 = newDirectory();
+ final ExecutorService executorService2 =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ createIndex(random(), dir2, true);
+ performDefaultTests(
+ new TestReopen() {
+
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ TestDirectoryReaderReopen.modifyIndex(i, dir2);
+ }
+
+ @Override
+ protected DirectoryReader openReader() throws IOException {
+ return DirectoryReader.open(dir2, executorService2);
+ }
+ });
+ } finally {
+ executorService2.shutdown();
+ }
+ dir2.close();
+ }
+
+ public void testCommitReopenWithExecutor() throws IOException {
+ Directory dir = newDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ doTestReopenWithCommit(random(), dir, true, executorService);
+ } finally {
+ executorService.shutdown();
+ }
+ dir.close();
+ }
+
+ private void doTestReopenWithCommit(
+ Random random, Directory dir, boolean withReopen, ExecutorService executorService)
+ throws IOException {
+ IndexWriter iwriter =
+ new IndexWriter(
+ dir,
+ newIndexWriterConfig(new MockAnalyzer(random))
+ .setOpenMode(OpenMode.CREATE)
+ .setMergeScheduler(new SerialMergeScheduler())
+ .setMergePolicy(newLogMergePolicy()));
+ iwriter.commit();
+ DirectoryReader reader = DirectoryReader.open(dir, executorService);
+
+ try {
+ int M = 3;
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.setTokenized(false);
+ FieldType customType2 = new FieldType(TextField.TYPE_STORED);
+ customType2.setTokenized(false);
+ customType2.setOmitNorms(true);
+ FieldType customType3 = new FieldType();
+ customType3.setStored(true);
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < M; j++) {
+ Document doc = new Document();
+ doc.add(newField("id", i + "_" + j, customType));
+ doc.add(newField("id2", i + "_" + j, customType2));
+ doc.add(newField("id3", i + "_" + j, customType3));
+
+ if (i > 0) {
+ int k = i - 1;
+ int n = j + k * M;
+ Document prevItereationDoc = reader.storedFields().document(n);
+ assertNotNull(prevItereationDoc);
+ String id = prevItereationDoc.get("id");
+ assertEquals(k + "_" + j, id);
+ }
+
+ iwriter.addDocument(doc);
+ }
+ iwriter.commit();
+ if (withReopen) {
+ DirectoryReader r2 = DirectoryReader.openIfChanged(reader, executorService);
+ if (r2 != null) {
+ reader.close();
+ reader = r2;
+ }
+ } else {
+ reader.close();
+ reader = DirectoryReader.open(dir, executorService);
+ }
+ }
+ } finally {
+ iwriter.close();
+ reader.close();
+ }
+ }
+
+ public void testCommitRecreateWithExecutor() throws IOException {
+ Directory dir = newDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ doTestReopenWithCommit(random(), dir, false, executorService);
+ } finally {
+ executorService.shutdown();
+ }
+ dir.close();
+ }
+
+ public void testThreadSafetyWithExecutor() throws Exception {
+ final Directory dir = newDirectory();
+ // NOTE: this also controls the number of threads!
+ final int n = TestUtil.nextInt(random(), 20, 40);
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ for (int i = 0; i < n; i++) {
+ writer.addDocument(createDocument(i, 3));
+ }
+ writer.forceMerge(1);
+ writer.close();
+
+ final TestReopen test =
+ new TestReopen() {
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ IndexWriter modifier =
+ new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
+ modifier.addDocument(createDocument(n + i, 6));
+ modifier.close();
+ }
+
+ @Override
+ protected DirectoryReader openReader() throws IOException {
+ return DirectoryReader.open(dir, executorService);
+ }
+ };
+
+ final List<ReaderCouple> readers =
+ Collections.synchronizedList(new ArrayList<ReaderCouple>());
+ DirectoryReader firstReader = DirectoryReader.open(dir, executorService);
+ DirectoryReader reader = firstReader;
+
+ ReaderThread[] threads = new ReaderThread[n];
+ final Set<DirectoryReader> readersToClose =
+ Collections.synchronizedSet(new HashSet<DirectoryReader>());
+
+ for (int i = 0; i < n; i++) {
+ if (i % 2 == 0) {
+ DirectoryReader refreshed = DirectoryReader.openIfChanged(reader, executorService);
+ if (refreshed != null) {
+ readersToClose.add(reader);
+ reader = refreshed;
+ }
+ }
+ final DirectoryReader r = reader;
+
+ final int index = i;
+
+ ReaderThreadTask task;
+
+ if (i < 4 || (i >= 10 && i < 14) || i > 18) {
+ task =
+ new ReaderThreadTask() {
+
+ @Override
+ public void run() throws Exception {
+ Random rnd = LuceneTestCase.random();
+ while (!stopped) {
+ if (index % 2 == 0) {
+ // refresh reader synchronized
+ ReaderCouple c = (refreshReader(r, test, index, true));
+ readersToClose.add(c.newReader);
+ readersToClose.add(c.refreshedReader);
+ readers.add(c);
+ // prevent too many readers
+ break;
+ } else {
+ // not synchronized
+ DirectoryReader refreshed = DirectoryReader.openIfChanged(r, executorService);
+ if (refreshed == null) {
+ refreshed = r;
+ }
+
+ IndexSearcher searcher = newSearcher(refreshed);
+ ScoreDoc[] hits =
+ searcher.search(
+ new TermQuery(
+ new Term("field1", "a" + rnd.nextInt(refreshed.maxDoc()))),
+ 1000)
+ .scoreDocs;
+ if (hits.length > 0) {
+ searcher.storedFields().document(hits[0].doc);
+ }
+ if (refreshed != r) {
+ refreshed.close();
+ }
+ }
+ synchronized (this) {
+ wait(TestUtil.nextInt(random(), 1, 100));
+ }
+ }
+ }
+ };
+ } else {
+ task =
+ new ReaderThreadTask() {
+ @Override
+ public void run() throws Exception {
+ Random rnd = LuceneTestCase.random();
+ while (!stopped) {
+ int numReaders = readers.size();
+ if (numReaders > 0) {
+ ReaderCouple c = readers.get(rnd.nextInt(numReaders));
+ TestDirectoryReader.assertIndexEquals(c.newReader, c.refreshedReader);
+ }
+
+ synchronized (this) {
+ wait(TestUtil.nextInt(random(), 1, 100));
+ }
+ }
+ }
+ };
+ }
+
+ threads[i] = new ReaderThread(task);
+ threads[i].start();
+ }
+
+ synchronized (this) {
+ wait(1000);
+ }
+
+ for (int i = 0; i < n; i++) {
+ if (threads[i] != null) {
+ threads[i].stopThread();
+ }
+ }
+
+ for (int i = 0; i < n; i++) {
+ if (threads[i] != null) {
+ threads[i].join();
+ if (threads[i].error != null) {
+ String msg =
+ "Error occurred in thread "
+ + threads[i].getName()
+ + ":\n"
+ + threads[i].error.getMessage();
+ fail(msg);
+ }
+ }
+ }
+
+ for (final DirectoryReader readerToClose : readersToClose) {
+ readerToClose.close();
+ }
+
+ firstReader.close();
+ reader.close();
+
+ for (final DirectoryReader readerToClose : readersToClose) {
+ assertReaderClosed(readerToClose, true);
+ }
+
+ assertReaderClosed(reader, true);
+ assertReaderClosed(firstReader, true);
+
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testReopenOnCommitWithExecutor() throws Throwable {
+ Directory dir = newDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriter writer =
+ new IndexWriter(
+ dir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setIndexDeletionPolicy(new KeepAllCommits())
+ .setMaxBufferedDocs(-1)
+ .setMergePolicy(newLogMergePolicy(10)));
+ for (int i = 0; i < 4; i++) {
+ Document doc = new Document();
+ doc.add(newStringField("id", "" + i, Field.Store.NO));
+ writer.addDocument(doc);
+ Map<String, String> data = new HashMap<>();
+ data.put("index", i + "");
+ writer.setLiveCommitData(data.entrySet());
+ writer.commit();
+ }
+ for (int i = 0; i < 4; i++) {
+ writer.deleteDocuments(new Term("id", "" + i));
+ Map<String, String> data = new HashMap<>();
+ data.put("index", (4 + i) + "");
+ writer.setLiveCommitData(data.entrySet());
+ writer.commit();
+ }
+ writer.close();
+
+ DirectoryReader r = DirectoryReader.open(dir, executorService);
+ assertEquals(0, r.numDocs());
+
+ Collection<IndexCommit> commits = DirectoryReader.listCommits(dir);
+ for (final IndexCommit commit : commits) {
+ DirectoryReader r2 = DirectoryReader.openIfChanged(r, commit);
+ assertNotNull(r2);
+ assertTrue(r2 != r);
+
+ final Map<String, String> s = commit.getUserData();
+ final int v;
+ if (s.size() == 0) {
+ v = -1;
+ } else {
+ v = Integer.parseInt(s.get("index"));
+ }
+ if (v < 4) {
+ assertEquals(1 + v, r2.numDocs());
+ } else {
+ assertEquals(7 - v, r2.numDocs());
+ }
+ r.close();
+ r = r2;
+ }
+ r.close();
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testOpenIfChangedNRTToCommitWithExecutor() throws Exception {
+ Directory dir = newDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ Document doc = new Document();
+ doc.add(newStringField("field", "value", Field.Store.NO));
+ w.addDocument(doc);
+ w.commit();
+ List<IndexCommit> commits = DirectoryReader.listCommits(dir);
+ assertEquals(1, commits.size());
+ w.addDocument(doc);
+ DirectoryReader r = DirectoryReader.open(w);
+
+ assertEquals(2, r.numDocs());
+ IndexReader r2 = DirectoryReader.openIfChanged(r, commits.get(0), executorService);
+ assertNotNull(r2);
+ r.close();
+ assertEquals(1, r2.numDocs());
+ w.close();
+ r2.close();
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testOverDecRefDuringReopenWithExecutor() throws Exception {
+ MockDirectoryWrapper dir = newMockDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriterConfig iwc =
+ new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
+ iwc.setCodec(TestUtil.getDefaultCodec());
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(newStringField("id", "id", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(newStringField("id", "id2", Field.Store.NO));
+ w.addDocument(doc);
+ w.commit();
+
+ DirectoryReader r = DirectoryReader.open(dir, executorService);
+
+ w.deleteDocuments(new Term("id", "id"));
+ w.commit();
+
+ dir.failOn(
+ new MockDirectoryWrapper.Failure() {
+
+ boolean failed;
+
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ if (failed) {
+ return;
+ }
+ if (callStackContainsAnyOf("readLiveDocs")) {
+ if (VERBOSE) {
+ System.out.println("TEST: now fail; exc:");
+ new Throwable().printStackTrace(System.out);
+ }
+ failed = true;
+ throw new FakeIOException();
+ }
+ }
+ });
+
+ expectThrows(
+ FakeIOException.class,
+ () -> {
+ DirectoryReader.openIfChanged(r, executorService);
+ });
+
+ IndexSearcher s = newSearcher(r);
+ assertEquals(1, s.count(new TermQuery(new Term("id", "id"))));
+
+ r.close();
+ w.close();
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testNPEAfterInvalidReindex1WithExecutor() throws Exception {
+ Directory dir = new ByteBuffersDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE));
+ Document doc = new Document();
+ doc.add(newStringField("id", "id", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(newStringField("id", "id2", Field.Store.NO));
+ w.addDocument(doc);
+ w.deleteDocuments(new Term("id", "id"));
+ w.commit();
+ w.close();
+
+ DirectoryReader r = DirectoryReader.open(dir, executorService);
+
+ for (String fileName : dir.listAll()) {
+ dir.deleteFile(fileName);
+ }
+
+ w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
+ doc = new Document();
+ doc.add(newStringField("id", "id", Field.Store.NO));
+ doc.add(new NumericDocValuesField("ndv", 13));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(newStringField("id", "id2", Field.Store.NO));
+ w.addDocument(doc);
+ w.commit();
+ doc = new Document();
+ doc.add(newStringField("id", "id2", Field.Store.NO));
+ w.addDocument(doc);
+ w.updateNumericDocValue(new Term("id", "id"), "ndv", 17L);
+ w.commit();
+ w.close();
+
+ expectThrows(
+ IllegalStateException.class,
+ () -> {
+ DirectoryReader.openIfChanged(r, executorService);
+ });
+
+ r.close();
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testNPEAfterInvalidReindex2WithExecutor() throws Exception {
+ Directory dir = new ByteBuffersDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE));
+ Document doc = new Document();
+ doc.add(newStringField("id", "id", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(newStringField("id", "id2", Field.Store.NO));
+ w.addDocument(doc);
+ w.deleteDocuments(new Term("id", "id"));
+ w.commit();
+ w.close();
+
+ DirectoryReader r = DirectoryReader.open(dir, executorService);
+
+ for (String name : dir.listAll()) {
+ dir.deleteFile(name);
+ }
+
+ w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
+ doc = new Document();
+ doc.add(newStringField("id", "id", Field.Store.NO));
+ doc.add(new NumericDocValuesField("ndv", 13));
+ w.addDocument(doc);
+ w.commit();
+ doc = new Document();
+ doc.add(newStringField("id", "id2", Field.Store.NO));
+ w.addDocument(doc);
+ w.commit();
+ w.close();
+
+ expectThrows(
+ IllegalStateException.class,
+ () -> {
+ DirectoryReader.openIfChanged(r, executorService);
+ });
+
+ r.close();
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testNRTMdeletesWithExecutor() throws Exception {
+ Directory dir = newDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriterConfig iwc =
+ new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
+ SnapshotDeletionPolicy snapshotter =
+ new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
+ iwc.setIndexDeletionPolicy(snapshotter);
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ writer.commit();
+
+ Document doc = new Document();
+ doc.add(new StringField("key", "value1", Field.Store.YES));
+ writer.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new StringField("key", "value2", Field.Store.YES));
+ writer.addDocument(doc);
+
+ writer.commit();
+
+ IndexCommit ic1 = snapshotter.snapshot();
+
+ doc = new Document();
+ doc.add(new StringField("key", "value3", Field.Store.YES));
+ writer.updateDocument(new Term("key", "value1"), doc);
+
+ writer.commit();
+
+ IndexCommit ic2 = snapshotter.snapshot();
+ DirectoryReader latest = DirectoryReader.open(ic2, executorService);
+ assertEquals(2, latest.leaves().size());
+
+ DirectoryReader oldest = DirectoryReader.openIfChanged(latest, ic1, executorService);
+ assertEquals(1, oldest.leaves().size());
+
+ assertSame(
+ latest.leaves().get(0).reader().getCoreCacheHelper().getKey(),
+ oldest.leaves().get(0).reader().getCoreCacheHelper().getKey());
+
+ latest.close();
+ oldest.close();
+
+ snapshotter.release(ic1);
+ snapshotter.release(ic2);
+ writer.close();
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testNRTMdeletes2WithExecutor() throws Exception {
+ Directory dir = newDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriterConfig iwc =
+ new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
+ SnapshotDeletionPolicy snapshotter =
+ new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
+ iwc.setIndexDeletionPolicy(snapshotter);
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ writer.commit();
+
+ Document doc = new Document();
+ doc.add(new StringField("key", "value1", Field.Store.YES));
+ writer.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new StringField("key", "value2", Field.Store.YES));
+ writer.addDocument(doc);
+
+ writer.commit();
+
+ IndexCommit ic1 = snapshotter.snapshot();
+
+ doc = new Document();
+ doc.add(new StringField("key", "value3", Field.Store.YES));
+ writer.updateDocument(new Term("key", "value1"), doc);
+
+ DirectoryReader latest = DirectoryReader.open(writer);
+ assertEquals(2, latest.leaves().size());
+
+ DirectoryReader oldest = DirectoryReader.openIfChanged(latest, ic1, executorService);
+
+ assertEquals(2, oldest.numDocs());
+ assertFalse(oldest.hasDeletions());
+
+ snapshotter.release(ic1);
+ assertEquals(1, oldest.leaves().size());
+
+ assertSame(
+ latest.leaves().get(0).reader().getCoreCacheHelper().getKey(),
+ oldest.leaves().get(0).reader().getCoreCacheHelper().getKey());
+
+ latest.close();
+ oldest.close();
+ writer.close();
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testNRTMupdatesWithExecutor() throws Exception {
+ Directory dir = newDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SnapshotDeletionPolicy snapshotter =
+ new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
+ iwc.setIndexDeletionPolicy(snapshotter);
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ writer.commit();
+
+ Document doc = new Document();
+ doc.add(new StringField("key", "value1", Field.Store.YES));
+ doc.add(new NumericDocValuesField("dv", 1));
+ writer.addDocument(doc);
+
+ writer.commit();
+
+ IndexCommit ic1 = snapshotter.snapshot();
+
+ writer.updateNumericDocValue(new Term("key", "value1"), "dv", 2);
+
+ writer.commit();
+
+ IndexCommit ic2 = snapshotter.snapshot();
+ DirectoryReader latest = DirectoryReader.open(ic2, executorService);
+ assertEquals(1, latest.leaves().size());
+
+ DirectoryReader oldest = DirectoryReader.openIfChanged(latest, ic1, executorService);
+ assertEquals(1, oldest.leaves().size());
+
+ assertSame(
+ latest.leaves().get(0).reader().getCoreCacheHelper().getKey(),
+ oldest.leaves().get(0).reader().getCoreCacheHelper().getKey());
+
+ NumericDocValues values = getOnlyLeafReader(oldest).getNumericDocValues("dv");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+
+ values = getOnlyLeafReader(latest).getNumericDocValues("dv");
+ assertEquals(0, values.nextDoc());
+ assertEquals(2, values.longValue());
+
+ latest.close();
+ oldest.close();
+
+ snapshotter.release(ic1);
+ snapshotter.release(ic2);
+ writer.close();
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testNRTMupdates2WithExecutor() throws Exception {
+ Directory dir = newDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SnapshotDeletionPolicy snapshotter =
+ new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
+ iwc.setIndexDeletionPolicy(snapshotter);
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ writer.commit();
+
+ Document doc = new Document();
+ doc.add(new StringField("key", "value1", Field.Store.YES));
+ doc.add(new NumericDocValuesField("dv", 1));
+ writer.addDocument(doc);
+
+ writer.commit();
+
+ IndexCommit ic1 = snapshotter.snapshot();
+
+ writer.updateNumericDocValue(new Term("key", "value1"), "dv", 2);
+
+ DirectoryReader latest = DirectoryReader.open(writer);
+ assertEquals(1, latest.leaves().size());
+
+ DirectoryReader oldest = DirectoryReader.openIfChanged(latest, ic1, executorService);
+ assertEquals(1, oldest.leaves().size());
+
+ assertSame(
+ latest.leaves().get(0).reader().getCoreCacheHelper().getKey(),
+ oldest.leaves().get(0).reader().getCoreCacheHelper().getKey());
+
+ NumericDocValues values = getOnlyLeafReader(oldest).getNumericDocValues("dv");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+
+ values = getOnlyLeafReader(latest).getNumericDocValues("dv");
+ assertEquals(0, values.nextDoc());
+ assertEquals(2, values.longValue());
+
+ latest.close();
+ oldest.close();
+
+ snapshotter.release(ic1);
+ writer.close();
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testDeleteIndexFilesWhileReaderStillOpenWithExecutor() throws Exception {
+ Directory dir = new ByteBuffersDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
+ Document doc = new Document();
+ doc.add(newStringField("field", "value", Field.Store.NO));
+ w.addDocument(doc);
+ w.close();
+
+ DirectoryReader r = DirectoryReader.open(dir, executorService);
+
+ for (String file : dir.listAll()) {
+ dir.deleteFile(file);
+ }
+
+ w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE));
+ doc = new Document();
+ doc.add(newStringField("field", "value", Field.Store.NO));
+ w.addDocument(doc);
+
+ doc = new Document();
+ doc.add(newStringField("field", "value2", Field.Store.NO));
+ w.addDocument(doc);
+
+ w.commit();
+
+ w.deleteDocuments(new Term("field", "value2"));
+
+ w.addDocument(doc);
+
+ w.close();
+
+ expectThrows(
+ IllegalStateException.class,
+ () -> {
+ DirectoryReader.openIfChanged(r, executorService);
+ });
+
+ r.close();
+ dir.close();
+ } finally {
+ executorService.shutdown();
+ }
+ }
+
+ public void testReuseUnchangedLeafReaderOnDVUpdateWithExecutor() throws IOException {
+ Directory dir = newDirectory();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(4, new NamedThreadFactory("TestDirectoryReaderReopen"));
+ try {
+ IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
+ indexWriterConfig.setMergePolicy(NoMergePolicy.INSTANCE);
+ IndexWriter writer = new IndexWriter(dir, indexWriterConfig);
+
+ Document doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ doc.add(new StringField("version", "1", Field.Store.YES));
+ doc.add(new NumericDocValuesField("some_docvalue", 2));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "2", Field.Store.YES));
+ doc.add(new StringField("version", "1", Field.Store.YES));
+ writer.addDocument(doc);
+ writer.commit();
+ DirectoryReader reader = DirectoryReader.open(dir, executorService);
+ assertEquals(2, reader.numDocs());
+ assertEquals(2, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ doc.add(new StringField("version", "2", Field.Store.YES));
+ writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("some_docvalue", 1));
+ writer.commit();
+ DirectoryReader newReader = DirectoryReader.openIfChanged(reader, executorService);
+ assertNotSame(newReader, reader);
+ reader.close();
+ reader = newReader;
+ assertEquals(2, reader.numDocs());
+ assertEquals(2, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+
+ doc = new Document();
+ doc.add(new StringField("id", "3", Field.Store.YES));
+ doc.add(new StringField("version", "3", Field.Store.YES));
+ writer.updateDocument(new Term("id", "3"), doc);
+ writer.commit();
+
+ newReader = DirectoryReader.openIfChanged(reader, executorService);
+ assertNotSame(newReader, reader);
+ assertEquals(2, newReader.getSequentialSubReaders().size());
+ assertEquals(1, reader.getSequentialSubReaders().size());
+ assertSame(
+ reader.getSequentialSubReaders().get(0), newReader.getSequentialSubReaders().get(0));
+ reader.close();
+ reader = newReader;
+ assertEquals(3, reader.numDocs());
+ assertEquals(3, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+ IOUtils.close(reader, writer, dir);
+ } finally {
+ executorService.shutdown();
+ }
+ }
}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/SegmentInfosSearcherManager.java b/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/SegmentInfosSearcherManager.java
index 9fcac51c912..2fd3e7d72eb 100644
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/SegmentInfosSearcherManager.java
+++ b/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/SegmentInfosSearcherManager.java
@@ -60,7 +60,9 @@ class SegmentInfosSearcherManager extends ReferenceManager<IndexSearcher> {
node.message("SegmentInfosSearcherManager.init: use incoming infos=" + infosIn.toString());
current =
SearcherManager.getSearcher(
- searcherFactory, StandardDirectoryReader.open(dir, currentInfos, null, null), null);
+ searcherFactory,
+ StandardDirectoryReader.open(dir, currentInfos, null, null, null),
+ null);
addReaderClosedListener(current.getIndexReader());
}
@@ -111,7 +113,7 @@ class SegmentInfosSearcherManager extends ReferenceManager<IndexSearcher> {
}
// Open a new reader, sharing any common segment readers with the old one:
- DirectoryReader r = StandardDirectoryReader.open(dir, currentInfos, subs, null);
+ DirectoryReader r = StandardDirectoryReader.open(dir, currentInfos, subs, null, null);
addReaderClosedListener(r);
node.message("refreshed to version=" + currentInfos.getVersion() + " r=" + r);
return SearcherManager.getSearcher(searcherFactory, r, old.getIndexReader());