Skip to content

Apache Lucene open-source search software

Merges read compound files through a mapping of their own (#16796)

Jim Ferenczif94888d85c12
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 448bf39d782..23f24c75d2f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -627,6 +627,9 @@ Optimizations
pool instead of serially on the merge thread, so reusing a graph with deletes is no longer
slower than a full rebuild. (Ethan Baker)
+* GITHUB#16796: Merges read inner files of compound files through a mapping of their own, so the
+ access they advise does not replace the advice of searches. (Jim Ferenczi)
+
Bug Fixes
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90CompoundReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90CompoundReader.java
index 95278296571..b33d6231caa 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90CompoundReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90CompoundReader.java
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.lucene90;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.NoSuchFileException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -27,6 +28,7 @@ import org.apache.lucene.codecs.CompoundDirectory;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -49,8 +51,12 @@ final class Lucene90CompoundReader extends CompoundDirectory {
private final Directory directory;
private final String segmentName;
+ private final String dataFileName;
private final Map<String, FileEntry> entries;
private final IndexInput handle;
+ // the mapping merges read through, opened by the first merge open and closed with this reader
+ private IndexInput mergeHandle;
+ private boolean closed;
private int version;
/** Create a new CompoundFileDirectory. */
@@ -59,7 +65,7 @@ final class Lucene90CompoundReader extends CompoundDirectory {
public Lucene90CompoundReader(Directory directory, SegmentInfo si) throws IOException {
this.directory = directory;
this.segmentName = si.name;
- String dataFileName =
+ this.dataFileName =
IndexFileNames.segmentFileName(segmentName, "", Lucene90CompoundFormat.DATA_EXTENSION);
String entriesFileName =
IndexFileNames.segmentFileName(segmentName, "", Lucene90CompoundFormat.ENTRIES_EXTENSION);
@@ -142,7 +148,13 @@ final class Lucene90CompoundReader extends CompoundDirectory {
@Override
public void close() throws IOException {
- IOUtils.close(handle);
+ IndexInput merge;
+ synchronized (this) {
+ closed = true;
+ merge = mergeHandle == handle ? null : mergeHandle;
+ mergeHandle = null;
+ }
+ IOUtils.close(handle, merge);
}
@Override
@@ -164,7 +176,28 @@ final class Lucene90CompoundReader extends CompoundDirectory {
+ entries.keySet()
+ ")");
}
- return handle.slice(name, entry.offset, entry.length, context);
+ IndexInput data = context.context() == IOContext.Context.MERGE ? mergeHandle() : handle;
+ return data.slice(name, entry.offset, entry.length, context);
+ }
+
+ /**
+ * A merge reads through a mapping of its own, so the access it advises does not replace the
+ * advice of searches reading the same inner files. Merges of this segment share it. Like the
+ * search mapping, it is opened without advice: each inner file is advised on its own region.
+ */
+ private synchronized IndexInput mergeHandle() throws IOException {
+ if (closed) {
+ throw new AlreadyClosedException("this Directory is closed");
+ }
+ if (mergeHandle == null) {
+ try {
+ mergeHandle = directory.openInput(dataFileName, IOContext.DEFAULT);
+ } catch (@SuppressWarnings("unused") FileNotFoundException | NoSuchFileException e) {
+ // an open reader outlives its files, so fall back to the mapping it already holds
+ mergeHandle = handle;
+ }
+ }
+ return mergeHandle;
}
/** Returns an array of strings, one for each file in the directory. */
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90CompoundFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90CompoundFormat.java
index 8d6ee823c79..11ae5a8c198 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90CompoundFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90CompoundFormat.java
@@ -17,16 +17,25 @@
package org.apache.lucene.codecs.lucene90;
import java.io.IOException;
+import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.CountDownLatch;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.store.MemorySegmentAccessInput;
import org.apache.lucene.tests.index.BaseCompoundFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
@@ -93,4 +102,210 @@ public class TestLucene90CompoundFormat extends BaseCompoundFormatTestCase {
}
dir.close();
}
+
+ public void testMergesReadThroughAMappingOfTheirOwn() throws IOException {
+ try (Directory base = newDirectory()) {
+ SegmentInfo si = writeCompound(base);
+ DataOpens dir = new DataOpens(base);
+ Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
+ try {
+ assertEquals(1, dir.opens.size());
+ String first = "_123.0";
+ String second = "_123.1";
+ try (IndexInput search = cfs.openInput(first, IOContext.DEFAULT)) {
+ assertEquals("searches read the mapping opened with the reader", 1, dir.opens.size());
+
+ IndexInput merge = cfs.openInput(first, IOContext.merge());
+ IndexInput otherMerge = cfs.openInput(second, IOContext.merge());
+ assertEquals("merges share one mapping of their own", 2, dir.opens.size());
+ assertEquals(
+ "opened without advice, like the search mapping",
+ IOContext.DEFAULT,
+ dir.opens.get(1));
+ try (IndexInput expected = base.openInput(first, IOContext.DEFAULT)) {
+ assertSameStreams(first, expected, merge);
+ }
+ try (IndexInput expected = base.openInput(first, IOContext.DEFAULT)) {
+ assertSameStreams(first, expected, search);
+ }
+ try (IndexInput expected = base.openInput(second, IOContext.DEFAULT)) {
+ assertSameStreams(second, expected, otherMerge);
+ }
+ merge.close();
+ otherMerge.close();
+ assertEquals("the merge mapping stays open with the reader", 0, dir.closes);
+ }
+ } finally {
+ cfs.close();
+ }
+ assertEquals("closing the reader closes both mappings", 2, dir.closes);
+ }
+ }
+
+ public void testMergesFallBackToTheReaderMappingWhenTheFileIsGone() throws IOException {
+ try (Directory base = newDirectory()) {
+ SegmentInfo si = writeCompound(base);
+ DataOpens dir = new DataOpens(base);
+ try (Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si)) {
+ dir.gone = true;
+ String file = "_123.0";
+ try (IndexInput merge = cfs.openInput(file, IOContext.merge());
+ IndexInput expected = base.openInput(file, IOContext.DEFAULT)) {
+ assertSameStreams(file, expected, merge);
+ }
+ assertEquals(1, dir.opens.size());
+ }
+ assertEquals(1, dir.closes);
+ }
+ }
+
+ /** Searches share the mapping opened with the reader; a merge reads one of its own. */
+ public void testAMergeDoesNotReadTheMappingSearchesRead() throws IOException {
+ try (MMapDirectory dir = new MMapDirectory(createTempDir("cfsMerge"))) {
+ dir.setReadAdvice(MMapDirectory.ADVISE_BY_CONTEXT);
+ SegmentInfo si = writeCompound(dir);
+ try (Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
+ IndexInput search = cfs.openInput("_123.0", IOContext.DEFAULT);
+ IndexInput randomSearch =
+ cfs.openInput("_123.0", IOContext.DEFAULT.withHints(DataAccessHint.RANDOM));
+ IndexInput merge =
+ cfs.openInput("_123.0", IOContext.merge().withHints(DataAccessHint.SEQUENTIAL))) {
+ assertEquals(address(search), address(randomSearch));
+ assertNotEquals(address(search), address(merge));
+ try (IndexInput expected = dir.openInput("_123.0", IOContext.DEFAULT)) {
+ assertSameStreams("_123.0", expected, merge);
+ }
+ }
+ }
+ }
+
+ private static long address(IndexInput in) throws IOException {
+ return ((MemorySegmentAccessInput) in).segmentSliceOrNull(0, 1).address();
+ }
+
+ public void testNoMergeMappingOnceClosed() throws IOException {
+ try (Directory base = newDirectory()) {
+ SegmentInfo si = writeCompound(base);
+ DataOpens dir = new DataOpens(base);
+ Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
+ cfs.close();
+ expectThrows(AlreadyClosedException.class, () -> cfs.openInput("_123.0", IOContext.merge()));
+ assertEquals(1, dir.opens.size());
+ assertEquals(1, dir.closes);
+ }
+ }
+
+ public void testAFailedMergeOpenKeepsNothingOpen() throws IOException {
+ try (Directory base = newDirectory()) {
+ SegmentInfo si = writeCompound(base);
+ DataOpens dir = new DataOpens(base);
+ try (Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si)) {
+ dir.failNextOpen = true;
+ expectThrows(IOException.class, () -> cfs.openInput("_123.0", IOContext.merge()));
+ assertEquals(1, dir.opens.size());
+ try (IndexInput merge = cfs.openInput("_123.0", IOContext.merge());
+ IndexInput expected = base.openInput("_123.0", IOContext.DEFAULT)) {
+ assertSameStreams("_123.0", expected, merge);
+ }
+ assertEquals("the next merge maps the file", 2, dir.opens.size());
+ }
+ assertEquals(2, dir.closes);
+ }
+ }
+
+ public void testConcurrentMergeOpensMapOnce() throws Exception {
+ try (Directory base = newDirectory()) {
+ SegmentInfo si = writeCompound(base);
+ DataOpens dir = new DataOpens(base);
+ try (Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si)) {
+ int threadCount = 2 + random().nextInt(4);
+ CountDownLatch start = new CountDownLatch(1);
+ List<Thread> threads = new ArrayList<>();
+ List<Throwable> failures = Collections.synchronizedList(new ArrayList<>());
+ for (int t = 0; t < threadCount; t++) {
+ String file = "_123." + (t % 2);
+ Thread thread =
+ new Thread(
+ () -> {
+ try {
+ start.await();
+ try (IndexInput in = cfs.openInput(file, IOContext.merge())) {
+ in.seek(in.length());
+ }
+ } catch (Throwable e) {
+ failures.add(e);
+ }
+ });
+ thread.start();
+ threads.add(thread);
+ }
+ start.countDown();
+ for (Thread thread : threads) {
+ thread.join();
+ }
+ assertEquals(List.of(), failures);
+ assertEquals(2, dir.opens.size());
+ }
+ assertEquals(2, dir.closes);
+ }
+ }
+
+ private static SegmentInfo writeCompound(Directory dir) throws IOException {
+ SegmentInfo si = newSegmentInfo(dir, "_123");
+ List<String> files = new ArrayList<>();
+ for (int i = 0; i < 2; i++) {
+ String name = "_123." + i;
+ createRandomFile(dir, name, random().nextInt(1, 4096), si.getId());
+ files.add(name);
+ }
+ si.setFiles(files);
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
+ return si;
+ }
+
+ /** Records the contexts the compound data file is opened with, and counts closes. */
+ private static class DataOpens extends FilterDirectory {
+ final List<IOContext> opens = Collections.synchronizedList(new ArrayList<>());
+ volatile int closes;
+ volatile boolean gone;
+ volatile boolean failNextOpen;
+
+ DataOpens(Directory in) {
+ super(in);
+ }
+
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ if (name.endsWith("." + Lucene90CompoundFormat.DATA_EXTENSION) == false) {
+ return super.openInput(name, context);
+ }
+ if (gone && opens.isEmpty() == false) {
+ throw new NoSuchFileException(name);
+ }
+ if (failNextOpen) {
+ failNextOpen = false;
+ throw new IOException("simulated failure opening " + name);
+ }
+ opens.add(context);
+ return new FilterIndexInput(name, super.openInput(name, context)) {
+ @Override
+ public void close() throws IOException {
+ synchronized (DataOpens.this) {
+ closes++;
+ }
+ super.close();
+ }
+
+ @Override
+ public IndexInput clone() {
+ return in.clone();
+ }
+
+ @Override
+ public IndexInput slice(String description, long offset, long length) throws IOException {
+ return in.slice(description, offset, length);
+ }
+ };
+ }
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorsMergeReadAdvice.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorsMergeReadAdvice.java
index d501ac47023..5e1b737d1d2 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorsMergeReadAdvice.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorsMergeReadAdvice.java
@@ -114,6 +114,62 @@ public class TestVectorsMergeReadAdvice extends LuceneTestCase {
}
}
+ /**
+ * In a compound segment, a merge reads the vectors through a mapping of the compound file of its
+ * own, opened without advice and closed with the segment's reader.
+ */
+ public void testACompoundMergeOpensItsOwnMapping() throws Exception {
+ KnnVectorsFormat format =
+ random().nextBoolean()
+ ? new Lucene99HnswVectorsFormat()
+ : new Lucene104HnswScalarQuantizedVectorsFormat();
+ Opens opens = new Opens();
+ List<Open> mergeOpens;
+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setCodec(TestUtil.alwaysKnnVectorsFormat(format));
+ iwc.setUseCompoundFile(true);
+ try (IndexWriter w = new IndexWriter(dir, iwc)) {
+ for (int segment = 0; segment < 2; segment++) {
+ for (int i = 0; i < 64; i++) {
+ Document doc = new Document();
+ doc.add(
+ new KnnFloatVectorField("field", vector(), VectorSimilarityFunction.DOT_PRODUCT));
+ w.addDocument(doc);
+ }
+ w.commit();
+ }
+
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ List<String> compoundFiles = new ArrayList<>();
+ for (var leaf : reader.leaves()) {
+ SegmentReader segmentReader = (SegmentReader) leaf.reader();
+ assertTrue(segmentReader.getSegmentInfo().info.getUseCompoundFile());
+ compoundFiles.add(segmentReader.getSegmentName() + ".cfs");
+ }
+ compoundFiles.sort(null);
+ opens.clear();
+ w.forceMerge(1);
+
+ // the segments were open before the merge, so opening their compound files again is the
+ // merge
+ mergeOpens = opens.all().stream().filter(o -> compoundFiles.contains(o.name())).toList();
+ assertEquals(
+ "the merge mapped each compound file once: " + opens,
+ compoundFiles,
+ mergeOpens.stream().map(Open::name).sorted().toList());
+ for (Open open : mergeOpens) {
+ assertTrue("the merge did not read its mapping: " + open, open.read());
+ assertEquals("opened without advice: " + open, Set.of(), open.hints());
+ }
+ }
+ }
+ }
+ for (Open open : mergeOpens) {
+ assertTrue("closed with the segment's reader: " + open, open.closed());
+ }
+ }
+
/** A reader a merge opened already reads the vectors the way a merge does. */
public void testAMergeReadsThroughTheReaderItOpened() throws Exception {
Opens opens = new Opens();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseCompoundFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseCompoundFormatTestCase.java
index 7e1792f5d55..55344ad063d 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseCompoundFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseCompoundFormatTestCase.java
@@ -387,8 +387,8 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
dir.close();
}
- // Make sure we don't somehow use more than 1 descriptor
- // when reading a CFS with many subs:
+ // Make sure we don't somehow use more than 1 descriptor when reading a CFS with many subs, plus
+ // 1 for the mapping merges may read through:
public void testManySubFiles() throws IOException {
final MockDirectoryWrapper dir = newMockFSDirectory(createTempDir("CFSManySubFiles"));
@@ -413,18 +413,24 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
final IndexInput[] ins = new IndexInput[FILE_COUNT];
+ int maxHandles = 1;
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
- ins[fileIdx] = cfs.openInput("_123." + fileIdx, newIOContext(random()));
+ IOContext context = newIOContext(random());
+ if (context.context() == IOContext.Context.MERGE) {
+ maxHandles = 2;
+ }
+ ins[fileIdx] = cfs.openInput("_123." + fileIdx, context);
CodecUtil.checkIndexHeader(ins[fileIdx], "Foo", 0, 0, si.getId(), "suffix");
}
- assertEquals(1, dir.getFileHandleCount());
+ long handles = dir.getFileHandleCount();
+ assertTrue("handles: " + handles, handles >= 1 && handles <= maxHandles);
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
assertEquals((byte) fileIdx, ins[fileIdx].readByte());
}
- assertEquals(1, dir.getFileHandleCount());
+ assertEquals(handles, dir.getFileHandleCount());
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
ins[fileIdx].close();