Skip to content

Apache Lucene open-source search software

Honor read advice on compound files. (#13467)

Adrien Grandcfbf8d93df81
diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
index 881f707d07a..ee84d908838 100644
--- a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
@@ -18,6 +18,7 @@ package org.apache.lucene.store;
import java.io.Closeable;
import java.io.IOException;
+import org.apache.lucene.codecs.CompoundFormat;
/**
* Abstract base class for input from a file in a {@link Directory}. A random-access input stream.
@@ -121,6 +122,19 @@ public abstract class IndexInput extends DataInput implements Closeable {
public abstract IndexInput slice(String sliceDescription, long offset, long length)
throws IOException;
+ /**
+ * Create a slice with a specific {@link ReadAdvice}. This is typically used by {@link
+ * CompoundFormat} implementations to honor the {@link ReadAdvice} of each file within the
+ * compound file.
+ *
+ * <p>The default implementation delegates to {@link #slice(String, long, long)} and ignores the
+ * {@link ReadAdvice}.
+ */
+ public IndexInput slice(String sliceDescription, long offset, long length, ReadAdvice readAdvice)
+ throws IOException {
+ return slice(sliceDescription, offset, length);
+ }
+
/**
* Subclasses call this to get the String for resourceDescription of a slice of this {@code
* IndexInput}.
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
index b689088c304..68f1e771195 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
@@ -28,6 +28,7 @@ import java.util.Optional;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.GroupVIntUtil;
+import org.apache.lucene.util.IOConsumer;
/**
* Base IndexInput implementation that uses an array of MemorySegments to represent a file.
@@ -333,14 +334,36 @@ abstract class MemorySegmentIndexInput extends IndexInput
return;
}
+ final NativeAccess nativeAccess = NATIVE_ACCESS.get();
+ advise(
+ offset,
+ length,
+ segment -> {
+ if (segment.isLoaded() == false) {
+ // We have a cache miss on at least one page, let's reset the counter.
+ consecutivePrefetchHitCount = 0;
+ nativeAccess.madviseWillNeed(segment);
+ }
+ });
+ }
+
+ void advise(long offset, long length, IOConsumer<MemorySegment> advice) throws IOException {
+ if (NATIVE_ACCESS.isEmpty()) {
+ return;
+ }
+
+ ensureOpen();
+
+ Objects.checkFromIndexSize(offset, length, length());
+
final NativeAccess nativeAccess = NATIVE_ACCESS.get();
try {
final MemorySegment segment = segments[(int) (offset >> chunkSizePower)];
offset &= chunkSizeMask;
- // Compute the intersection of the current segment and the region that should be prefetched.
+ // Compute the intersection of the current segment and the region that should be advised.
if (offset + length > segment.byteSize()) {
- // Only prefetch bytes that are stored in the current segment. There may be bytes on the
+ // Only advise bytes that are stored in the current segment. There may be bytes on the
// next segment but this case is rare enough that we don't try to optimize it and keep
// things simple instead.
length = segment.byteSize() - offset;
@@ -351,16 +374,17 @@ abstract class MemorySegmentIndexInput extends IndexInput
offset -= offsetInPage;
length += offsetInPage;
if (offset < 0) {
- // The start of the page is outside of this segment, ignore.
- return;
+ // The start of the page is before the start of this segment, ignore the first page.
+ offset += nativeAccess.getPageSize();
+ length -= nativeAccess.getPageSize();
+ if (length <= 0) {
+ // This segment has no data beyond the first page.
+ return;
+ }
}
- final MemorySegment prefetchSlice = segment.asSlice(offset, length);
- if (prefetchSlice.isLoaded() == false) {
- // We have a cache miss on at least one page, let's reset the counter.
- consecutivePrefetchHitCount = 0;
- nativeAccess.madviseWillNeed(prefetchSlice);
- }
+ final MemorySegment advisedSlice = segment.asSlice(offset, length);
+ advice.accept(advisedSlice);
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
@@ -527,6 +551,22 @@ abstract class MemorySegmentIndexInput extends IndexInput
return buildSlice(sliceDescription, offset, length);
}
+ @Override
+ public final MemorySegmentIndexInput slice(
+ String sliceDescription, long offset, long length, ReadAdvice advice) throws IOException {
+ MemorySegmentIndexInput slice = slice(sliceDescription, offset, length);
+ if (NATIVE_ACCESS.isPresent()) {
+ final NativeAccess nativeAccess = NATIVE_ACCESS.get();
+ slice.advise(
+ 0,
+ slice.length,
+ segment -> {
+ nativeAccess.madvise(segment, advice);
+ });
+ }
+ return slice;
+ }
+
/** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */
MemorySegmentIndexInput buildSlice(String sliceDescription, long offset, long length) {
ensureOpen();
diff --git a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java b/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java
index 93caca788b1..80c1665cdd1 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java
+++ b/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java
@@ -158,10 +158,10 @@ final class PosixNativeAccess extends NativeAccess {
private Integer mapReadAdvice(ReadAdvice readAdvice) {
return switch (readAdvice) {
- case NORMAL -> null;
+ case NORMAL -> POSIX_MADV_NORMAL;
case RANDOM -> POSIX_MADV_RANDOM;
case SEQUENTIAL -> POSIX_MADV_SEQUENTIAL;
- case RANDOM_PRELOAD -> null;
+ case RANDOM_PRELOAD -> POSIX_MADV_NORMAL;
};
}