Apache Lucene open-source search software
Honor read advice on compound files. (#13467)
Adrien Grandcfbf8d93df81
diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java b/lucene/core/src/java/org/apache/lucene/store/IndexInput.javaindex 881f707d07a..ee84d908838 100644--- a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java+++ b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java@@ -18,6 +18,7 @@ package org.apache.lucene.store;import java.io.Closeable;import java.io.IOException;+import org.apache.lucene.codecs.CompoundFormat;/*** Abstract base class for input from a file in a {@link Directory}. A random-access input stream.@@ -121,6 +122,19 @@ public abstract class IndexInput extends DataInput implements Closeable {public abstract IndexInput slice(String sliceDescription, long offset, long length)throws IOException;+ /**+ * Create a slice with a specific {@link ReadAdvice}. This is typically used by {@link+ * CompoundFormat} implementations to honor the {@link ReadAdvice} of each file within the+ * compound file.+ *+ * <p>The default implementation delegates to {@link #slice(String, long, long)} and ignores the+ * {@link ReadAdvice}.+ */+ public IndexInput slice(String sliceDescription, long offset, long length, ReadAdvice readAdvice)+ throws IOException {+ return slice(sliceDescription, offset, length);+ }+/*** Subclasses call this to get the String for resourceDescription of a slice of this {@code* IndexInput}.diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.javaindex b689088c304..68f1e771195 100644--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java@@ -28,6 +28,7 @@ import java.util.Optional;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BitUtil;import org.apache.lucene.util.GroupVIntUtil;+import org.apache.lucene.util.IOConsumer;/*** Base IndexInput implementation that uses an array of MemorySegments to represent a file.@@ -333,14 +334,36 @@ abstract class MemorySegmentIndexInput extends IndexInputreturn;}+ final NativeAccess nativeAccess = NATIVE_ACCESS.get();+ advise(+ offset,+ length,+ segment -> {+ if (segment.isLoaded() == false) {+ // We have a cache miss on at least one page, let's reset the counter.+ consecutivePrefetchHitCount = 0;+ nativeAccess.madviseWillNeed(segment);+ }+ });+ }++ void advise(long offset, long length, IOConsumer<MemorySegment> advice) throws IOException {+ if (NATIVE_ACCESS.isEmpty()) {+ return;+ }++ ensureOpen();++ Objects.checkFromIndexSize(offset, length, length());+final NativeAccess nativeAccess = NATIVE_ACCESS.get();try {final MemorySegment segment = segments[(int) (offset >> chunkSizePower)];offset &= chunkSizeMask;- // Compute the intersection of the current segment and the region that should be prefetched.+ // Compute the intersection of the current segment and the region that should be advised.if (offset + length > segment.byteSize()) {- // Only prefetch bytes that are stored in the current segment. There may be bytes on the+ // Only advise bytes that are stored in the current segment. There may be bytes on the// next segment but this case is rare enough that we don't try to optimize it and keep// things simple instead.length = segment.byteSize() - offset;@@ -351,16 +374,17 @@ abstract class MemorySegmentIndexInput extends IndexInputoffset -= offsetInPage;length += offsetInPage;if (offset < 0) {- // The start of the page is outside of this segment, ignore.- return;+ // The start of the page is before the start of this segment, ignore the first page.+ offset += nativeAccess.getPageSize();+ length -= nativeAccess.getPageSize();+ if (length <= 0) {+ // This segment has no data beyond the first page.+ return;+ }}- final MemorySegment prefetchSlice = segment.asSlice(offset, length);- if (prefetchSlice.isLoaded() == false) {- // We have a cache miss on at least one page, let's reset the counter.- consecutivePrefetchHitCount = 0;- nativeAccess.madviseWillNeed(prefetchSlice);- }+ final MemorySegment advisedSlice = segment.asSlice(offset, length);+ advice.accept(advisedSlice);} catch (@SuppressWarnings("unused")IndexOutOfBoundsException e) {@@ -527,6 +551,22 @@ abstract class MemorySegmentIndexInput extends IndexInputreturn buildSlice(sliceDescription, offset, length);}+ @Override+ public final MemorySegmentIndexInput slice(+ String sliceDescription, long offset, long length, ReadAdvice advice) throws IOException {+ MemorySegmentIndexInput slice = slice(sliceDescription, offset, length);+ if (NATIVE_ACCESS.isPresent()) {+ final NativeAccess nativeAccess = NATIVE_ACCESS.get();+ slice.advise(+ 0,+ slice.length,+ segment -> {+ nativeAccess.madvise(segment, advice);+ });+ }+ return slice;+ }+/** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */MemorySegmentIndexInput buildSlice(String sliceDescription, long offset, long length) {ensureOpen();diff --git a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java b/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.javaindex 93caca788b1..80c1665cdd1 100644--- a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java+++ b/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java@@ -158,10 +158,10 @@ final class PosixNativeAccess extends NativeAccess {private Integer mapReadAdvice(ReadAdvice readAdvice) {return switch (readAdvice) {- case NORMAL -> null;+ case NORMAL -> POSIX_MADV_NORMAL;case RANDOM -> POSIX_MADV_RANDOM;case SEQUENTIAL -> POSIX_MADV_SEQUENTIAL;- case RANDOM_PRELOAD -> null;+ case RANDOM_PRELOAD -> POSIX_MADV_NORMAL;};}