Skip to content

Apache Lucene open-source search software

Make `IndexInput#prefetch` take an offset. (#13363)

Adrien Grand46f1f95ceb03
diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
index ec7a1294d40..51d2131df7b 100644
--- a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
@@ -199,7 +199,8 @@ public abstract class IndexInput extends DataInput implements Closeable {
*
* <p>The default implementation is a no-op.
*
+ * @param offset start offset
* @param length the number of bytes to prefetch
*/
- public void prefetch(long length) throws IOException {}
+ public void prefetch(long offset, long length) throws IOException {}
}
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
index a8b8d6da3cd..a0e7e60e99e 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
@@ -313,30 +313,29 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAcces
}
@Override
- public void prefetch(long length) throws IOException {
+ public void prefetch(long offset, long length) throws IOException {
ensureOpen();
- Objects.checkFromIndexSize(getFilePointer(), length, length());
+ Objects.checkFromIndexSize(offset, length, length());
if (NATIVE_ACCESS.isEmpty()) {
return;
}
final NativeAccess nativeAccess = NATIVE_ACCESS.get();
- // If at the boundary between two chunks, move to the next one.
- seek(getFilePointer());
try {
+ final MemorySegment segment = segments[(int) (offset >> chunkSizePower)];
+ offset &= chunkSizeMask;
// Compute the intersection of the current segment and the region that should be prefetched.
- long offset = curPosition;
- if (offset + length > curSegment.byteSize()) {
+ if (offset + length > segment.byteSize()) {
// Only prefetch bytes that are stored in the current segment. There may be bytes on the
// next segment but this case is rare enough that we don't try to optimize it and keep
// things simple instead.
- length = curSegment.byteSize() - curPosition;
+ length = segment.byteSize() - offset;
}
// Now align offset with the page size, this is required for madvise.
// Compute the offset of the current position in the OS's page.
- final long offsetInPage = (curSegment.address() + offset) % nativeAccess.getPageSize();
+ final long offsetInPage = (segment.address() + offset) % nativeAccess.getPageSize();
offset -= offsetInPage;
length += offsetInPage;
if (offset < 0) {
@@ -344,7 +343,7 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAcces
return;
}
- final MemorySegment prefetchSlice = curSegment.asSlice(offset, length);
+ final MemorySegment prefetchSlice = segment.asSlice(offset, length);
nativeAccess.madviseWillNeed(prefetchSlice);
} catch (
@SuppressWarnings("unused")
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
index 775c6d5d318..bb252982c25 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
@@ -1540,36 +1540,32 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
in = orig.slice("slice", startOffset, totalLength - startOffset);
}
for (int i = 0; i < 10_000; ++i) {
- final int startPointer = (int) in.getFilePointer();
- assertTrue(startPointer < in.length());
+ int offset = TestUtil.nextInt(random(), 0, (int) in.length() - 1);
if (random().nextBoolean()) {
- final long prefetchLength = TestUtil.nextLong(random(), 1, in.length() - startPointer);
- in.prefetch(prefetchLength);
+ final long prefetchLength = TestUtil.nextLong(random(), 1, in.length() - offset);
+ in.prefetch(offset, prefetchLength);
}
- assertEquals(startPointer, in.getFilePointer());
+ in.seek(offset);
+ assertEquals(offset, in.getFilePointer());
switch (random().nextInt(100)) {
case 0:
- assertEquals(arr[startOffset + startPointer], in.readByte());
+ assertEquals(arr[startOffset + offset], in.readByte());
break;
case 1:
- if (in.length() - startPointer >= Long.BYTES) {
+ if (in.length() - offset >= Long.BYTES) {
assertEquals(
- (long) BitUtil.VH_LE_LONG.get(arr, startOffset + startPointer), in.readLong());
+ (long) BitUtil.VH_LE_LONG.get(arr, startOffset + offset), in.readLong());
}
break;
default:
final int readLength =
- TestUtil.nextInt(
- random(), 1, (int) Math.min(temp.length, in.length() - startPointer));
+ TestUtil.nextInt(random(), 1, (int) Math.min(temp.length, in.length() - offset));
in.readBytes(temp, 0, readLength);
assertArrayEquals(
ArrayUtil.copyOfSubArray(
- arr, startOffset + startPointer, startOffset + startPointer + readLength),
+ arr, startOffset + offset, startOffset + offset + readLength),
ArrayUtil.copyOfSubArray(temp, 0, readLength));
}
- if (in.getFilePointer() == in.length() || random().nextBoolean()) {
- in.seek(TestUtil.nextInt(random(), 0, (int) in.length() - 1));
- }
}
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
index 7bf3bf56055..4d6b5b061ce 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
@@ -131,9 +131,9 @@ public class MockIndexInputWrapper extends FilterIndexInput {
}
@Override
- public void prefetch(long length) throws IOException {
+ public void prefetch(long offset, long length) throws IOException {
ensureOpen();
- in.prefetch(length);
+ in.prefetch(offset, length);
}
@Override