Skip to content

Apache Lucene open-source search software

Add IndexInput isLoaded (#13998)

Chris Hegarty7dbbd0daa97e
diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
index 09f9211ac17..b649ace8c4d 100644
--- a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
@@ -18,6 +18,7 @@ package org.apache.lucene.store;
import java.io.Closeable;
import java.io.IOException;
+import java.util.Optional;
import org.apache.lucene.codecs.CompoundFormat;
/**
@@ -234,4 +235,19 @@ public abstract class IndexInput extends DataInput implements Closeable {
* <p>The default implementation is a no-op.
*/
public void updateReadAdvice(ReadAdvice readAdvice) throws IOException {}
+
+ /**
+ * Returns a hint whether all the contents of this input are resident in physical memory. It's a
+ * hint because the operating system may have paged out some of the data by the time this method
+ * returns. If the optional is true, then it's likely that the contents of this input are resident
+ * in physical memory. A value of false does not imply that the contents are not resident in
+ * physical memory. An empty optional is returned if it is not possible to determine.
+ *
+ * <p>This runs in linear time with the {@link #length()} of this input / page size.
+ *
+ * <p>The default implementation returns an empty optional.
+ */
+ public Optional<Boolean> isLoaded() {
+ return Optional.empty();
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java b/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java
index 08b2e83d36d..44127d90a98 100644
--- a/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java
@@ -17,6 +17,7 @@
package org.apache.lucene.store;
import java.io.IOException;
+import java.util.Optional;
import org.apache.lucene.util.BitUtil; // javadocs
/**
@@ -77,4 +78,13 @@ public interface RandomAccessInput {
* @see IndexInput#prefetch
*/
default void prefetch(long offset, long length) throws IOException {}
+
+ /**
+ * Returns a hint whether all the contents of this input are resident in physical memory.
+ *
+ * @see IndexInput#isLoaded()
+ */
+ default Optional<Boolean> isLoaded() {
+ return Optional.empty();
+ }
}
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
index 5824e78bdc7..2fc9f6369b7 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
@@ -420,6 +420,16 @@ abstract class MemorySegmentIndexInput extends IndexInput
}
}
+ @Override
+ public Optional<Boolean> isLoaded() {
+ for (MemorySegment seg : segments) {
+ if (seg.isLoaded() == false) {
+ return Optional.of(Boolean.FALSE);
+ }
+ }
+ return Optional.of(Boolean.TRUE);
+ }
+
@Override
public byte readByte(long pos) throws IOException {
try {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
index e041afdbcb3..d8ec0529b5f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
@@ -51,9 +51,11 @@ import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.tests.mockfile.ExtrasFS;
@@ -1636,4 +1638,44 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
}
}
}
+
+ public void testIsLoaded() throws IOException {
+ testIsLoaded(0);
+ }
+
+ public void testIsLoadedOnSlice() throws IOException {
+ testIsLoaded(TestUtil.nextInt(random(), 1, 1024));
+ }
+
+ private void testIsLoaded(int startOffset) throws IOException {
+ try (Directory dir = getDirectory(createTempDir())) {
+ if (FilterDirectory.unwrap(dir) instanceof MMapDirectory mMapDirectory) {
+ mMapDirectory.setPreload(MMapDirectory.ALL_FILES);
+ }
+ final int totalLength = startOffset + TestUtil.nextInt(random(), 16384, 65536);
+ byte[] arr = new byte[totalLength];
+ random().nextBytes(arr);
+ try (IndexOutput out = dir.createOutput("temp.bin", IOContext.DEFAULT)) {
+ out.writeBytes(arr, arr.length);
+ }
+
+ try (IndexInput orig = dir.openInput("temp.bin", IOContext.DEFAULT)) {
+ IndexInput in;
+ if (startOffset == 0) {
+ in = orig.clone();
+ } else {
+ in = orig.slice("slice", startOffset, totalLength - startOffset);
+ }
+ var loaded = in.isLoaded();
+ if (FilterDirectory.unwrap(dir) instanceof MMapDirectory
+ // direct IO wraps MMap but does not support isLoaded
+ && !(dir.getClass().getName().contains("DirectIO"))) {
+ assertTrue(loaded.isPresent());
+ assertTrue(loaded.get());
+ } else {
+ assertFalse(loaded.isPresent());
+ }
+ }
+ }
+ }
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
index aa5c841bf3b..1dc5456d64e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
@@ -19,6 +19,7 @@ package org.apache.lucene.tests.store;
import java.io.Closeable;
import java.io.IOException;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.lucene.internal.tests.TestSecrets;
import org.apache.lucene.store.FilterIndexInput;
@@ -184,6 +185,13 @@ public class MockIndexInputWrapper extends FilterIndexInput {
in.prefetch(offset, length);
}
+ @Override
+ public Optional<Boolean> isLoaded() {
+ ensureOpen();
+ ensureAccessible();
+ return in.isLoaded();
+ }
+
@Override
public void updateReadAdvice(ReadAdvice readAdvice) throws IOException {
ensureOpen();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/SerialIOCountingDirectory.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/SerialIOCountingDirectory.java
index 2afbc2fd4a7..1b4234c3d79 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/SerialIOCountingDirectory.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/SerialIOCountingDirectory.java
@@ -17,6 +17,7 @@
package org.apache.lucene.tests.store;
import java.io.IOException;
+import java.util.Optional;
import java.util.concurrent.atomic.LongAdder;
import org.apache.lucene.internal.hppc.LongHashSet;
import org.apache.lucene.store.ChecksumIndexInput;
@@ -206,5 +207,10 @@ public class SerialIOCountingDirectory extends FilterDirectory {
IndexInput clone = in.clone();
return new SerializedIOCountingIndexInput(clone, readAdvice, sliceOffset, sliceLength);
}
+
+ @Override
+ public Optional<Boolean> isLoaded() {
+ return in.isLoaded();
+ }
}
}