Skip to content

Apache Lucene open-source search software

Add a preload hint for preloading mmap data on specific open calls (#14604)

Simon Cooper8d45ad6d3bb9
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
index f3b62a7523a..8128998a84d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
@@ -45,6 +45,7 @@ import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.PreloadHint;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.Bits;
@@ -110,7 +111,10 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader
Lucene99HnswVectorsFormat.VECTOR_INDEX_EXTENSION,
Lucene99HnswVectorsFormat.VECTOR_INDEX_CODEC_NAME,
state.context.withHints(
- FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM));
+ FileTypeHint.DATA,
+ FileDataHint.KNN_VECTORS,
+ DataAccessHint.RANDOM,
+ PreloadHint.INSTANCE));
success = true;
} finally {
if (success == false) {
diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
index c76e8d1c72c..2d0eb3621bd 100644
--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
@@ -99,6 +99,13 @@ public class MMapDirectory extends FSDirectory {
*/
public static final BiPredicate<String, IOContext> NO_FILES = (_, _) -> false;
+ /**
+ * Argument for {@link #setPreload(BiPredicate)} that configures files to be preloaded when they
+ * are hinted to do so.
+ */
+ public static final BiPredicate<String, IOContext> PRELOAD_HINT =
+ (_, c) -> c.hints().contains(PreloadHint.INSTANCE);
+
/**
* This sysprop allows to control the total maximum number of mmapped files that can be associated
* with a single shared {@link java.lang.foreign.Arena foreign Arena}. For example, to set the max
@@ -233,6 +240,7 @@ public class MMapDirectory extends FSDirectory {
* @param preload a {@link BiPredicate} whose first argument is the file name, and second argument
* is the {@link IOContext} used to open the file
* @see #ALL_FILES
+ * @see #PRELOAD_HINT
* @see #NO_FILES
*/
public void setPreload(BiPredicate<String, IOContext> preload) {
diff --git a/lucene/core/src/java/org/apache/lucene/store/PreloadHint.java b/lucene/core/src/java/org/apache/lucene/store/PreloadHint.java
new file mode 100644
index 00000000000..cbcd33c15b5
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/store/PreloadHint.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.store;
+
+/** A hint that the file should be preloaded into memory */
+public enum PreloadHint implements IOContext.FileOpenHint {
+ INSTANCE
+}
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
index 3d217efae20..b947a65c573 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
@@ -131,6 +131,31 @@ public class TestMMapDirectory extends BaseDirectoryTestCase {
}
}
+ public void testPreload() throws Exception {
+ assumeTrue("madvise for preloading only works on linux/mac", MMapDirectory.supportsMadvise());
+
+ final int size = 8 * 1024;
+ byte[] bytes = new byte[size];
+ random().nextBytes(bytes);
+
+ try (MMapDirectory dir = new MMapDirectory(createTempDir("testPreload"))) {
+ dir.setPreload(MMapDirectory.PRELOAD_HINT);
+ try (IndexOutput out =
+ dir.createOutput("test", IOContext.DEFAULT.withHints(PreloadHint.INSTANCE))) {
+ out.writeBytes(bytes, 0, bytes.length);
+ }
+
+ try (final IndexInput in =
+ dir.openInput("test", IOContext.DEFAULT.withHints(PreloadHint.INSTANCE))) {
+ // the data should be loaded in memory
+ assertTrue(in.isLoaded().orElse(false));
+ final byte[] readBytes = new byte[size];
+ in.readBytes(readBytes, 0, readBytes.length);
+ assertArrayEquals(bytes, readBytes);
+ }
+ }
+ }
+
// Opens the input with ReadAdvice.READONCE to ensure slice and clone are appropriately confined
public void testConfined() throws Exception {
final int size = 16;