Apache Lucene open-source search software
Replace boolean flags on `IOContext` with an enum. (#13219)
Adrien Grand8558934501d2
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 20c1f84a49c..987927ce8de 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -89,6 +89,9 @@ API Changes* GITHUB#13205: Convert IOContext, MergeInfo, and FlushInfo to record classes. (Uwe Schindler)+* GITHUB#13219: The `readOnce`, `load` and `random` flags on `IOContext` have+ been replaced with a new `ReadAdvice` enum. (Adrien Grand)+New Features---------------------diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.mdindex 6427daf5ec1..1449166bdcb 100644--- a/lucene/MIGRATE.md+++ b/lucene/MIGRATE.md@@ -172,6 +172,15 @@ access the members using method calls instead of field accesses. Affected classe- `IOContext`, `MergeInfo`, and `FlushInfo` (GITHUB#13205)+### Boolean flags on IOContext replaced with a new ReadAdvice enum.++The `readOnce`, `load` and `random` flags on `IOContext` have been replaced with a new `ReadAdvice`+enum.++### IOContext.LOAD renamed to IOContext.PRELOAD++`IOContext#LOAD` has been replaced with `IOContext#PRELOAD`.+## Migration from Lucene 9.0 to Lucene 9.1### Test framework package migration and module (LUCENE-10301)diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.javaindex ae8322c7f56..d47e4c5f452 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java@@ -27,6 +27,7 @@ import org.apache.lucene.index.CorruptIndexException;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.IndexFileNames;import org.apache.lucene.index.SegmentReadState;+import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;import org.apache.lucene.util.Accountable;import org.apache.lucene.util.Accountables;@@ -53,7 +54,7 @@ public class VariableGapTermsIndexReader extends TermsIndexReaderBase {state.segmentInfo.name,state.segmentSuffix,VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION);- final IndexInput in = state.directory.openInput(fileName, state.context.toReadOnce());+ final IndexInput in = state.directory.openInput(fileName, IOContext.READONCE);boolean success = false;try {diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.javaindex e63c85a3050..b0b14c73d78 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java@@ -74,7 +74,7 @@ public class FSTTermsReader extends FieldsProducer {state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);this.postingsReader = postingsReader;- this.fstTermsInput = state.directory.openInput(termsFileName, IOContext.LOAD);+ this.fstTermsInput = state.directory.openInput(termsFileName, IOContext.PRELOAD);IndexInput in = this.fstTermsInput;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsReader.javaindex eeda6798e5d..08e33dbcca0 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsReader.java@@ -60,7 +60,7 @@ public class Lucene90PointsReader extends PointsReader {boolean success = false;try {- indexIn = readState.directory.openInput(indexFileName, IOContext.LOAD);+ indexIn = readState.directory.openInput(indexFileName, IOContext.PRELOAD);CodecUtil.checkIndexHeader(indexIn,Lucene90PointsFormat.INDEX_CODEC_NAME,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.javaindex 5d218f55faa..da9ac2309e2 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java@@ -143,7 +143,7 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {String indexName =IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_INDEX_EXTENSION);- indexIn = state.directory.openInput(indexName, IOContext.LOAD);+ indexIn = state.directory.openInput(indexName, IOContext.PRELOAD);CodecUtil.checkIndexHeader(indexIn,TERMS_INDEX_CODEC_NAME,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/FieldsIndexReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/FieldsIndexReader.javaindex 3b46d12edf9..7b79f800c19 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/FieldsIndexReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/FieldsIndexReader.java@@ -65,7 +65,7 @@ final class FieldsIndexReader extends FieldsIndex {maxPointer = metaIn.readLong();indexInput =- dir.openInput(IndexFileNames.segmentFileName(name, suffix, extension), IOContext.LOAD);+ dir.openInput(IndexFileNames.segmentFileName(name, suffix, extension), IOContext.PRELOAD);boolean success = false;try {CodecUtil.checkIndexHeader(diff --git a/lucene/core/src/java/org/apache/lucene/store/IOContext.java b/lucene/core/src/java/org/apache/lucene/store/IOContext.javaindex e0a8cd55367..678a62bfecb 100644--- a/lucene/core/src/java/org/apache/lucene/store/IOContext.java+++ b/lucene/core/src/java/org/apache/lucene/store/IOContext.java@@ -27,22 +27,10 @@ import java.util.Objects;* @param context An object of a enumerator Context type* @param mergeInfo must be given when {@code context == MERGE}* @param flushInfo must be given when {@code context == FLUSH}- * @param readOnce This flag indicates that the file will be opened, then fully read sequentially- * then closed.- * @param load This flag is used for files that are a small fraction of the total index size and are- * expected to be heavily accessed in random-access fashion. Some {@link Directory}- * implementations may choose to load such files into physical memory (e.g. Java heap) as a way- * to provide stronger guarantees on query latency.- * @param randomAccess This flag indicates that the file will be accessed randomly. If this flag is- * set, then readOnce will be false.+ * @param readAdvice Advice regarding the read access pattern*/public record IOContext(- Context context,- MergeInfo mergeInfo,- FlushInfo flushInfo,- boolean readOnce,- boolean load,- boolean randomAccess) {+ Context context, MergeInfo mergeInfo, FlushInfo flushInfo, ReadAdvice readAdvice) {/*** Context is a enumerator which specifies the context in which the Directory is being used for.@@ -54,58 +42,50 @@ public record IOContext(DEFAULT};- public static final IOContext DEFAULT = new IOContext(Context.DEFAULT);+ public static final IOContext DEFAULT =+ new IOContext(Context.DEFAULT, null, null, ReadAdvice.NORMAL);- public static final IOContext READONCE = new IOContext(true, false, false);+ public static final IOContext READONCE = new IOContext(ReadAdvice.SEQUENTIAL);- public static final IOContext READ = new IOContext(false, false, false);+ public static final IOContext READ = new IOContext(ReadAdvice.NORMAL);- public static final IOContext LOAD = new IOContext(false, true, true);+ public static final IOContext PRELOAD = new IOContext(ReadAdvice.RANDOM_PRELOAD);- public static final IOContext RANDOM = new IOContext(false, false, true);+ public static final IOContext RANDOM = new IOContext(ReadAdvice.RANDOM);@SuppressWarnings("incomplete-switch")public IOContext {+ Objects.requireNonNull(context, "context must not be null");+ Objects.requireNonNull(readAdvice, "readAdvice must not be null");switch (context) {case MERGE -> Objects.requireNonNull(mergeInfo, "mergeInfo must not be null if context is MERGE");case FLUSH -> Objects.requireNonNull(flushInfo, "flushInfo must not be null if context is FLUSH");}- if (load && readOnce) {- throw new IllegalArgumentException("load and readOnce are mutually exclusive");+ if (context == Context.MERGE && readAdvice != ReadAdvice.SEQUENTIAL) {+ throw new IllegalArgumentException(+ "The MERGE context must use the SEQUENTIAL read access advice");}- if (readOnce && randomAccess) {- throw new IllegalArgumentException("readOnce and randomAccess are mutually exclusive");+ if ((context == Context.FLUSH || context == Context.DEFAULT)+ && readAdvice != ReadAdvice.NORMAL) {+ throw new IllegalArgumentException(+ "The FLUSH and DEFAULT contexts must use the NORMAL read access advice");}- if (load && randomAccess == false) {- throw new IllegalArgumentException("cannot be load but not randomAccess");- }- }-- private IOContext(boolean readOnce, boolean load, boolean randomAccess) {- this(Context.READ, null, null, readOnce, load, randomAccess);}- private IOContext(Context context) {- this(context, null, null, false, false, false);+ private IOContext(ReadAdvice accessAdvice) {+ this(Context.READ, null, null, accessAdvice);}/** Creates an IOContext for flushing. */public IOContext(FlushInfo flushInfo) {- this(Context.FLUSH, null, flushInfo, false, false, false);+ this(Context.FLUSH, null, flushInfo, ReadAdvice.NORMAL);}/** Creates an IOContext for merging. */public IOContext(MergeInfo mergeInfo) {- this(Context.MERGE, mergeInfo, null, false, false, false);- }-- /**- * Return a copy of this IOContext with {@link #readOnce} set to {@code true}. The {@link #load}- * flag is set to {@code false}.- */- public IOContext toReadOnce() {- return new IOContext(context, mergeInfo, flushInfo, true, false, randomAccess);+ // Merges read input segments sequentially.+ this(Context.MERGE, mergeInfo, null, ReadAdvice.SEQUENTIAL);}}diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.javaindex c7e3518ab35..57edaedf020 100644--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java@@ -85,10 +85,10 @@ public class MMapDirectory extends FSDirectory {/*** Argument for {@link #setPreload(BiPredicate)} that configures files to be preloaded upon- * opening them if they use the {@link IOContext#LOAD} I/O context.+ * opening them if they use the {@link IOContext#PRELOAD} I/O context.*/public static final BiPredicate<String, IOContext> BASED_ON_LOAD_IO_CONTEXT =- (filename, context) -> context.load();+ (filename, context) -> context.readAdvice() == ReadAdvice.RANDOM_PRELOAD;private BiPredicate<String, IOContext> preload = NO_FILES;diff --git a/lucene/core/src/java/org/apache/lucene/store/ReadAdvice.java b/lucene/core/src/java/org/apache/lucene/store/ReadAdvice.javanew file mode 100644index 00000000000..5c706e2abf5--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/store/ReadAdvice.java@@ -0,0 +1,44 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.store;++/** Advice regarding the read access pattern. */+public enum ReadAdvice {+ /**+ * Normal behavior. Data is expected to be read mostly sequentially. The system is expected to+ * cache the hottest pages.+ */+ NORMAL,+ /**+ * Data is expected to be read in a random-access fashion, either by {@link IndexInput#seek(long)+ * seeking} often and reading relatively short sequences of bytes at once, or by reading data+ * through the {@link RandomAccessInput} abstraction in random order.+ */+ RANDOM,+ /**+ * Data is expected to be read sequentially with very little seeking at most. The system may read+ * ahead aggressively and free pages soon after they are accessed.+ */+ SEQUENTIAL,+ /**+ * Data is treated as random-access memory in practice. {@link Directory} implementations may+ * explicitly load the content of the file in memory, or provide hints to the system so that it+ * loads the content of the file into the page cache at open time. This should only be used on+ * very small files that can be expected to fit in RAM with very high confidence.+ */+ RANDOM_PRELOAD+}diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.javaindex 3a0787f4c0a..887956f306c 100644--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java@@ -52,7 +52,14 @@ final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInMemorySegmentIndexInput.newInstance(resourceDescription,arena,- map(arena, resourceDescription, fc, context, chunkSizePower, preload, fileSize),+ map(+ arena,+ resourceDescription,+ fc,+ context.readAdvice(),+ chunkSizePower,+ preload,+ fileSize),fileSize,chunkSizePower);success = true;@@ -78,7 +85,7 @@ final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInArena arena,String resourceDescription,FileChannel fc,- IOContext context,+ ReadAdvice readAdvice,int chunkSizePower,boolean preload,long length)@@ -108,7 +115,7 @@ final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInif (preload) {segment.load();} else if (nativeAccess.isPresent() && chunkSizePower >= 21) {- nativeAccess.get().madvise(segment, context);+ nativeAccess.get().madvise(segment, readAdvice);}segments[segNr] = segment;startOffset += segSize;diff --git a/lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java b/lucene/core/src/java21/org/apache/lucene/store/NativeAccess.javaindex 30c37901e5c..f4bc4f89d58 100644--- a/lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java+++ b/lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java@@ -25,7 +25,7 @@ import org.apache.lucene.util.Constants;abstract class NativeAccess {/** Invoke the {@code madvise} call for the given {@link MemorySegment}. */- public abstract void madvise(MemorySegment segment, IOContext context) throws IOException;+ public abstract void madvise(MemorySegment segment, ReadAdvice readAdvice) throws IOException;/*** Return the NativeAccess instance for this platform. At moment we only support Linux and MacOSdiff --git a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java b/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.javaindex d620f0b5b45..b74bd7fe365 100644--- a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java+++ b/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java@@ -26,7 +26,6 @@ import java.lang.invoke.MethodHandle;import java.util.Locale;import java.util.Optional;import java.util.logging.Logger;-import org.apache.lucene.store.IOContext.Context;@SuppressWarnings("preview")final class PosixNativeAccess extends NativeAccess {@@ -110,12 +109,12 @@ final class PosixNativeAccess extends NativeAccess {}@Override- public void madvise(MemorySegment segment, IOContext context) throws IOException {+ public void madvise(MemorySegment segment, ReadAdvice readAdvice) throws IOException {// Note: madvise is bypassed if the segment should be preloaded via MemorySegment#load.if (segment.byteSize() == 0L) {return; // empty segments should be excluded, because they may have no address at all}- final Integer advice = mapIOContext(context);+ final Integer advice = mapReadAdvice(readAdvice);if (advice == null) {return; // do nothing}@@ -136,18 +135,12 @@ final class PosixNativeAccess extends NativeAccess {}}- private Integer mapIOContext(IOContext ctx) {- // Merging always wins and implies sequential access, because kernel is advised to free pages- // after use:- if (ctx.context() == Context.MERGE) {- return POSIX_MADV_SEQUENTIAL;- }- if (ctx.randomAccess()) {- return POSIX_MADV_RANDOM;- }- if (ctx.readOnce()) {- return POSIX_MADV_SEQUENTIAL;- }- return null;+ private Integer mapReadAdvice(ReadAdvice readAdvice) {+ return switch (readAdvice) {+ case NORMAL -> null;+ case RANDOM -> POSIX_MADV_RANDOM;+ case SEQUENTIAL -> POSIX_MADV_SEQUENTIAL;+ case RANDOM_PRELOAD -> null;+ };}}