Apache Lucene open-source search software
Remove abstractions of MMapDirectory and its "fake" MR-JAR support and move it to main sourceSet (vectors untouched) (#14564)
Uwe Schindler2a5597e83b97
diff --git a/gradle/generation/extract-jdk-apis.gradle b/gradle/generation/extract-jdk-apis.gradleindex 3adde87da83..8a5fbd9e797 100644--- a/gradle/generation/extract-jdk-apis.gradle+++ b/gradle/generation/extract-jdk-apis.gradle@@ -20,7 +20,7 @@ def resources = scriptResources(buildscript)configure(project(":lucene:core")) {ext {apijars = layout.projectDirectory.dir("src/generated/jdk")- mrjarJavaVersions = [ 21 ]+ mrjarJavaVersions = [ 24 ]}configurations {diff --git a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java b/gradle/generation/extract-jdk-apis/ExtractJdkApis.javaindex c84c8f16996..7b4f54e3139 100644--- a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java+++ b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java@@ -47,14 +47,13 @@ import org.objectweb.asm.Type;public final class ExtractJdkApis {- private static final FileTime FIXED_FILEDATE = FileTime.from(Instant.parse("2022-01-01T00:00:00Z"));+ private static final FileTime FIXED_FILEDATE = FileTime.from(Instant.parse("2025-04-29T00:00:00Z"));- private static final String PATTERN_PANAMA_FOREIGN = "java.base/{java/lang/foreign/*,java/nio/channels/FileChannel}";private static final String PATTERN_VECTOR_INCUBATOR = "jdk.incubator.vector/jdk/incubator/vector/*";private static final String PATTERN_VECTOR_VM_INTERNALS = "java.base/jdk/internal/vm/vector/VectorSupport{,$Vector,$VectorMask,$VectorPayload,$VectorShuffle}";static final Map<Integer,List<String>> CLASSFILE_PATTERNS = Map.of(- 21, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR)+ 24, List.of(PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR));public static void main(String... args) throws IOException {diff --git a/gradle/java/core-mrjar.gradle b/gradle/java/core-mrjar.gradleindex 89bfc2c6d8c..8599fc39a10 100644--- a/gradle/java/core-mrjar.gradle+++ b/gradle/java/core-mrjar.gradle@@ -15,7 +15,7 @@* limitations under the License.*/-// Produce an MR-JAR with Java 19+ foreign and vector implementations+// Produce an MR-JAR for panama vector implementationsconfigure(project(":lucene:core")) {plugins.withType(JavaPlugin) {@@ -52,7 +52,7 @@ configure(project(":lucene:core")) {mrjarJavaVersions.each { jdkVersion ->// the sourceSet which corresponds to the minimum/base Java version// will copy its output to root of JAR, all other sourceSets will go into MR-JAR folders:- boolean isBaseVersion = (jdkVersion.toString() == rootProject.minJavaVersion.toString())+ boolean isBaseVersion = (jdkVersion.toString() as int <= rootProject.minJavaVersion.toString() as int)into(isBaseVersion ? '' : "META-INF/versions/${jdkVersion}") {from sourceSets["main${jdkVersion}"].output}diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 0a64dce99f4..aa9fab13575 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -52,6 +52,8 @@ Other* GITHUB#14229: Bump minimum required Java version to 25+* GITHUB#14564: Remove abstractions from MMapDirectory as we need no MR-JAR anymore. (Uwe Schindler)+======================= Lucene 10.3.0 =======================API Changesdiff --git a/lucene/core/src/generated/jdk/jdk21.apijar b/lucene/core/src/generated/jdk/jdk21.apijardeleted file mode 100644index 8120f23b8b4..00000000000Binary files a/lucene/core/src/generated/jdk/jdk21.apijar and /dev/null differdiff --git a/lucene/core/src/generated/jdk/jdk24.apijar b/lucene/core/src/generated/jdk/jdk24.apijarnew file mode 100644index 00000000000..3fa35f71485Binary files /dev/null and b/lucene/core/src/generated/jdk/jdk24.apijar differdiff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.javaindex 35b83f77db2..0fe3810bdc7 100644--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java@@ -19,12 +19,16 @@ package org.apache.lucene.store;import static org.apache.lucene.index.IndexFileNames.CODEC_FILE_PATTERN;import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.lang.invoke.MethodType;+import java.lang.foreign.Arena;+import java.lang.foreign.MemorySegment;import java.nio.channels.ClosedChannelException; // javadoc @link+import java.nio.channels.FileChannel;+import java.nio.channels.FileChannel.MapMode;import java.nio.file.Path;+import java.nio.file.StandardOpenOption;import java.util.Locale;import java.util.Optional;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.Future;import java.util.function.BiFunction;import java.util.function.BiPredicate;@@ -32,6 +36,7 @@ import java.util.function.Function;import java.util.logging.Logger;import org.apache.lucene.index.IndexFileNames;import org.apache.lucene.util.Constants;+import org.apache.lucene.util.Unwrappable;/*** File-based {@link Directory} implementation that uses mmap for reading, and {@link@@ -56,11 +61,10 @@ import org.apache.lucene.util.Constants;** <p>This class will use the modern {@link java.lang.foreign.MemorySegment} API available since* Java 21 which allows to safely unmap previously mmapped files after closing the {@link- * IndexInput}s. There is no need to enable the "preview feature" of your Java version; it works out- * of box with some compilation tricks. For more information about the foreign memory API read- * documentation of the {@link java.lang.foreign} package.+ * IndexInput}s. For more information about the foreign memory API read documentation of the {@link+ * java.lang.foreign} package.*- * <p>On some platforms like Linux and MacOS X, this class will invoke the syscall {@code madvise()}+ * <p>On some platforms like Linux and macOS, this class will invoke the syscall {@code madvise()}* to advise how OS kernel should handle paging after opening a file. For this to work, Java code* must be able to call native code. If this is not allowed, a warning is logged. To enable native* access for Lucene in a modularized application, pass {@code@@ -130,7 +134,7 @@ public class MMapDirectory extends FSDirectory {return Optional.of(groupKey);};- private BiFunction<String, IOContext, Optional<ReadAdvice>> readAdvice =+ private BiFunction<String, IOContext, Optional<ReadAdvice>> readAdviceOverride =(_, _) -> Optional.empty();private BiPredicate<String, IOContext> preload = NO_FILES;@@ -142,15 +146,19 @@ public class MMapDirectory extends FSDirectory {* <li>256 MiBytes for 32 bit JVMs* </ul>*/- public static final long DEFAULT_MAX_CHUNK_SIZE;+ public static final long DEFAULT_MAX_CHUNK_SIZE =+ Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);- /** A provider specific context object or null, that will be passed to openInput. */- final Object attachment = PROVIDER.attachment();+ private static final Optional<NativeAccess> NATIVE_ACCESS = NativeAccess.getImplementation();+ private static final int SHARED_ARENA_PERMITS =+ checkMaxPermits(getSharedArenaMaxPermitsSysprop());private Function<String, Optional<String>> groupingFunction = GROUP_BY_SEGMENT;final int chunkSizePower;+ final ConcurrentHashMap<String, RefCountedSharedArena> arenas = new ConcurrentHashMap<>();+/*** Create a new MMapDirectory for the named location. The directory is created at the named* location if it does not yet exist.@@ -195,9 +203,9 @@ public class MMapDirectory extends FSDirectory {* files cannot be mapped. Using a lower chunk size makes the directory implementation a little* bit slower (as the correct chunk may be resolved on lots of seeks) but the chance is higher* that mmap does not fail. On 64 bit Java platforms, this parameter should always be large (like- * 1 GiBytes, or even larger with recent Java versions), as the address space is big enough. If it- * is larger, fragmentation of address space increases, but number of file handles and mappings is- * lower for huge installations with many open indexes.+ * 16 GiBytes), as the address space is big enough. If it is larger, fragmentation of address+ * space increases, but number of file handles and mappings is lower for huge installations with+ * many open indexes.** <p><b>Please note:</b> The chunk size is always rounded down to a power of 2.*@@ -242,7 +250,7 @@ public class MMapDirectory extends FSDirectory {*/public void setReadAdviceOverride(BiFunction<String, IOContext, Optional<ReadAdvice>> toReadAdvice) {- this.readAdvice = toReadAdvice;+ this.readAdviceOverride = toReadAdvice;}/**@@ -275,82 +283,175 @@ public class MMapDirectory extends FSDirectory {ensureOpen();ensureCanRead(name);Path path = directory.resolve(name);- return PROVIDER.openInput(- path,- chunkSizePower,- readAdvice+ final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")";++ // Work around for JDK-8259028: we need to unwrap our test-only file system layers+ path = Unwrappable.unwrapAll(path);++ boolean success = false;+ final boolean confined = context == IOContext.READONCE;+ final ReadAdvice readAdvice =+ readAdviceOverride.apply(name, context)- .orElseGet(() -> context.readAdvice().orElse(Constants.DEFAULT_READADVICE)),- context == IOContext.READONCE,- preload.test(name, context),- groupingFunction.apply(name),- attachment);+ .orElseGet(() -> context.readAdvice().orElse(Constants.DEFAULT_READADVICE));+ final Arena arena = confined ? Arena.ofConfined() : getSharedArena(name, arenas);+ try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {+ final long fileSize = fc.size();+ final IndexInput in =+ MemorySegmentIndexInput.newInstance(+ resourceDescription,+ arena,+ map(+ arena,+ resourceDescription,+ fc,+ readAdvice,+ chunkSizePower,+ preload.test(name, context),+ fileSize),+ fileSize,+ chunkSizePower,+ confined);+ success = true;+ return in;+ } finally {+ if (success == false) {+ arena.close();+ }+ }}- // visible for tests:- static final MMapIndexInputProvider<Object> PROVIDER;+ private final MemorySegment[] map(+ Arena arena,+ String resourceDescription,+ FileChannel fc,+ ReadAdvice readAdvice,+ int chunkSizePower,+ boolean preload,+ long length)+ throws IOException {+ if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)+ throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription);++ final long chunkSize = 1L << chunkSizePower;++ // we always allocate one more segments, the last one may be a 0 byte one+ final int nrSegments = (int) (length >>> chunkSizePower) + 1;++ final MemorySegment[] segments = new MemorySegment[nrSegments];++ long startOffset = 0L;+ for (int segNr = 0; segNr < nrSegments; segNr++) {+ final long segSize =+ (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset);+ final MemorySegment segment;+ try {+ segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena);+ } catch (IOException ioe) {+ throw convertMapFailedIOException(ioe, resourceDescription, segSize);+ }+ // if preload apply it without madvise.+ // skip madvise if the address of our segment is not page-aligned (small segments due to+ // internal FileChannel logic)+ if (preload) {+ segment.load();+ } else if (readAdvice != ReadAdvice.NORMAL+ && NATIVE_ACCESS.filter(na -> segment.address() % na.getPageSize() == 0).isPresent()) {+ // No need to madvise with ReadAdvice.NORMAL since it is the OS' default read advice.+ NATIVE_ACCESS.get().madvise(segment, readAdvice);+ }+ segments[segNr] = segment;+ startOffset += segSize;+ }+ return segments;+ }- interface MMapIndexInputProvider<A> {- IndexInput openInput(- Path path,- int chunkSizePower,- ReadAdvice readAdvice,- boolean readOnce,- boolean preload,- Optional<String> group,- A attachment)- throws IOException;+ /**+ * Gets an arena for the given filename, potentially aggregating files from the same segment into+ * a single ref counted shared arena. A ref counted shared arena, if created will be added to the+ * given arenas map.+ */+ private Arena getSharedArena(+ String name, ConcurrentHashMap<String, RefCountedSharedArena> arenas) {+ final var group = groupingFunction.apply(name);- long getDefaultMaxChunkSize();+ if (group.isEmpty()) {+ return Arena.ofShared();+ }- boolean supportsMadvise();+ String key = group.get();+ var refCountedArena =+ arenas.computeIfAbsent(+ key, s -> new RefCountedSharedArena(s, () -> arenas.remove(s), SHARED_ARENA_PERMITS));+ if (refCountedArena.acquire()) {+ return refCountedArena;+ } else {+ return arenas.compute(+ key,+ (s, v) -> {+ if (v != null && v.acquire()) {+ return v;+ } else {+ v = new RefCountedSharedArena(s, () -> arenas.remove(s), SHARED_ARENA_PERMITS);+ v.acquire(); // guaranteed to succeed+ return v;+ }+ });+ }+ }- /** An optional attachment of the provider, that will be passed to openInput. */- default A attachment() {- return null;+ private static IOException convertMapFailedIOException(+ IOException ioe, String resourceDescription, long bufSize) {+ final String originalMessage;+ final Throwable originalCause;+ if (ioe.getCause() instanceof OutOfMemoryError) {+ // nested OOM confuses users, because it's "incorrect", just print a plain message:+ originalMessage = "Map failed";+ originalCause = null;+ } else {+ originalMessage = ioe.getMessage();+ originalCause = ioe.getCause();+ }+ final String moreInfo;+ if (!Constants.JRE_IS_64BIT) {+ moreInfo =+ "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. ";+ } else if (Constants.WINDOWS) {+ moreInfo =+ "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. ";+ } else if (Constants.LINUX) {+ moreInfo =+ "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. ";+ } else {+ moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). ";}+ final IOException newIoe =+ new IOException(+ String.format(+ Locale.ENGLISH,+ "%s: %s [this may be caused by lack of enough unfragmented virtual address space "+ + "or too restrictive virtual memory limits enforced by the operating system, "+ + "preventing us to map a chunk of %d bytes. %sMore information: "+ + "https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]",+ originalMessage,+ resourceDescription,+ bufSize,+ moreInfo),+ originalCause);+ newIoe.setStackTrace(ioe.getStackTrace());+ return newIoe;+ }- default IOException convertMapFailedIOException(- IOException ioe, String resourceDescription, long bufSize) {- final String originalMessage;- final Throwable originalCause;- if (ioe.getCause() instanceof OutOfMemoryError) {- // nested OOM confuses users, because it's "incorrect", just print a plain message:- originalMessage = "Map failed";- originalCause = null;- } else {- originalMessage = ioe.getMessage();- originalCause = ioe.getCause();- }- final String moreInfo;- if (!Constants.JRE_IS_64BIT) {- moreInfo =- "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. ";- } else if (Constants.WINDOWS) {- moreInfo =- "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. ";- } else if (Constants.LINUX) {- moreInfo =- "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. ";- } else {- moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). ";- }- final IOException newIoe =- new IOException(- String.format(- Locale.ENGLISH,- "%s: %s [this may be caused by lack of enough unfragmented virtual address space "- + "or too restrictive virtual memory limits enforced by the operating system, "- + "preventing us to map a chunk of %d bytes. %sMore information: "- + "https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]",- originalMessage,- resourceDescription,- bufSize,- moreInfo),- originalCause);- newIoe.setStackTrace(ioe.getStackTrace());- return newIoe;+ private static int checkMaxPermits(int maxPermits) {+ if (RefCountedSharedArena.validMaxPermits(maxPermits)) {+ return maxPermits;}+ Logger.getLogger(MMapDirectory.class.getName())+ .warning(+ "Invalid value for sysprop "+ + MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP+ + ", must be positive and <= 0x07FF. The default value will be used.");+ return RefCountedSharedArena.DEFAULT_MAX_PERMITS;}private static int getSharedArenaMaxPermitsSysprop() {@@ -370,41 +471,11 @@ public class MMapDirectory extends FSDirectory {return ret;}- private static <A> MMapIndexInputProvider<A> lookupProvider() {- final var maxPermits = getSharedArenaMaxPermitsSysprop();- final var lookup = MethodHandles.lookup();- try {- final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");- // we use method handles, so we do not need to deal with setAccessible as we have private- // access through the lookup:- final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class, int.class));- try {- @SuppressWarnings("unchecked")- var res = (MMapIndexInputProvider<A>) constr.invoke(maxPermits);- return res;- } catch (RuntimeException | Error e) {- throw e;- } catch (Throwable th) {- throw new AssertionError(th);- }- } catch (NoSuchMethodException | IllegalAccessException e) {- throw new LinkageError(- "MemorySegmentIndexInputProvider is missing correctly typed constructor", e);- } catch (ClassNotFoundException cnfe) {- throw new LinkageError("MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);- }- }-/*** Returns true, if MMapDirectory uses the platform's {@code madvise()} syscall to advise how OS* kernel should handle paging after opening a file.*/public static boolean supportsMadvise() {- return PROVIDER.supportsMadvise();- }-- static {- PROVIDER = lookupProvider();- DEFAULT_MAX_CHUNK_SIZE = PROVIDER.getDefaultMaxChunkSize();+ return NATIVE_ACCESS.isPresent();}}diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentAccessInput.java b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentAccessInput.javasimilarity index 100%rename from lucene/core/src/java21/org/apache/lucene/store/MemorySegmentAccessInput.javarename to lucene/core/src/java/org/apache/lucene/store/MemorySegmentAccessInput.javadiff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.javasimilarity index 97%rename from lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.javarename to lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.javaindex 5404e1aafc7..64bed7cc9f1 100644--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java+++ b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java@@ -16,6 +16,8 @@*/package org.apache.lucene.store;+import static java.util.function.Predicate.not;+import java.io.EOFException;import java.io.IOException;import java.lang.foreign.Arena;@@ -37,9 +39,7 @@ import org.apache.lucene.util.IOConsumer;* <p>For efficiency, this class requires that the segment size are a power-of-two (<code>* chunkSizePower</code>).*/-@SuppressWarnings("preview")-abstract class MemorySegmentIndexInput extends IndexInput- implements RandomAccessInput, MemorySegmentAccessInput {+abstract class MemorySegmentIndexInput extends IndexInput implements MemorySegmentAccessInput {static final ValueLayout.OfByte LAYOUT_BYTE = ValueLayout.JAVA_BYTE;static final ValueLayout.OfShort LAYOUT_LE_SHORT =ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);@@ -129,16 +129,9 @@ abstract class MemorySegmentIndexInput extends IndexInput}// in Java 22 or later we can check the isAlive status of all segments// (see https://bugs.openjdk.org/browse/JDK-8310644):- if (Arrays.stream(segments).allMatch(s -> s.scope().isAlive()) == false) {+ if (Arrays.stream(segments).anyMatch(not(s -> s.scope().isAlive()))) {return new AlreadyClosedException("Already closed: " + this);}- // fallback for Java 21: ISE can be thrown by MemorySegment and contains "closed" in message:- if (e instanceof IllegalStateException- && e.getMessage() != null- && e.getMessage().contains("closed")) {- // the check is on message only, so preserve original cause for debugging:- return new AlreadyClosedException("Already closed: " + this, e);- }// otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to// the IndexInput method):throw e;diff --git a/lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java b/lucene/core/src/java/org/apache/lucene/store/NativeAccess.javasimilarity index 98%rename from lucene/core/src/java21/org/apache/lucene/store/NativeAccess.javarename to lucene/core/src/java/org/apache/lucene/store/NativeAccess.javaindex affc0e2ac71..838a40f6aa6 100644--- a/lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java+++ b/lucene/core/src/java/org/apache/lucene/store/NativeAccess.java@@ -21,7 +21,6 @@ import java.lang.foreign.MemorySegment;import java.util.Optional;import org.apache.lucene.util.Constants;-@SuppressWarnings("preview")abstract class NativeAccess {/** Invoke the {@code madvise} call for the given {@link MemorySegment}. */diff --git a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java b/lucene/core/src/java/org/apache/lucene/store/PosixNativeAccess.javasimilarity index 99%rename from lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.javarename to lucene/core/src/java/org/apache/lucene/store/PosixNativeAccess.javaindex 05eb6157118..477383fda32 100644--- a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java+++ b/lucene/core/src/java/org/apache/lucene/store/PosixNativeAccess.java@@ -27,7 +27,6 @@ import java.util.Locale;import java.util.Optional;import java.util.logging.Logger;-@SuppressWarnings("preview")final class PosixNativeAccess extends NativeAccess {private static final Logger LOG = Logger.getLogger(PosixNativeAccess.class.getName());diff --git a/lucene/core/src/java21/org/apache/lucene/store/RefCountedSharedArena.java b/lucene/core/src/java/org/apache/lucene/store/RefCountedSharedArena.javasimilarity index 99%rename from lucene/core/src/java21/org/apache/lucene/store/RefCountedSharedArena.javarename to lucene/core/src/java/org/apache/lucene/store/RefCountedSharedArena.javaindex e89de280a3c..2d6dd9b737a 100644--- a/lucene/core/src/java21/org/apache/lucene/store/RefCountedSharedArena.java+++ b/lucene/core/src/java/org/apache/lucene/store/RefCountedSharedArena.java@@ -38,7 +38,6 @@ import java.util.concurrent.atomic.AtomicInteger;* permitted and {@link #acquire()} returns false. This is independent of the actual number of the* ref count.*/-@SuppressWarnings("preview")final class RefCountedSharedArena implements Arena {// default maximum permitsdiff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.javadeleted file mode 100644index f740b1a372b..00000000000--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java+++ /dev/null@@ -1,184 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.store;--import java.io.IOException;-import java.lang.foreign.Arena;-import java.lang.foreign.MemorySegment;-import java.nio.channels.FileChannel;-import java.nio.channels.FileChannel.MapMode;-import java.nio.file.Path;-import java.nio.file.StandardOpenOption;-import java.util.Optional;-import java.util.concurrent.ConcurrentHashMap;-import java.util.logging.Logger;-import org.apache.lucene.util.Constants;-import org.apache.lucene.util.Unwrappable;--@SuppressWarnings("preview")-final class MemorySegmentIndexInputProvider- implements MMapDirectory.MMapIndexInputProvider<- ConcurrentHashMap<String, RefCountedSharedArena>> {-- private final Optional<NativeAccess> nativeAccess;- private final int sharedArenaMaxPermits;-- MemorySegmentIndexInputProvider(int maxPermits) {- this.nativeAccess = NativeAccess.getImplementation();- this.sharedArenaMaxPermits = checkMaxPermits(maxPermits);- }-- @Override- public IndexInput openInput(- Path path,- int chunkSizePower,- ReadAdvice readAdvice,- boolean confined,- boolean preload,- Optional<String> group,- ConcurrentHashMap<String, RefCountedSharedArena> arenas)- throws IOException {- final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")";-- // Work around for JDK-8259028: we need to unwrap our test-only file system layers- path = Unwrappable.unwrapAll(path);-- boolean success = false;- final Arena arena = confined ? Arena.ofConfined() : getSharedArena(group, arenas);- try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {- final long fileSize = fc.size();- final IndexInput in =- MemorySegmentIndexInput.newInstance(- resourceDescription,- arena,- map(arena, resourceDescription, fc, readAdvice, chunkSizePower, preload, fileSize),- fileSize,- chunkSizePower,- confined);- success = true;- return in;- } finally {- if (success == false) {- arena.close();- }- }- }-- @Override- public long getDefaultMaxChunkSize() {- return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);- }-- @Override- public boolean supportsMadvise() {- return nativeAccess.isPresent();- }-- private final MemorySegment[] map(- Arena arena,- String resourceDescription,- FileChannel fc,- ReadAdvice readAdvice,- int chunkSizePower,- boolean preload,- long length)- throws IOException {- if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)- throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription);-- final long chunkSize = 1L << chunkSizePower;-- // we always allocate one more segments, the last one may be a 0 byte one- final int nrSegments = (int) (length >>> chunkSizePower) + 1;-- final MemorySegment[] segments = new MemorySegment[nrSegments];-- long startOffset = 0L;- for (int segNr = 0; segNr < nrSegments; segNr++) {- final long segSize =- (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset);- final MemorySegment segment;- try {- segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena);- } catch (IOException ioe) {- throw convertMapFailedIOException(ioe, resourceDescription, segSize);- }- // if preload apply it without madvise.- // skip madvise if the address of our segment is not page-aligned (small segments due to- // internal FileChannel logic)- if (preload) {- segment.load();- } else if (readAdvice != ReadAdvice.NORMAL- && nativeAccess.filter(na -> segment.address() % na.getPageSize() == 0).isPresent()) {- // No need to madvise with ReadAdvice.NORMAL since it is the OS' default read advice.- nativeAccess.get().madvise(segment, readAdvice);- }- segments[segNr] = segment;- startOffset += segSize;- }- return segments;- }-- @Override- public ConcurrentHashMap<String, RefCountedSharedArena> attachment() {- return new ConcurrentHashMap<>();- }-- private static int checkMaxPermits(int maxPermits) {- if (RefCountedSharedArena.validMaxPermits(maxPermits)) {- return maxPermits;- }- Logger.getLogger(MemorySegmentIndexInputProvider.class.getName())- .warning(- "Invalid value for sysprop "- + MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP- + ", must be positive and <= 0x07FF. The default value will be used.");- return RefCountedSharedArena.DEFAULT_MAX_PERMITS;- }-- /**- * Gets an arena for the given group, potentially aggregating files from the same segment into a- * single ref counted shared arena. A ref counted shared arena, if created will be added to the- * given arenas map.- */- private Arena getSharedArena(- Optional<String> group, ConcurrentHashMap<String, RefCountedSharedArena> arenas) {- if (group.isEmpty()) {- return Arena.ofShared();- }-- String key = group.get();- var refCountedArena =- arenas.computeIfAbsent(- key, s -> new RefCountedSharedArena(s, () -> arenas.remove(s), sharedArenaMaxPermits));- if (refCountedArena.acquire()) {- return refCountedArena;- } else {- return arenas.compute(- key,- (s, v) -> {- if (v != null && v.acquire()) {- return v;- } else {- v = new RefCountedSharedArena(s, () -> arenas.remove(s), sharedArenaMaxPermits);- v.acquire(); // guaranteed to succeed- return v;- }- });- }- }-}diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.javasimilarity index 100%rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.javarename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.javadiff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.javasimilarity index 100%rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.javarename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.javadiff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.javasimilarity index 100%rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.javarename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.javadiff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.javasimilarity index 100%rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.javarename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.javadiff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorConstants.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorConstants.javasimilarity index 100%rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorConstants.javarename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorConstants.javadiff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.javasimilarity index 100%rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.javarename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.javadiff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.javasimilarity index 100%rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.javarename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.javadiff --git a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.javaindex 971847a2ec0..a91ae5414ba 100644--- a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java+++ b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java@@ -25,7 +25,6 @@ import java.util.Collections;import java.util.List;import java.util.Random;import java.util.concurrent.Callable;-import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.CountDownLatch;import java.util.concurrent.ExecutionException;import java.util.concurrent.Executors;@@ -227,10 +226,7 @@ public class TestMMapDirectory extends BaseDirectoryTestCase {}}- if (!(dir.attachment instanceof ConcurrentHashMap<?, ?> map)) {- throw new AssertionError("unexpected attachment: " + dir.attachment);- }- assertEquals(0, map.size());+ assertEquals(0, dir.arenas.size());}}@@ -279,10 +275,7 @@ public class TestMMapDirectory extends BaseDirectoryTestCase {closeable.close();}- if (!(dir.attachment instanceof ConcurrentHashMap<?, ?> map)) {- throw new AssertionError("unexpected attachment: " + dir.attachment);- }- assertEquals(0, map.size());+ assertEquals(0, dir.arenas.size());}}diff --git a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.javaindex 9c412664053..f19819941db 100644--- a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java+++ b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java@@ -207,10 +207,10 @@ public class TestModularLayer extends AbstractLuceneDistributionTest {ClassLoader loader = layer.findLoader(coreModuleId);- final Set<Integer> mrJarVersions = Set.of(21);- final Integer baseVersion = 21;+ final Set<Integer> mrJarVersions = Set.of(23);+ final Integer baseVersion = 23;- // the Java 21 PanamaVectorizationProvider must always be in main section of JAR file:+ // the Java 23 PanamaVectorizationProvider must always be in main section of JAR file:final String panamaClassName ="org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.class";Assertions.assertThat(loader.getResource(panamaClassName)).isNotNull();@@ -224,11 +224,6 @@ public class TestModularLayer extends AbstractLuceneDistributionTest {loader.getResource("META-INF/versions/" + v + panamaClassName)).isNotNull();});-- // you must be always able to load MemorySegmentIndexInput:- Assertions.assertThat(- loader.loadClass("org.apache.lucene.store.MemorySegmentIndexInput"))- .isNotNull();});}