Skip to content

Apache Lucene open-source search software

Remove abstractions of MMapDirectory and its "fake" MR-JAR support and move it to main sourceSet (vectors untouched) (#14564)

Uwe Schindler2a5597e83b97
diff --git a/gradle/generation/extract-jdk-apis.gradle b/gradle/generation/extract-jdk-apis.gradle
index 3adde87da83..8a5fbd9e797 100644
--- a/gradle/generation/extract-jdk-apis.gradle
+++ b/gradle/generation/extract-jdk-apis.gradle
@@ -20,7 +20,7 @@ def resources = scriptResources(buildscript)
configure(project(":lucene:core")) {
ext {
apijars = layout.projectDirectory.dir("src/generated/jdk")
- mrjarJavaVersions = [ 21 ]
+ mrjarJavaVersions = [ 24 ]
}
configurations {
diff --git a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java
index c84c8f16996..7b4f54e3139 100644
--- a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java
+++ b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java
@@ -47,14 +47,13 @@ import org.objectweb.asm.Type;
public final class ExtractJdkApis {
- private static final FileTime FIXED_FILEDATE = FileTime.from(Instant.parse("2022-01-01T00:00:00Z"));
+ private static final FileTime FIXED_FILEDATE = FileTime.from(Instant.parse("2025-04-29T00:00:00Z"));
- private static final String PATTERN_PANAMA_FOREIGN = "java.base/{java/lang/foreign/*,java/nio/channels/FileChannel}";
private static final String PATTERN_VECTOR_INCUBATOR = "jdk.incubator.vector/jdk/incubator/vector/*";
private static final String PATTERN_VECTOR_VM_INTERNALS = "java.base/jdk/internal/vm/vector/VectorSupport{,$Vector,$VectorMask,$VectorPayload,$VectorShuffle}";
static final Map<Integer,List<String>> CLASSFILE_PATTERNS = Map.of(
- 21, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR)
+ 24, List.of(PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR)
);
public static void main(String... args) throws IOException {
diff --git a/gradle/java/core-mrjar.gradle b/gradle/java/core-mrjar.gradle
index 89bfc2c6d8c..8599fc39a10 100644
--- a/gradle/java/core-mrjar.gradle
+++ b/gradle/java/core-mrjar.gradle
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-// Produce an MR-JAR with Java 19+ foreign and vector implementations
+// Produce an MR-JAR for panama vector implementations
configure(project(":lucene:core")) {
plugins.withType(JavaPlugin) {
@@ -52,7 +52,7 @@ configure(project(":lucene:core")) {
mrjarJavaVersions.each { jdkVersion ->
// the sourceSet which corresponds to the minimum/base Java version
// will copy its output to root of JAR, all other sourceSets will go into MR-JAR folders:
- boolean isBaseVersion = (jdkVersion.toString() == rootProject.minJavaVersion.toString())
+ boolean isBaseVersion = (jdkVersion.toString() as int <= rootProject.minJavaVersion.toString() as int)
into(isBaseVersion ? '' : "META-INF/versions/${jdkVersion}") {
from sourceSets["main${jdkVersion}"].output
}
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0a64dce99f4..aa9fab13575 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -52,6 +52,8 @@ Other
* GITHUB#14229: Bump minimum required Java version to 25
+* GITHUB#14564: Remove abstractions from MMapDirectory as we need no MR-JAR anymore. (Uwe Schindler)
+
======================= Lucene 10.3.0 =======================
API Changes
diff --git a/lucene/core/src/generated/jdk/jdk21.apijar b/lucene/core/src/generated/jdk/jdk21.apijar
deleted file mode 100644
index 8120f23b8b4..00000000000
Binary files a/lucene/core/src/generated/jdk/jdk21.apijar and /dev/null differ
diff --git a/lucene/core/src/generated/jdk/jdk24.apijar b/lucene/core/src/generated/jdk/jdk24.apijar
new file mode 100644
index 00000000000..3fa35f71485
Binary files /dev/null and b/lucene/core/src/generated/jdk/jdk24.apijar differ
diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
index 35b83f77db2..0fe3810bdc7 100644
--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
@@ -19,12 +19,16 @@ package org.apache.lucene.store;
import static org.apache.lucene.index.IndexFileNames.CODEC_FILE_PATTERN;
import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.lang.invoke.MethodType;
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
import java.nio.channels.ClosedChannelException; // javadoc @link
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.util.Locale;
import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.function.BiFunction;
import java.util.function.BiPredicate;
@@ -32,6 +36,7 @@ import java.util.function.Function;
import java.util.logging.Logger;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.Unwrappable;
/**
* File-based {@link Directory} implementation that uses mmap for reading, and {@link
@@ -56,11 +61,10 @@ import org.apache.lucene.util.Constants;
*
* <p>This class will use the modern {@link java.lang.foreign.MemorySegment} API available since
* Java 21 which allows to safely unmap previously mmapped files after closing the {@link
- * IndexInput}s. There is no need to enable the "preview feature" of your Java version; it works out
- * of box with some compilation tricks. For more information about the foreign memory API read
- * documentation of the {@link java.lang.foreign} package.
+ * IndexInput}s. For more information about the foreign memory API read documentation of the {@link
+ * java.lang.foreign} package.
*
- * <p>On some platforms like Linux and MacOS X, this class will invoke the syscall {@code madvise()}
+ * <p>On some platforms like Linux and macOS, this class will invoke the syscall {@code madvise()}
* to advise how OS kernel should handle paging after opening a file. For this to work, Java code
* must be able to call native code. If this is not allowed, a warning is logged. To enable native
* access for Lucene in a modularized application, pass {@code
@@ -130,7 +134,7 @@ public class MMapDirectory extends FSDirectory {
return Optional.of(groupKey);
};
- private BiFunction<String, IOContext, Optional<ReadAdvice>> readAdvice =
+ private BiFunction<String, IOContext, Optional<ReadAdvice>> readAdviceOverride =
(_, _) -> Optional.empty();
private BiPredicate<String, IOContext> preload = NO_FILES;
@@ -142,15 +146,19 @@ public class MMapDirectory extends FSDirectory {
* <li>256 MiBytes for 32 bit JVMs
* </ul>
*/
- public static final long DEFAULT_MAX_CHUNK_SIZE;
+ public static final long DEFAULT_MAX_CHUNK_SIZE =
+ Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);
- /** A provider specific context object or null, that will be passed to openInput. */
- final Object attachment = PROVIDER.attachment();
+ private static final Optional<NativeAccess> NATIVE_ACCESS = NativeAccess.getImplementation();
+ private static final int SHARED_ARENA_PERMITS =
+ checkMaxPermits(getSharedArenaMaxPermitsSysprop());
private Function<String, Optional<String>> groupingFunction = GROUP_BY_SEGMENT;
final int chunkSizePower;
+ final ConcurrentHashMap<String, RefCountedSharedArena> arenas = new ConcurrentHashMap<>();
+
/**
* Create a new MMapDirectory for the named location. The directory is created at the named
* location if it does not yet exist.
@@ -195,9 +203,9 @@ public class MMapDirectory extends FSDirectory {
* files cannot be mapped. Using a lower chunk size makes the directory implementation a little
* bit slower (as the correct chunk may be resolved on lots of seeks) but the chance is higher
* that mmap does not fail. On 64 bit Java platforms, this parameter should always be large (like
- * 1 GiBytes, or even larger with recent Java versions), as the address space is big enough. If it
- * is larger, fragmentation of address space increases, but number of file handles and mappings is
- * lower for huge installations with many open indexes.
+ * 16 GiBytes), as the address space is big enough. If it is larger, fragmentation of address
+ * space increases, but number of file handles and mappings is lower for huge installations with
+ * many open indexes.
*
* <p><b>Please note:</b> The chunk size is always rounded down to a power of 2.
*
@@ -242,7 +250,7 @@ public class MMapDirectory extends FSDirectory {
*/
public void setReadAdviceOverride(
BiFunction<String, IOContext, Optional<ReadAdvice>> toReadAdvice) {
- this.readAdvice = toReadAdvice;
+ this.readAdviceOverride = toReadAdvice;
}
/**
@@ -275,82 +283,175 @@ public class MMapDirectory extends FSDirectory {
ensureOpen();
ensureCanRead(name);
Path path = directory.resolve(name);
- return PROVIDER.openInput(
- path,
- chunkSizePower,
- readAdvice
+ final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")";
+
+ // Work around for JDK-8259028: we need to unwrap our test-only file system layers
+ path = Unwrappable.unwrapAll(path);
+
+ boolean success = false;
+ final boolean confined = context == IOContext.READONCE;
+ final ReadAdvice readAdvice =
+ readAdviceOverride
.apply(name, context)
- .orElseGet(() -> context.readAdvice().orElse(Constants.DEFAULT_READADVICE)),
- context == IOContext.READONCE,
- preload.test(name, context),
- groupingFunction.apply(name),
- attachment);
+ .orElseGet(() -> context.readAdvice().orElse(Constants.DEFAULT_READADVICE));
+ final Arena arena = confined ? Arena.ofConfined() : getSharedArena(name, arenas);
+ try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {
+ final long fileSize = fc.size();
+ final IndexInput in =
+ MemorySegmentIndexInput.newInstance(
+ resourceDescription,
+ arena,
+ map(
+ arena,
+ resourceDescription,
+ fc,
+ readAdvice,
+ chunkSizePower,
+ preload.test(name, context),
+ fileSize),
+ fileSize,
+ chunkSizePower,
+ confined);
+ success = true;
+ return in;
+ } finally {
+ if (success == false) {
+ arena.close();
+ }
+ }
}
- // visible for tests:
- static final MMapIndexInputProvider<Object> PROVIDER;
+ private final MemorySegment[] map(
+ Arena arena,
+ String resourceDescription,
+ FileChannel fc,
+ ReadAdvice readAdvice,
+ int chunkSizePower,
+ boolean preload,
+ long length)
+ throws IOException {
+ if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
+ throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription);
+
+ final long chunkSize = 1L << chunkSizePower;
+
+ // we always allocate one more segments, the last one may be a 0 byte one
+ final int nrSegments = (int) (length >>> chunkSizePower) + 1;
+
+ final MemorySegment[] segments = new MemorySegment[nrSegments];
+
+ long startOffset = 0L;
+ for (int segNr = 0; segNr < nrSegments; segNr++) {
+ final long segSize =
+ (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset);
+ final MemorySegment segment;
+ try {
+ segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena);
+ } catch (IOException ioe) {
+ throw convertMapFailedIOException(ioe, resourceDescription, segSize);
+ }
+ // if preload apply it without madvise.
+ // skip madvise if the address of our segment is not page-aligned (small segments due to
+ // internal FileChannel logic)
+ if (preload) {
+ segment.load();
+ } else if (readAdvice != ReadAdvice.NORMAL
+ && NATIVE_ACCESS.filter(na -> segment.address() % na.getPageSize() == 0).isPresent()) {
+ // No need to madvise with ReadAdvice.NORMAL since it is the OS' default read advice.
+ NATIVE_ACCESS.get().madvise(segment, readAdvice);
+ }
+ segments[segNr] = segment;
+ startOffset += segSize;
+ }
+ return segments;
+ }
- interface MMapIndexInputProvider<A> {
- IndexInput openInput(
- Path path,
- int chunkSizePower,
- ReadAdvice readAdvice,
- boolean readOnce,
- boolean preload,
- Optional<String> group,
- A attachment)
- throws IOException;
+ /**
+ * Gets an arena for the given filename, potentially aggregating files from the same segment into
+ * a single ref counted shared arena. A ref counted shared arena, if created will be added to the
+ * given arenas map.
+ */
+ private Arena getSharedArena(
+ String name, ConcurrentHashMap<String, RefCountedSharedArena> arenas) {
+ final var group = groupingFunction.apply(name);
- long getDefaultMaxChunkSize();
+ if (group.isEmpty()) {
+ return Arena.ofShared();
+ }
- boolean supportsMadvise();
+ String key = group.get();
+ var refCountedArena =
+ arenas.computeIfAbsent(
+ key, s -> new RefCountedSharedArena(s, () -> arenas.remove(s), SHARED_ARENA_PERMITS));
+ if (refCountedArena.acquire()) {
+ return refCountedArena;
+ } else {
+ return arenas.compute(
+ key,
+ (s, v) -> {
+ if (v != null && v.acquire()) {
+ return v;
+ } else {
+ v = new RefCountedSharedArena(s, () -> arenas.remove(s), SHARED_ARENA_PERMITS);
+ v.acquire(); // guaranteed to succeed
+ return v;
+ }
+ });
+ }
+ }
- /** An optional attachment of the provider, that will be passed to openInput. */
- default A attachment() {
- return null;
+ private static IOException convertMapFailedIOException(
+ IOException ioe, String resourceDescription, long bufSize) {
+ final String originalMessage;
+ final Throwable originalCause;
+ if (ioe.getCause() instanceof OutOfMemoryError) {
+ // nested OOM confuses users, because it's "incorrect", just print a plain message:
+ originalMessage = "Map failed";
+ originalCause = null;
+ } else {
+ originalMessage = ioe.getMessage();
+ originalCause = ioe.getCause();
+ }
+ final String moreInfo;
+ if (!Constants.JRE_IS_64BIT) {
+ moreInfo =
+ "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. ";
+ } else if (Constants.WINDOWS) {
+ moreInfo =
+ "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. ";
+ } else if (Constants.LINUX) {
+ moreInfo =
+ "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. ";
+ } else {
+ moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). ";
}
+ final IOException newIoe =
+ new IOException(
+ String.format(
+ Locale.ENGLISH,
+ "%s: %s [this may be caused by lack of enough unfragmented virtual address space "
+ + "or too restrictive virtual memory limits enforced by the operating system, "
+ + "preventing us to map a chunk of %d bytes. %sMore information: "
+ + "https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]",
+ originalMessage,
+ resourceDescription,
+ bufSize,
+ moreInfo),
+ originalCause);
+ newIoe.setStackTrace(ioe.getStackTrace());
+ return newIoe;
+ }
- default IOException convertMapFailedIOException(
- IOException ioe, String resourceDescription, long bufSize) {
- final String originalMessage;
- final Throwable originalCause;
- if (ioe.getCause() instanceof OutOfMemoryError) {
- // nested OOM confuses users, because it's "incorrect", just print a plain message:
- originalMessage = "Map failed";
- originalCause = null;
- } else {
- originalMessage = ioe.getMessage();
- originalCause = ioe.getCause();
- }
- final String moreInfo;
- if (!Constants.JRE_IS_64BIT) {
- moreInfo =
- "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. ";
- } else if (Constants.WINDOWS) {
- moreInfo =
- "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. ";
- } else if (Constants.LINUX) {
- moreInfo =
- "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. ";
- } else {
- moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). ";
- }
- final IOException newIoe =
- new IOException(
- String.format(
- Locale.ENGLISH,
- "%s: %s [this may be caused by lack of enough unfragmented virtual address space "
- + "or too restrictive virtual memory limits enforced by the operating system, "
- + "preventing us to map a chunk of %d bytes. %sMore information: "
- + "https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]",
- originalMessage,
- resourceDescription,
- bufSize,
- moreInfo),
- originalCause);
- newIoe.setStackTrace(ioe.getStackTrace());
- return newIoe;
+ private static int checkMaxPermits(int maxPermits) {
+ if (RefCountedSharedArena.validMaxPermits(maxPermits)) {
+ return maxPermits;
}
+ Logger.getLogger(MMapDirectory.class.getName())
+ .warning(
+ "Invalid value for sysprop "
+ + MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP
+ + ", must be positive and <= 0x07FF. The default value will be used.");
+ return RefCountedSharedArena.DEFAULT_MAX_PERMITS;
}
private static int getSharedArenaMaxPermitsSysprop() {
@@ -370,41 +471,11 @@ public class MMapDirectory extends FSDirectory {
return ret;
}
- private static <A> MMapIndexInputProvider<A> lookupProvider() {
- final var maxPermits = getSharedArenaMaxPermitsSysprop();
- final var lookup = MethodHandles.lookup();
- try {
- final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
- // we use method handles, so we do not need to deal with setAccessible as we have private
- // access through the lookup:
- final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class, int.class));
- try {
- @SuppressWarnings("unchecked")
- var res = (MMapIndexInputProvider<A>) constr.invoke(maxPermits);
- return res;
- } catch (RuntimeException | Error e) {
- throw e;
- } catch (Throwable th) {
- throw new AssertionError(th);
- }
- } catch (NoSuchMethodException | IllegalAccessException e) {
- throw new LinkageError(
- "MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
- } catch (ClassNotFoundException cnfe) {
- throw new LinkageError("MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);
- }
- }
-
/**
* Returns true, if MMapDirectory uses the platform's {@code madvise()} syscall to advise how OS
* kernel should handle paging after opening a file.
*/
public static boolean supportsMadvise() {
- return PROVIDER.supportsMadvise();
- }
-
- static {
- PROVIDER = lookupProvider();
- DEFAULT_MAX_CHUNK_SIZE = PROVIDER.getDefaultMaxChunkSize();
+ return NATIVE_ACCESS.isPresent();
}
}
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentAccessInput.java b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentAccessInput.java
similarity index 100%
rename from lucene/core/src/java21/org/apache/lucene/store/MemorySegmentAccessInput.java
rename to lucene/core/src/java/org/apache/lucene/store/MemorySegmentAccessInput.java
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java
similarity index 97%
rename from lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
rename to lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java
index 5404e1aafc7..64bed7cc9f1 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.store;
+import static java.util.function.Predicate.not;
+
import java.io.EOFException;
import java.io.IOException;
import java.lang.foreign.Arena;
@@ -37,9 +39,7 @@ import org.apache.lucene.util.IOConsumer;
* <p>For efficiency, this class requires that the segment size are a power-of-two (<code>
* chunkSizePower</code>).
*/
-@SuppressWarnings("preview")
-abstract class MemorySegmentIndexInput extends IndexInput
- implements RandomAccessInput, MemorySegmentAccessInput {
+abstract class MemorySegmentIndexInput extends IndexInput implements MemorySegmentAccessInput {
static final ValueLayout.OfByte LAYOUT_BYTE = ValueLayout.JAVA_BYTE;
static final ValueLayout.OfShort LAYOUT_LE_SHORT =
ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
@@ -129,16 +129,9 @@ abstract class MemorySegmentIndexInput extends IndexInput
}
// in Java 22 or later we can check the isAlive status of all segments
// (see https://bugs.openjdk.org/browse/JDK-8310644):
- if (Arrays.stream(segments).allMatch(s -> s.scope().isAlive()) == false) {
+ if (Arrays.stream(segments).anyMatch(not(s -> s.scope().isAlive()))) {
return new AlreadyClosedException("Already closed: " + this);
}
- // fallback for Java 21: ISE can be thrown by MemorySegment and contains "closed" in message:
- if (e instanceof IllegalStateException
- && e.getMessage() != null
- && e.getMessage().contains("closed")) {
- // the check is on message only, so preserve original cause for debugging:
- return new AlreadyClosedException("Already closed: " + this, e);
- }
// otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to
// the IndexInput method):
throw e;
diff --git a/lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java b/lucene/core/src/java/org/apache/lucene/store/NativeAccess.java
similarity index 98%
rename from lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java
rename to lucene/core/src/java/org/apache/lucene/store/NativeAccess.java
index affc0e2ac71..838a40f6aa6 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java
+++ b/lucene/core/src/java/org/apache/lucene/store/NativeAccess.java
@@ -21,7 +21,6 @@ import java.lang.foreign.MemorySegment;
import java.util.Optional;
import org.apache.lucene.util.Constants;
-@SuppressWarnings("preview")
abstract class NativeAccess {
/** Invoke the {@code madvise} call for the given {@link MemorySegment}. */
diff --git a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java b/lucene/core/src/java/org/apache/lucene/store/PosixNativeAccess.java
similarity index 99%
rename from lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java
rename to lucene/core/src/java/org/apache/lucene/store/PosixNativeAccess.java
index 05eb6157118..477383fda32 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java
+++ b/lucene/core/src/java/org/apache/lucene/store/PosixNativeAccess.java
@@ -27,7 +27,6 @@ import java.util.Locale;
import java.util.Optional;
import java.util.logging.Logger;
-@SuppressWarnings("preview")
final class PosixNativeAccess extends NativeAccess {
private static final Logger LOG = Logger.getLogger(PosixNativeAccess.class.getName());
diff --git a/lucene/core/src/java21/org/apache/lucene/store/RefCountedSharedArena.java b/lucene/core/src/java/org/apache/lucene/store/RefCountedSharedArena.java
similarity index 99%
rename from lucene/core/src/java21/org/apache/lucene/store/RefCountedSharedArena.java
rename to lucene/core/src/java/org/apache/lucene/store/RefCountedSharedArena.java
index e89de280a3c..2d6dd9b737a 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/RefCountedSharedArena.java
+++ b/lucene/core/src/java/org/apache/lucene/store/RefCountedSharedArena.java
@@ -38,7 +38,6 @@ import java.util.concurrent.atomic.AtomicInteger;
* permitted and {@link #acquire()} returns false. This is independent of the actual number of the
* ref count.
*/
-@SuppressWarnings("preview")
final class RefCountedSharedArena implements Arena {
// default maximum permits
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
deleted file mode 100644
index f740b1a372b..00000000000
--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.store;
-
-import java.io.IOException;
-import java.lang.foreign.Arena;
-import java.lang.foreign.MemorySegment;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileChannel.MapMode;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.logging.Logger;
-import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.Unwrappable;
-
-@SuppressWarnings("preview")
-final class MemorySegmentIndexInputProvider
- implements MMapDirectory.MMapIndexInputProvider<
- ConcurrentHashMap<String, RefCountedSharedArena>> {
-
- private final Optional<NativeAccess> nativeAccess;
- private final int sharedArenaMaxPermits;
-
- MemorySegmentIndexInputProvider(int maxPermits) {
- this.nativeAccess = NativeAccess.getImplementation();
- this.sharedArenaMaxPermits = checkMaxPermits(maxPermits);
- }
-
- @Override
- public IndexInput openInput(
- Path path,
- int chunkSizePower,
- ReadAdvice readAdvice,
- boolean confined,
- boolean preload,
- Optional<String> group,
- ConcurrentHashMap<String, RefCountedSharedArena> arenas)
- throws IOException {
- final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")";
-
- // Work around for JDK-8259028: we need to unwrap our test-only file system layers
- path = Unwrappable.unwrapAll(path);
-
- boolean success = false;
- final Arena arena = confined ? Arena.ofConfined() : getSharedArena(group, arenas);
- try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {
- final long fileSize = fc.size();
- final IndexInput in =
- MemorySegmentIndexInput.newInstance(
- resourceDescription,
- arena,
- map(arena, resourceDescription, fc, readAdvice, chunkSizePower, preload, fileSize),
- fileSize,
- chunkSizePower,
- confined);
- success = true;
- return in;
- } finally {
- if (success == false) {
- arena.close();
- }
- }
- }
-
- @Override
- public long getDefaultMaxChunkSize() {
- return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);
- }
-
- @Override
- public boolean supportsMadvise() {
- return nativeAccess.isPresent();
- }
-
- private final MemorySegment[] map(
- Arena arena,
- String resourceDescription,
- FileChannel fc,
- ReadAdvice readAdvice,
- int chunkSizePower,
- boolean preload,
- long length)
- throws IOException {
- if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
- throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription);
-
- final long chunkSize = 1L << chunkSizePower;
-
- // we always allocate one more segments, the last one may be a 0 byte one
- final int nrSegments = (int) (length >>> chunkSizePower) + 1;
-
- final MemorySegment[] segments = new MemorySegment[nrSegments];
-
- long startOffset = 0L;
- for (int segNr = 0; segNr < nrSegments; segNr++) {
- final long segSize =
- (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset);
- final MemorySegment segment;
- try {
- segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena);
- } catch (IOException ioe) {
- throw convertMapFailedIOException(ioe, resourceDescription, segSize);
- }
- // if preload apply it without madvise.
- // skip madvise if the address of our segment is not page-aligned (small segments due to
- // internal FileChannel logic)
- if (preload) {
- segment.load();
- } else if (readAdvice != ReadAdvice.NORMAL
- && nativeAccess.filter(na -> segment.address() % na.getPageSize() == 0).isPresent()) {
- // No need to madvise with ReadAdvice.NORMAL since it is the OS' default read advice.
- nativeAccess.get().madvise(segment, readAdvice);
- }
- segments[segNr] = segment;
- startOffset += segSize;
- }
- return segments;
- }
-
- @Override
- public ConcurrentHashMap<String, RefCountedSharedArena> attachment() {
- return new ConcurrentHashMap<>();
- }
-
- private static int checkMaxPermits(int maxPermits) {
- if (RefCountedSharedArena.validMaxPermits(maxPermits)) {
- return maxPermits;
- }
- Logger.getLogger(MemorySegmentIndexInputProvider.class.getName())
- .warning(
- "Invalid value for sysprop "
- + MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP
- + ", must be positive and <= 0x07FF. The default value will be used.");
- return RefCountedSharedArena.DEFAULT_MAX_PERMITS;
- }
-
- /**
- * Gets an arena for the given group, potentially aggregating files from the same segment into a
- * single ref counted shared arena. A ref counted shared arena, if created will be added to the
- * given arenas map.
- */
- private Arena getSharedArena(
- Optional<String> group, ConcurrentHashMap<String, RefCountedSharedArena> arenas) {
- if (group.isEmpty()) {
- return Arena.ofShared();
- }
-
- String key = group.get();
- var refCountedArena =
- arenas.computeIfAbsent(
- key, s -> new RefCountedSharedArena(s, () -> arenas.remove(s), sharedArenaMaxPermits));
- if (refCountedArena.acquire()) {
- return refCountedArena;
- } else {
- return arenas.compute(
- key,
- (s, v) -> {
- if (v != null && v.acquire()) {
- return v;
- } else {
- v = new RefCountedSharedArena(s, () -> arenas.remove(s), sharedArenaMaxPermits);
- v.acquire(); // guaranteed to succeed
- return v;
- }
- });
- }
- }
-}
diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java
similarity index 100%
rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java
rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java
diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java
similarity index 100%
rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java
rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java
diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java
similarity index 100%
rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java
rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java
diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.java
similarity index 100%
rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.java
rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.java
diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorConstants.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorConstants.java
similarity index 100%
rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorConstants.java
rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorConstants.java
diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
similarity index 100%
rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
similarity index 100%
rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
index 971847a2ec0..a91ae5414ba 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
@@ -25,7 +25,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
@@ -227,10 +226,7 @@ public class TestMMapDirectory extends BaseDirectoryTestCase {
}
}
- if (!(dir.attachment instanceof ConcurrentHashMap<?, ?> map)) {
- throw new AssertionError("unexpected attachment: " + dir.attachment);
- }
- assertEquals(0, map.size());
+ assertEquals(0, dir.arenas.size());
}
}
@@ -279,10 +275,7 @@ public class TestMMapDirectory extends BaseDirectoryTestCase {
closeable.close();
}
- if (!(dir.attachment instanceof ConcurrentHashMap<?, ?> map)) {
- throw new AssertionError("unexpected attachment: " + dir.attachment);
- }
- assertEquals(0, map.size());
+ assertEquals(0, dir.arenas.size());
}
}
diff --git a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java
index 9c412664053..f19819941db 100644
--- a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java
+++ b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java
@@ -207,10 +207,10 @@ public class TestModularLayer extends AbstractLuceneDistributionTest {
ClassLoader loader = layer.findLoader(coreModuleId);
- final Set<Integer> mrJarVersions = Set.of(21);
- final Integer baseVersion = 21;
+ final Set<Integer> mrJarVersions = Set.of(23);
+ final Integer baseVersion = 23;
- // the Java 21 PanamaVectorizationProvider must always be in main section of JAR file:
+ // the Java 23 PanamaVectorizationProvider must always be in main section of JAR file:
final String panamaClassName =
"org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.class";
Assertions.assertThat(loader.getResource(panamaClassName)).isNotNull();
@@ -224,11 +224,6 @@ public class TestModularLayer extends AbstractLuceneDistributionTest {
loader.getResource("META-INF/versions/" + v + panamaClassName))
.isNotNull();
});
-
- // you must be always able to load MemorySegmentIndexInput:
- Assertions.assertThat(
- loader.loadClass("org.apache.lucene.store.MemorySegmentIndexInput"))
- .isNotNull();
});
}