Skip to content

Apache Lucene open-source search software

Remove ByteBufferIndexInput and update all Panama implementations (MMap and Vector) to Java 21 (#13146)

Uwe Schindlere446904c6147
diff --git a/gradle/generation/extract-jdk-apis.gradle b/gradle/generation/extract-jdk-apis.gradle
index 8ebdd6b1e33..1371fe9d391 100644
--- a/gradle/generation/extract-jdk-apis.gradle
+++ b/gradle/generation/extract-jdk-apis.gradle
@@ -20,14 +20,14 @@ def resources = scriptResources(buildscript)
configure(rootProject) {
ext {
// also change this in extractor tool: ExtractForeignAPI
- vectorIncubatorJavaVersions = [ JavaVersion.VERSION_20, JavaVersion.VERSION_21, JavaVersion.VERSION_22 ] as Set
+ vectorIncubatorJavaVersions = [ JavaVersion.VERSION_21, JavaVersion.VERSION_22 ] as Set
}
}
configure(project(":lucene:core")) {
ext {
apijars = layout.projectDirectory.dir("src/generated/jdk")
- mrjarJavaVersions = [ 19, 20, 21 ]
+ mrjarJavaVersions = [ 21 ]
}
configurations {
diff --git a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java
index 82f43c1336f..c84c8f16996 100644
--- a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java
+++ b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java
@@ -54,9 +54,7 @@ public final class ExtractJdkApis {
private static final String PATTERN_VECTOR_VM_INTERNALS = "java.base/jdk/internal/vm/vector/VectorSupport{,$Vector,$VectorMask,$VectorPayload,$VectorShuffle}";
static final Map<Integer,List<String>> CLASSFILE_PATTERNS = Map.of(
- 19, List.of(PATTERN_PANAMA_FOREIGN),
- 20, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR),
- 21, List.of(PATTERN_PANAMA_FOREIGN)
+ 21, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR)
);
public static void main(String... args) throws IOException {
@@ -143,7 +141,7 @@ public final class ExtractJdkApis {
@Override
public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
- super.visit(Opcodes.V11, access, name, signature, superName, interfaces);
+ super.visit(Opcodes.V21, access, name, signature, superName, interfaces);
if (isVisible(access)) {
classesToInclude.add(name);
}
@@ -188,10 +186,6 @@ public final class ExtractJdkApis {
}
}
- @Override
- public void visitPermittedSubclass(String c) {
- }
-
}
}
diff --git a/gradle/testing/defaults-tests.gradle b/gradle/testing/defaults-tests.gradle
index 89c1d49c3c7..dc42a2216b4 100644
--- a/gradle/testing/defaults-tests.gradle
+++ b/gradle/testing/defaults-tests.gradle
@@ -126,7 +126,7 @@ allprojects {
// Lucene needs to optional modules at runtime, which we want to enforce for testing
// (if the runner JVM does not support them, it will fail tests):
- jvmArgs '--add-modules', 'jdk.unsupported,jdk.management'
+ jvmArgs '--add-modules', 'jdk.management'
// Enable the vector incubator module on supported Java versions:
if (rootProject.vectorIncubatorJavaVersions.contains(rootProject.runtimeJavaVersion)) {
diff --git a/gradle/testing/randomization/policies/replicator-tests.policy b/gradle/testing/randomization/policies/replicator-tests.policy
index 4e78f4650e5..9a8f6aa7026 100644
--- a/gradle/testing/randomization/policies/replicator-tests.policy
+++ b/gradle/testing/randomization/policies/replicator-tests.policy
@@ -54,9 +54,6 @@ grant {
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
- // needed to test unmap hack on platforms that support it
- permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
- permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};
// Permissions to support ant build
diff --git a/gradle/testing/randomization/policies/tests.policy b/gradle/testing/randomization/policies/tests.policy
index 46a351f4c61..85edc3e572f 100644
--- a/gradle/testing/randomization/policies/tests.policy
+++ b/gradle/testing/randomization/policies/tests.policy
@@ -50,9 +50,6 @@ grant {
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
- // needed to test unmap hack on platforms that support it
- permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
- permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
// needed by cyberneko usage by benchmarks on J9
permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util";
diff --git a/lucene/core.tests/src/test/org/apache/lucene/core/tests/TestMMap.java b/lucene/core.tests/src/test/org/apache/lucene/core/tests/TestMMap.java
deleted file mode 100644
index fcaae01e982..00000000000
--- a/lucene/core.tests/src/test/org/apache/lucene/core/tests/TestMMap.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.core.tests;
-
-import org.apache.lucene.store.MMapDirectory;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.junit.Assert;
-
-public class TestMMap extends LuceneTestCase {
- public void testUnmapSupported() {
- final Module module = MMapDirectory.class.getModule();
- Assert.assertTrue("Lucene Core is not loaded as module", module.isNamed());
- Assert.assertTrue(
- "Lucene Core can't read 'jdk.unsupported' module",
- module.getLayer().findModule("jdk.unsupported").map(module::canRead).orElse(false));
-
- // check that MMapDirectory can unmap by running the autodetection logic:
- Assert.assertTrue(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, MMapDirectory.UNMAP_SUPPORTED);
- }
-}
diff --git a/lucene/core/src/generated/jdk/jdk19.apijar b/lucene/core/src/generated/jdk/jdk19.apijar
deleted file mode 100644
index cce32639b05..00000000000
Binary files a/lucene/core/src/generated/jdk/jdk19.apijar and /dev/null differ
diff --git a/lucene/core/src/generated/jdk/jdk20.apijar b/lucene/core/src/generated/jdk/jdk20.apijar
deleted file mode 100644
index d6340fd0938..00000000000
Binary files a/lucene/core/src/generated/jdk/jdk20.apijar and /dev/null differ
diff --git a/lucene/core/src/generated/jdk/jdk21.apijar b/lucene/core/src/generated/jdk/jdk21.apijar
index 69906ccf4b0..8120f23b8b4 100644
Binary files a/lucene/core/src/generated/jdk/jdk21.apijar and b/lucene/core/src/generated/jdk/jdk21.apijar differ
diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.java
index 684f32960e2..04ee0727541 100644
--- a/lucene/core/src/java/module-info.java
+++ b/lucene/core/src/java/module-info.java
@@ -21,7 +21,6 @@ import org.apache.lucene.codecs.lucene99.Lucene99Codec;
@SuppressWarnings("module") // the test framework is compiled after the core...
module org.apache.lucene.core {
requires java.logging;
- requires static jdk.unsupported; // this is optional but without it MMapDirectory won't be enabled
requires static jdk.management; // this is optional but explicit declaration is recommended
exports org.apache.lucene.analysis;
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 8c09cf98b9e..2bfa25fea43 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -74,7 +74,6 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.LockValidatingDirectoryWrapper;
-import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.Accountable;
@@ -1303,12 +1302,6 @@ public class IndexWriter
+ Version.LATEST.toString()
+ "\n"
+ config.toString());
- final StringBuilder unmapInfo =
- new StringBuilder(Boolean.toString(MMapDirectory.UNMAP_SUPPORTED));
- if (!MMapDirectory.UNMAP_SUPPORTED) {
- unmapInfo.append(" (").append(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON).append(")");
- }
- infoStream.message("IW", "MMapDirectory.UNMAP_SUPPORTED=" + unmapInfo);
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
index a1f07c6a043..158cdde5e0e 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
@@ -17,11 +17,9 @@
package org.apache.lucene.internal.vectorization;
-import java.lang.Runtime.Version;
import java.lang.StackWalker.StackFrame;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
-import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.OptionalInt;
@@ -97,20 +95,11 @@ public abstract class VectorizationProvider {
private static final Logger LOG = Logger.getLogger(VectorizationProvider.class.getName());
- /** The minimal version of Java that has the bugfix for JDK-8301190. */
- private static final Version VERSION_JDK8301190_FIXED = Version.parse("20.0.2");
-
// visible for tests
static VectorizationProvider lookup(boolean testMode) {
final int runtimeVersion = Runtime.version().feature();
- if (runtimeVersion >= 20 && runtimeVersion <= 22) {
- // is locale sane (only buggy in Java 20)
- if (isAffectedByJDK8301190()) {
- LOG.warning(
- "Java runtime is using a buggy default locale; Java vector incubator API can't be enabled: "
- + Locale.getDefault());
- return new DefaultVectorizationProvider();
- }
+ assert runtimeVersion >= 21;
+ if (runtimeVersion <= 22) {
// only use vector module with Hotspot VM
if (!Constants.IS_HOTSPOT_VM) {
LOG.warning(
@@ -169,13 +158,11 @@ public abstract class VectorizationProvider {
} catch (ClassNotFoundException cnfe) {
throw new LinkageError("PanamaVectorizationProvider is missing in Lucene JAR file", cnfe);
}
- } else if (runtimeVersion >= 23) {
- LOG.warning(
- "You are running with Java 23 or later. To make full use of the Vector API, please update Apache Lucene.");
- } else if (lookupVectorModule().isPresent()) {
+ } else {
LOG.warning(
- "Java vector incubator module was enabled by command line flags, but your Java version is too old: "
- + runtimeVersion);
+ "You are running with unsupported Java "
+ + runtimeVersion
+ + ". To make full use of the Vector API, please update Apache Lucene.");
}
return new DefaultVectorizationProvider();
}
@@ -189,15 +176,6 @@ public abstract class VectorizationProvider {
.findModule("jdk.incubator.vector");
}
- /**
- * Check if runtime is affected by JDK-8301190 (avoids assertion when default language is say
- * "tr").
- */
- private static boolean isAffectedByJDK8301190() {
- return VERSION_JDK8301190_FIXED.compareToIgnoreOptional(Runtime.version()) > 0
- && !Objects.equals("I", "i".toUpperCase(Locale.getDefault()));
- }
-
// add all possible callers here as FQCN:
private static final Set<String> VALID_CALLERS = Set.of("org.apache.lucene.util.VectorUtil");
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBufferGuard.java b/lucene/core/src/java/org/apache/lucene/store/ByteBufferGuard.java
deleted file mode 100644
index da5db6586e2..00000000000
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBufferGuard.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.store;
-
-import java.io.IOException;
-import java.lang.invoke.VarHandle;
-import java.nio.ByteBuffer;
-import java.nio.FloatBuffer;
-import java.nio.IntBuffer;
-import java.nio.LongBuffer;
-
-/**
- * A guard that is created for every {@link ByteBufferIndexInput} that tries on best effort to
- * reject any access to the {@link ByteBuffer} behind, once it is unmapped. A single instance of
- * this is used for the original and all clones, so once the original is closed and unmapped all
- * clones also throw {@link AlreadyClosedException}, triggered by a {@link NullPointerException}.
- *
- * <p>This code tries to hopefully flush any CPU caches using a store-store barrier. It also yields
- * the current thread to give other threads a chance to finish in-flight requests...
- */
-final class ByteBufferGuard {
-
- /**
- * Pass in an implementation of this interface to cleanup ByteBuffers. MMapDirectory implements
- * this to allow unmapping of bytebuffers with private Java APIs.
- */
- @FunctionalInterface
- interface BufferCleaner {
- void freeBuffer(String resourceDescription, ByteBuffer b) throws IOException;
- }
-
- private final String resourceDescription;
- private final BufferCleaner cleaner;
-
- /** Not volatile; see comments on visibility below! */
- private boolean invalidated = false;
-
- /**
- * Creates an instance to be used for a single {@link ByteBufferIndexInput} which must be shared
- * by all of its clones.
- */
- public ByteBufferGuard(String resourceDescription, BufferCleaner cleaner) {
- this.resourceDescription = resourceDescription;
- this.cleaner = cleaner;
- }
-
- /** Invalidates this guard and unmaps (if supported). */
- public void invalidateAndUnmap(ByteBuffer... bufs) throws IOException {
- if (cleaner != null) {
- invalidated = true;
- // This call should hopefully flush any CPU caches and as a result make
- // the "invalidated" field update visible to other threads. We specifically
- // don't make "invalidated" field volatile for performance reasons, hoping the
- // JVM won't optimize away reads of that field and hardware should ensure
- // caches are in sync after this call.
- // For previous implementation (based on `AtomicInteger#lazySet(0)`) see LUCENE-7409.
- VarHandle.fullFence();
- // we give other threads a bit of time to finish reads on their ByteBuffer...:
- Thread.yield();
- // finally unmap the ByteBuffers:
- for (ByteBuffer b : bufs) {
- cleaner.freeBuffer(resourceDescription, b);
- }
- }
- }
-
- public boolean isInvalidated() {
- return invalidated;
- }
-
- private void ensureValid() {
- if (invalidated) {
- // this triggers an AlreadyClosedException in ByteBufferIndexInput:
- throw new NullPointerException();
- }
- }
-
- public void getBytes(ByteBuffer receiver, int pos, byte[] dst, int offset, int length) {
- ensureValid();
- receiver.get(pos, dst, offset, length);
- }
-
- public void getBytes(ByteBuffer receiver, byte[] dst, int offset, int length) {
- ensureValid();
- receiver.get(dst, offset, length);
- }
-
- public byte getByte(ByteBuffer receiver) {
- ensureValid();
- return receiver.get();
- }
-
- public short getShort(ByteBuffer receiver) {
- ensureValid();
- return receiver.getShort();
- }
-
- public int getInt(ByteBuffer receiver) {
- ensureValid();
- return receiver.getInt();
- }
-
- public long getLong(ByteBuffer receiver) {
- ensureValid();
- return receiver.getLong();
- }
-
- public byte getByte(ByteBuffer receiver, int pos) {
- ensureValid();
- return receiver.get(pos);
- }
-
- public short getShort(ByteBuffer receiver, int pos) {
- ensureValid();
- return receiver.getShort(pos);
- }
-
- public int getInt(ByteBuffer receiver, int pos) {
- ensureValid();
- return receiver.getInt(pos);
- }
-
- public long getLong(ByteBuffer receiver, int pos) {
- ensureValid();
- return receiver.getLong(pos);
- }
-
- public void getLongs(LongBuffer receiver, long[] dst, int offset, int length) {
- ensureValid();
- receiver.get(dst, offset, length);
- }
-
- public void getInts(IntBuffer receiver, int[] dst, int offset, int length) {
- ensureValid();
- receiver.get(dst, offset, length);
- }
-
- public void getFloats(FloatBuffer receiver, float[] dst, int offset, int length) {
- ensureValid();
- receiver.get(dst, offset, length);
- }
-}
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java
deleted file mode 100644
index 04c78a10242..00000000000
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java
+++ /dev/null
@@ -1,732 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.store;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.nio.BufferUnderflowException;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.FloatBuffer;
-import java.nio.IntBuffer;
-import java.nio.LongBuffer;
-
-/**
- * Base IndexInput implementation that uses an array of ByteBuffers to represent a file.
- *
- * <p>Because Java's ByteBuffer uses an int to address the values, it's necessary to access a file
- * greater Integer.MAX_VALUE in size using multiple byte buffers.
- *
- * <p>For efficiency, this class requires that the buffers are a power-of-two (<code>chunkSizePower
- * </code>).
- *
- * @deprecated This class was made public for internal reasons ({@code instanceof} checks). In
- * {@link MMapDirectory} it was replaced by {@code MemorySegment} based {@link IndexInput}
- * implementations and will be no longer required in Lucene 10.
- * @lucene.internal
- */
-@Deprecated
-public abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessInput {
- private static final FloatBuffer EMPTY_FLOATBUFFER = FloatBuffer.allocate(0);
- private static final LongBuffer EMPTY_LONGBUFFER = LongBuffer.allocate(0);
- private static final IntBuffer EMPTY_INTBUFFER = IntBuffer.allocate(0);
-
- protected final long length;
- protected final long chunkSizeMask;
- protected final int chunkSizePower;
- protected final ByteBufferGuard guard;
-
- protected ByteBuffer[] buffers;
- protected int curBufIndex = -1;
- protected ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
- private LongBuffer[] curLongBufferViews;
- private IntBuffer[] curIntBufferViews;
- private FloatBuffer[] curFloatBufferViews;
-
- protected boolean isClone = false;
-
- public static ByteBufferIndexInput newInstance(
- String resourceDescription,
- ByteBuffer[] buffers,
- long length,
- int chunkSizePower,
- ByteBufferGuard guard) {
- if (buffers.length == 1) {
- return new SingleBufferImpl(resourceDescription, buffers[0], length, chunkSizePower, guard);
- } else {
- return new MultiBufferImpl(resourceDescription, buffers, 0, length, chunkSizePower, guard);
- }
- }
-
- ByteBufferIndexInput(
- String resourceDescription,
- ByteBuffer[] buffers,
- long length,
- int chunkSizePower,
- ByteBufferGuard guard) {
- super(resourceDescription);
- this.buffers = buffers;
- this.length = length;
- this.chunkSizePower = chunkSizePower;
- this.chunkSizeMask = (1L << chunkSizePower) - 1L;
- this.guard = guard;
- assert chunkSizePower >= 0 && chunkSizePower <= 30;
- assert (length >>> chunkSizePower) < Integer.MAX_VALUE;
- }
-
- protected void setCurBuf(ByteBuffer curBuf) {
- this.curBuf = curBuf;
- curLongBufferViews = null;
- curFloatBufferViews = null;
- curIntBufferViews = null;
- }
-
- // the unused parameter is just to silence javac about unused variables
- RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
- throws IOException {
- if (pos < 0L) {
- return new IllegalArgumentException(action + " negative position (pos=" + pos + "): " + this);
- } else {
- throw new EOFException(action + " past EOF (pos=" + pos + "): " + this);
- }
- }
-
- AlreadyClosedException alreadyClosed(NullPointerException npe) {
- // we use NPE to signal if this input is closed (to not have checks everywhere). If NPE happens,
- // we check the "is closed" condition explicitly by checking that our "buffers" are null or
- // the guard was invalidated.
- if (this.buffers == null || this.curBuf == null || guard.isInvalidated()) {
- return new AlreadyClosedException("Already closed: " + this);
- }
- // otherwise rethrow unmodified NPE (as it possibly a bug with passing a null parameter to the
- // IndexInput method):
- throw npe;
- }
-
- @Override
- public final byte readByte() throws IOException {
- try {
- return guard.getByte(curBuf);
- } catch (
- @SuppressWarnings("unused")
- BufferUnderflowException e) {
- do {
- curBufIndex++;
- if (curBufIndex >= buffers.length) {
- throw new EOFException("read past EOF: " + this);
- }
- setCurBuf(buffers[curBufIndex]);
- curBuf.position(0);
- } while (!curBuf.hasRemaining());
- return guard.getByte(curBuf);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final void readBytes(byte[] b, int offset, int len) throws IOException {
- try {
- guard.getBytes(curBuf, b, offset, len);
- } catch (
- @SuppressWarnings("unused")
- BufferUnderflowException e) {
- int curAvail = curBuf.remaining();
- while (len > curAvail) {
- guard.getBytes(curBuf, b, offset, curAvail);
- len -= curAvail;
- offset += curAvail;
- curBufIndex++;
- if (curBufIndex >= buffers.length) {
- throw new EOFException("read past EOF: " + this);
- }
- setCurBuf(buffers[curBufIndex]);
- curBuf.position(0);
- curAvail = curBuf.remaining();
- }
- guard.getBytes(curBuf, b, offset, len);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final void readLongs(long[] dst, int offset, int length) throws IOException {
- // ByteBuffer#getLong could work but it has some per-long overhead and there
- // is no ByteBuffer#getLongs to read multiple longs at once. So we use the
- // below trick in order to be able to leverage LongBuffer#get(long[]) to
- // read multiple longs at once with as little overhead as possible.
- if (curLongBufferViews == null) {
- // readLELongs is only used for postings today, so we compute the long
- // views lazily so that other data-structures don't have to pay for the
- // associated initialization/memory overhead.
- curLongBufferViews = new LongBuffer[Long.BYTES];
- for (int i = 0; i < Long.BYTES; ++i) {
- // Compute a view for each possible alignment. We cache these views
- // because #asLongBuffer() has some cost that we don't want to pay on
- // each invocation of #readLELongs.
- if (i < curBuf.limit()) {
- curLongBufferViews[i] =
- curBuf.duplicate().position(i).order(ByteOrder.LITTLE_ENDIAN).asLongBuffer();
- } else {
- curLongBufferViews[i] = EMPTY_LONGBUFFER;
- }
- }
- }
- try {
- final int position = curBuf.position();
- guard.getLongs(
- curLongBufferViews[position & 0x07].position(position >>> 3), dst, offset, length);
- // if the above call succeeded, then we know the below sum cannot overflow
- curBuf.position(position + (length << 3));
- } catch (
- @SuppressWarnings("unused")
- BufferUnderflowException e) {
- super.readLongs(dst, offset, length);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readInts(int[] dst, int offset, int length) throws IOException {
- // See notes about readLongs above
- if (curIntBufferViews == null) {
- curIntBufferViews = new IntBuffer[Integer.BYTES];
- for (int i = 0; i < Integer.BYTES; ++i) {
- if (i < curBuf.limit()) {
- curIntBufferViews[i] =
- curBuf.duplicate().position(i).order(ByteOrder.LITTLE_ENDIAN).asIntBuffer();
- } else {
- curIntBufferViews[i] = EMPTY_INTBUFFER;
- }
- }
- }
- try {
- final int position = curBuf.position();
- guard.getInts(
- curIntBufferViews[position & 0x03].position(position >>> 2), dst, offset, length);
- // if the above call succeeded, then we know the below sum cannot overflow
- curBuf.position(position + (length << 2));
- } catch (
- @SuppressWarnings("unused")
- BufferUnderflowException e) {
- super.readInts(dst, offset, length);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final void readFloats(float[] floats, int offset, int len) throws IOException {
- // See notes about readLongs above
- if (curFloatBufferViews == null) {
- curFloatBufferViews = new FloatBuffer[Float.BYTES];
- for (int i = 0; i < Float.BYTES; ++i) {
- // Compute a view for each possible alignment.
- if (i < curBuf.limit()) {
- ByteBuffer dup = curBuf.duplicate().order(ByteOrder.LITTLE_ENDIAN);
- dup.position(i);
- curFloatBufferViews[i] = dup.asFloatBuffer();
- } else {
- curFloatBufferViews[i] = EMPTY_FLOATBUFFER;
- }
- }
- }
- try {
- final int position = curBuf.position();
- FloatBuffer floatBuffer = curFloatBufferViews[position & 0x03];
- floatBuffer.position(position >>> 2);
- guard.getFloats(floatBuffer, floats, offset, len);
- // if the above call succeeded, then we know the below sum cannot overflow
- curBuf.position(position + (len << 2));
- } catch (
- @SuppressWarnings("unused")
- BufferUnderflowException e) {
- super.readFloats(floats, offset, len);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final short readShort() throws IOException {
- try {
- return guard.getShort(curBuf);
- } catch (
- @SuppressWarnings("unused")
- BufferUnderflowException e) {
- return super.readShort();
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final int readInt() throws IOException {
- try {
- return guard.getInt(curBuf);
- } catch (
- @SuppressWarnings("unused")
- BufferUnderflowException e) {
- return super.readInt();
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final int readVInt() throws IOException {
- // this can make JVM less confused (see LUCENE-10366)
- return super.readVInt();
- }
-
- @Override
- public final long readVLong() throws IOException {
- // this can make JVM less confused (see LUCENE-10366)
- return super.readVLong();
- }
-
- @Override
- public final long readLong() throws IOException {
- try {
- return guard.getLong(curBuf);
- } catch (
- @SuppressWarnings("unused")
- BufferUnderflowException e) {
- return super.readLong();
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public long getFilePointer() {
- try {
- return (((long) curBufIndex) << chunkSizePower) + curBuf.position();
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void seek(long pos) throws IOException {
- // we use >> here to preserve negative, so we will catch AIOOBE,
- // in case pos + offset overflows.
- final int bi = (int) (pos >> chunkSizePower);
- try {
- if (bi == curBufIndex) {
- curBuf.position((int) (pos & chunkSizeMask));
- } else {
- final ByteBuffer b = buffers[bi];
- b.position((int) (pos & chunkSizeMask));
- // write values, on exception all is unchanged
- this.curBufIndex = bi;
- setCurBuf(b);
- }
- } catch (ArrayIndexOutOfBoundsException | IllegalArgumentException e) {
- throw handlePositionalIOOBE(e, "seek", pos);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public byte readByte(long pos) throws IOException {
- try {
- final int bi = (int) (pos >> chunkSizePower);
- return guard.getByte(buffers[bi], (int) (pos & chunkSizeMask));
- } catch (IndexOutOfBoundsException ioobe) {
- throw handlePositionalIOOBE(ioobe, "read", pos);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- // used only by random access methods to handle reads across boundaries
- private void setPos(long pos, int bi) throws IOException {
- try {
- final ByteBuffer b = buffers[bi];
- b.position((int) (pos & chunkSizeMask));
- this.curBufIndex = bi;
- setCurBuf(b);
- } catch (ArrayIndexOutOfBoundsException | IllegalArgumentException aioobe) {
- throw handlePositionalIOOBE(aioobe, "read", pos);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readBytes(long pos, byte[] bytes, int offset, int len) throws IOException {
- int bi = (int) (pos >> chunkSizePower);
- int bufferPos = (int) (pos & chunkSizeMask);
- try {
- int curAvail = Math.min(buffers[bi].capacity() - bufferPos, len);
- while (len > curAvail) {
- guard.getBytes(buffers[bi], bufferPos, bytes, offset, curAvail);
- len -= curAvail;
- offset += curAvail;
- bi++;
- if (bi >= buffers.length) {
- throw new EOFException("read past EOF: " + this);
- }
- bufferPos = 0;
- curAvail = Math.min(len, buffers[bi].capacity());
- }
- guard.getBytes(buffers[bi], bufferPos, bytes, offset, curAvail);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- final int bi = (int) (pos >> chunkSizePower);
- try {
- return guard.getShort(buffers[bi], (int) (pos & chunkSizeMask));
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException ioobe) {
- // either it's a boundary, or read past EOF, fall back:
- setPos(pos, bi);
- return readShort();
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- final int bi = (int) (pos >> chunkSizePower);
- try {
- return guard.getInt(buffers[bi], (int) (pos & chunkSizeMask));
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException ioobe) {
- // either it's a boundary, or read past EOF, fall back:
- setPos(pos, bi);
- return readInt();
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- final int bi = (int) (pos >> chunkSizePower);
- try {
- return guard.getLong(buffers[bi], (int) (pos & chunkSizeMask));
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException ioobe) {
- // either it's a boundary, or read past EOF, fall back:
- setPos(pos, bi);
- return readLong();
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final long length() {
- return length;
- }
-
- @Override
- public final ByteBufferIndexInput clone() {
- final ByteBufferIndexInput clone = buildSlice((String) null, 0L, this.length);
- try {
- clone.seek(getFilePointer());
- } catch (IOException ioe) {
- throw new AssertionError(ioe);
- }
-
- return clone;
- }
-
- /**
- * Creates a slice of this index input, with the given description, offset, and length. The slice
- * is seeked to the beginning.
- */
- @Override
- public final ByteBufferIndexInput slice(String sliceDescription, long offset, long length) {
- if (offset < 0 || length < 0 || offset + length > this.length) {
- throw new IllegalArgumentException(
- "slice() "
- + sliceDescription
- + " out of bounds: offset="
- + offset
- + ",length="
- + length
- + ",fileLength="
- + this.length
- + ": "
- + this);
- }
-
- return buildSlice(sliceDescription, offset, length);
- }
-
- /** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */
- protected ByteBufferIndexInput buildSlice(String sliceDescription, long offset, long length) {
- if (buffers == null || guard.isInvalidated()) {
- throw alreadyClosed(null);
- }
-
- final ByteBuffer[] newBuffers = buildSlice(buffers, offset, length);
- final int ofs = (int) (offset & chunkSizeMask);
-
- final ByteBufferIndexInput clone =
- newCloneInstance(getFullSliceDescription(sliceDescription), newBuffers, ofs, length);
- clone.isClone = true;
-
- return clone;
- }
-
- /**
- * Factory method that creates a suitable implementation of this class for the given ByteBuffers.
- */
- @SuppressWarnings("resource")
- protected ByteBufferIndexInput newCloneInstance(
- String newResourceDescription, ByteBuffer[] newBuffers, int offset, long length) {
- if (newBuffers.length == 1) {
- newBuffers[0].position(offset);
- return new SingleBufferImpl(
- newResourceDescription,
- newBuffers[0].slice().order(ByteOrder.LITTLE_ENDIAN),
- length,
- chunkSizePower,
- this.guard);
- } else {
- return new MultiBufferImpl(
- newResourceDescription, newBuffers, offset, length, chunkSizePower, guard);
- }
- }
-
- /**
- * Returns a sliced view from a set of already-existing buffers: the last buffer's limit() will be
- * correct, but you must deal with offset separately (the first buffer will not be adjusted)
- */
- private ByteBuffer[] buildSlice(ByteBuffer[] buffers, long offset, long length) {
- final long sliceEnd = offset + length;
-
- final int startIndex = (int) (offset >>> chunkSizePower);
- final int endIndex = (int) (sliceEnd >>> chunkSizePower);
-
- // we always allocate one more slice, the last one may be a 0 byte one
- final ByteBuffer[] slices = new ByteBuffer[endIndex - startIndex + 1];
-
- for (int i = 0; i < slices.length; i++) {
- slices[i] = buffers[startIndex + i].duplicate().order(ByteOrder.LITTLE_ENDIAN);
- }
-
- // set the last buffer's limit for the sliced view.
- slices[slices.length - 1].limit((int) (sliceEnd & chunkSizeMask));
-
- return slices;
- }
-
- @Override
- public final void close() throws IOException {
- try {
- if (buffers == null) return;
-
- // make local copy, then un-set early
- final ByteBuffer[] bufs = buffers;
- unsetBuffers();
-
- if (isClone) return;
-
- // tell the guard to invalidate and later unmap the bytebuffers (if supported):
- guard.invalidateAndUnmap(bufs);
- } finally {
- unsetBuffers();
- }
- }
-
- /** Called to remove all references to byte buffers, so we can throw AlreadyClosed on NPE. */
- private void unsetBuffers() {
- buffers = null;
- curBuf = null;
- curBufIndex = 0;
- curLongBufferViews = null;
- curIntBufferViews = null;
- }
-
- /** Optimization of ByteBufferIndexInput for when there is only one buffer */
- static final class SingleBufferImpl extends ByteBufferIndexInput {
-
- SingleBufferImpl(
- String resourceDescription,
- ByteBuffer buffer,
- long length,
- int chunkSizePower,
- ByteBufferGuard guard) {
- super(resourceDescription, new ByteBuffer[] {buffer}, length, chunkSizePower, guard);
- this.curBufIndex = 0;
- assert buffer.order() == ByteOrder.LITTLE_ENDIAN;
- setCurBuf(buffer);
- buffer.position(0);
- }
-
- // TODO: investigate optimizing readByte() & Co?
-
- @Override
- public void seek(long pos) throws IOException {
- try {
- curBuf.position((int) pos);
- } catch (IllegalArgumentException e) {
- throw handlePositionalIOOBE(e, "seek", pos);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public long getFilePointer() {
- try {
- return curBuf.position();
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public byte readByte(long pos) throws IOException {
- try {
- return guard.getByte(curBuf, (int) pos);
- } catch (IllegalArgumentException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readBytes(long pos, byte[] bytes, int offset, int len) throws IOException {
- try {
- guard.getBytes(curBuf, (int) pos, bytes, offset, len);
- } catch (IllegalArgumentException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- try {
- return guard.getShort(curBuf, (int) pos);
- } catch (IllegalArgumentException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- try {
- return guard.getInt(curBuf, (int) pos);
- } catch (IllegalArgumentException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- try {
- return guard.getLong(curBuf, (int) pos);
- } catch (IllegalArgumentException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException e) {
- throw alreadyClosed(e);
- }
- }
- }
-
- /** This class adds offset support to ByteBufferIndexInput, which is needed for slices. */
- static final class MultiBufferImpl extends ByteBufferIndexInput {
- private final int offset;
-
- MultiBufferImpl(
- String resourceDescription,
- ByteBuffer[] buffers,
- int offset,
- long length,
- int chunkSizePower,
- ByteBufferGuard guard) {
- super(resourceDescription, buffers, length, chunkSizePower, guard);
- this.offset = offset;
- try {
- seek(0L);
- } catch (IOException ioe) {
- throw new AssertionError(ioe);
- }
- }
-
- @Override
- RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
- throws IOException {
- return super.handlePositionalIOOBE(unused, action, pos - offset);
- }
-
- @Override
- public void seek(long pos) throws IOException {
- assert pos >= 0L : "negative position";
- super.seek(pos + offset);
- }
-
- @Override
- public long getFilePointer() {
- return super.getFilePointer() - offset;
- }
-
- @Override
- public byte readByte(long pos) throws IOException {
- return super.readByte(pos + offset);
- }
-
- @Override
- public void readBytes(long pos, byte[] bytes, int offset, int len) throws IOException {
- super.readBytes(pos + this.offset, bytes, offset, len);
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- return super.readShort(pos + offset);
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- return super.readInt(pos + offset);
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- return super.readLong(pos + offset);
- }
-
- @Override
- protected ByteBufferIndexInput buildSlice(String sliceDescription, long ofs, long length) {
- return super.buildSlice(sliceDescription, this.offset + ofs, length);
- }
- }
-}
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDirectory.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDirectory.java
index dbefd6a03cc..dd31d13c0c9 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDirectory.java
@@ -25,7 +25,6 @@ import java.nio.file.NoSuchFileException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;
@@ -36,7 +35,6 @@ import java.util.function.Function;
import java.util.function.Supplier;
import java.util.zip.CRC32;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.util.BitUtil;
/**
* A {@link ByteBuffer}-based {@link Directory} implementation that can be used to store index files
@@ -82,45 +80,6 @@ public final class ByteBuffersDirectory extends BaseDirectory {
public static final BiFunction<String, ByteBuffersDataOutput, IndexInput> OUTPUT_AS_BYTE_ARRAY =
OUTPUT_AS_ONE_BUFFER;
- /**
- * Use {@link ByteBufferIndexInput} for reading, otherwise identical to {@link
- * #OUTPUT_AS_MANY_BUFFERS}.
- *
- * @deprecated Use {@link #OUTPUT_AS_MANY_BUFFERS} instead.
- */
- @Deprecated
- public static final BiFunction<String, ByteBuffersDataOutput, IndexInput>
- OUTPUT_AS_MANY_BUFFERS_LUCENE =
- (fileName, output) -> {
- List<ByteBuffer> bufferList = output.toBufferList();
- bufferList.add(ByteBuffer.allocate(0).order(ByteOrder.LITTLE_ENDIAN));
-
- int chunkSizePower;
- int blockSize = ByteBuffersDataInput.determineBlockPage(bufferList);
- if (blockSize == 0) {
- chunkSizePower = 30;
- } else {
- chunkSizePower =
- Integer.numberOfTrailingZeros(BitUtil.nextHighestPowerOfTwo(blockSize));
- }
-
- String inputName =
- String.format(
- Locale.ROOT,
- "%s (file=%s)",
- ByteBuffersDirectory.class.getSimpleName(),
- fileName);
-
- ByteBufferGuard guard =
- new ByteBufferGuard("none", (String resourceDescription, ByteBuffer b) -> {});
- return ByteBufferIndexInput.newInstance(
- inputName,
- bufferList.toArray(new ByteBuffer[bufferList.size()]),
- output.size(),
- chunkSizePower,
- guard);
- };
-
private final Function<String, String> tempFileName =
new Function<String, String>() {
private final AtomicLong counter = new AtomicLong();
diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
index 22da71384ff..413e22c45ae 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
@@ -44,31 +44,27 @@ import org.apache.lucene.util.IOUtils;
/**
* Base class for Directory implementations that store index files in the file system. <a
- * id="subclasses"></a> There are currently three core subclasses:
+ * id="subclasses"></a> There are currently two core subclasses:
*
* <ul>
+ * <li>{@link MMapDirectory} uses memory-mapped IO when reading. This is a good choice if you have
+ * plenty of virtual memory relative to your index size, eg if you are running on a 64 bit
+ * JRE, or you are running on a 32 bit JRE but your index sizes are small enough to fit into
+ * the virtual memory space. This class will use the modern {@link
+ * java.lang.foreign.MemorySegment} API available since Java 21 which allows to safely unmap
+ * previously mmapped files after closing the {@link IndexInput}s. There is no need to enable
+ * the "preview feature" of your Java version; it works out of box with some compilation
+ * tricks. For more information about the foreign memory API read documentation of the {@link
+ * java.lang.foreign} package and <a
+ * href="https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html">Uwe's blog
+ * post</a>.
* <li>{@link NIOFSDirectory} uses java.nio's FileChannel's positional io when reading to avoid
* synchronization when reading from the same file. Unfortunately, due to a Windows-only <a
* href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734">Sun JRE bug</a> this is a
* poor choice for Windows, but on all other platforms this is the preferred choice.
* Applications using {@link Thread#interrupt()} or {@link Future#cancel(boolean)} should use
- * {@code RAFDirectory} instead. See {@link NIOFSDirectory} java doc for details.
- * <li>{@link MMapDirectory} uses memory-mapped IO when reading. This is a good choice if you have
- * plenty of virtual memory relative to your index size, eg if you are running on a 64 bit
- * JRE, or you are running on a 32 bit JRE but your index sizes are small enough to fit into
- * the virtual memory space. Java has currently the limitation of not being able to unmap
- * files from user code. The files are unmapped, when GC releases the byte buffers. Due to <a
- * href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">this bug</a> in Sun's
- * JRE, MMapDirectory's {@link IndexInput#close} is unable to close the underlying OS file
- * handle. Only when GC finally collects the underlying objects, which could be quite some
- * time later, will the file handle be closed. This will consume additional transient disk
- * usage: on Windows, attempts to delete or overwrite the files will result in an exception;
- * on other platforms, which typically have a &quot;delete on last close&quot; semantics,
- * while such operations will succeed, the bytes are still consuming space on disk. For many
- * applications this limitation is not a problem (e.g. if you have plenty of disk space, and
- * you don't rely on overwriting files on Windows) but it's still an important limitation to
- * be aware of. This class supplies a (possibly dangerous) workaround mentioned in the bug
- * report, which may fail on non-Sun JVMs.
+ * {@code RAFDirectory} instead, which is provided in the {@code misc} module. See {@link
+ * NIOFSDirectory} javadoc for details.
* </ul>
*
* <p>Unfortunately, because of system peculiarities, there is no single overall best
@@ -157,7 +153,7 @@ public abstract class FSDirectory extends BaseDirectory {
/** Just like {@link #open(Path)}, but allows you to also specify a custom {@link LockFactory}. */
public static FSDirectory open(Path path, LockFactory lockFactory) throws IOException {
- if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
+ if (Constants.JRE_IS_64BIT) {
return new MMapDirectory(path, lockFactory);
} else {
return new NIOFSDirectory(path, lockFactory);
diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
index 3bf8683266a..5638f7c33d0 100644
--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
@@ -21,15 +21,10 @@ import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.file.Path;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
import java.util.Locale;
-import java.util.Optional;
import java.util.concurrent.Future;
import java.util.function.BiPredicate;
-import java.util.logging.Logger;
import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.SuppressForbidden;
/**
* File-based {@link Directory} implementation that uses mmap for reading, and {@link
@@ -40,45 +35,18 @@ import org.apache.lucene.util.SuppressForbidden;
* plenty of virtual address space, e.g. by using a 64 bit JRE, or a 32 bit JRE with indexes that
* are guaranteed to fit within the address space. On 32 bit platforms also consult {@link
* #MMapDirectory(Path, LockFactory, long)} if you have problems with mmap failing because of
- * fragmented address space. If you get an OutOfMemoryException, it is recommended to reduce the
- * chunk size, until it works.
+ * fragmented address space. If you get an {@link IOException} about mapping failed, it is
+ * recommended to reduce the chunk size, until it works.
*
* <p>This class supports preloading files into physical memory upon opening. This can help improve
* performance of searches on a cold page cache at the expense of slowing down opening an index. See
* {@link #setPreload(BiPredicate)} for more details.
*
- * <p>Due to <a href="https://bugs.openjdk.org/browse/JDK-4724038">this bug</a> in OpenJDK,
- * MMapDirectory's {@link IndexInput#close} is unable to close the underlying OS file handle. Only
- * when GC finally collects the underlying objects, which could be quite some time later, will the
- * file handle be closed.
- *
- * <p>This will consume additional transient disk usage: on Windows, attempts to delete or overwrite
- * the files will result in an exception; on other platforms, which typically have a &quot;delete on
- * last close&quot; semantics, while such operations will succeed, the bytes are still consuming
- * space on disk. For many applications this limitation is not a problem (e.g. if you have plenty of
- * disk space, and you don't rely on overwriting files on Windows) but it's still an important
- * limitation to be aware of.
- *
- * <p>This class supplies the workaround mentioned in the bug report, which may fail on
- * non-Oracle/OpenJDK JVMs. It forcefully unmaps the buffer on close by using an undocumented
- * internal cleanup functionality. If {@link #UNMAP_SUPPORTED} is <code>true</code>, the workaround
- * will be automatically enabled (with no guarantees; if you discover any problems, you can disable
- * it by using system property {@link #ENABLE_UNMAP_HACK_SYSPROP}).
- *
- * <p>For the hack to work correct, the following requirements need to be fulfilled: The used JVM
- * must be at least Oracle Java / OpenJDK. In addition, the following permissions need to be granted
- * to {@code lucene-core.jar} in your <a
- * href="http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html">policy
- * file</a>:
- *
- * <ul>
- * <li>{@code permission java.lang.reflect.ReflectPermission "suppressAccessChecks";}
- * <li>{@code permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";}
- * </ul>
- *
- * <p>On exactly <b>Java 19 / 20 / 21</b> this class will use the modern {@code MemorySegment} API
- * which allows to safely unmap (if you discover any problems with this preview API, you can disable
- * it by using system property {@link #ENABLE_MEMORY_SEGMENTS_SYSPROP}).
+ * <p>This class will use the modern {@link java.lang.foreign.MemorySegment} API available since
+ * Java 21 which allows to safely unmap previously mmapped files after closing the {@link
+ * IndexInput}s. There is no need to enable the "preview feature" of your Java version; it works out
+ * of box with some compilation tricks. For more information about the foreign memory API read
+ * documentation of the {@link java.lang.foreign} package.
*
* <p><b>NOTE:</b> Accessing this class either directly or indirectly from a thread while it's
* interrupted can close the underlying channel immediately if at the same time the thread is
@@ -91,13 +59,11 @@ import org.apache.lucene.util.SuppressForbidden;
* synchronize on the <code>MMapDirectory</code> instance as this may cause deadlock; use your own
* (non-Lucene) objects instead.
*
- * @see <a href="http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html">Blog post
+ * @see <a href="https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html">Blog post
* about MMapDirectory</a>
*/
public class MMapDirectory extends FSDirectory {
- private static final Logger LOG = Logger.getLogger(MMapDirectory.class.getName());
-
/**
* Argument for {@link #setPreload(BiPredicate)} that configures all files to be preloaded upon
* opening them.
@@ -123,35 +89,12 @@ public class MMapDirectory extends FSDirectory {
* Default max chunk size:
*
* <ul>
- * <li>16 GiBytes for 64 bit <b>Java 19 / 20 / 21</b> JVMs
- * <li>1 GiBytes for other 64 bit JVMs
+ * <li>16 GiBytes for 64 bit JVMs
* <li>256 MiBytes for 32 bit JVMs
* </ul>
*/
public static final long DEFAULT_MAX_CHUNK_SIZE;
- /**
- * This sysprop allows to control the workaround/hack for unmapping the buffers from address space
- * after closing {@link IndexInput}. By default it is enabled; set to {@code false} to disable the
- * unmap hack globally. On command line pass {@code
- * -Dorg.apache.lucene.store.MMapDirectory.enableUnmapHack=false} to disable.
- *
- * @lucene.internal
- */
- public static final String ENABLE_UNMAP_HACK_SYSPROP =
- "org.apache.lucene.store.MMapDirectory.enableUnmapHack";
-
- /**
- * This sysprop allows to control if {@code MemorySegment} API should be used on supported Java
- * versions. By default it is enabled; set to {@code false} to use legacy {@code ByteBuffer}
- * implementation. On command line pass {@code
- * -Dorg.apache.lucene.store.MMapDirectory.enableMemorySegments=false} to disable.
- *
- * @lucene.internal
- */
- public static final String ENABLE_MEMORY_SEGMENTS_SYSPROP =
- "org.apache.lucene.store.MMapDirectory.enableMemorySegments";
-
final int chunkSizePower;
/**
@@ -255,25 +198,12 @@ public class MMapDirectory extends FSDirectory {
// visible for tests:
static final MMapIndexInputProvider PROVIDER;
- /** <code>true</code>, if this platform supports unmapping mmapped files. */
- public static final boolean UNMAP_SUPPORTED;
-
- /**
- * if {@link #UNMAP_SUPPORTED} is {@code false}, this contains the reason why unmapping is not
- * supported.
- */
- public static final String UNMAP_NOT_SUPPORTED_REASON;
-
interface MMapIndexInputProvider {
IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
throws IOException;
long getDefaultMaxChunkSize();
- boolean isUnmapSupported();
-
- String getUnmapNotSupportedReason();
-
default IOException convertMapFailedIOException(
IOException ioe, String resourceDescription, long bufSize) {
final String originalMessage;
@@ -317,63 +247,30 @@ public class MMapDirectory extends FSDirectory {
}
}
- // Extracted to a method to be able to apply the SuppressForbidden annotation
- @SuppressWarnings("removal")
- @SuppressForbidden(reason = "security manager")
- private static <T> T doPrivileged(PrivilegedAction<T> action) {
- return AccessController.doPrivileged(action);
- }
-
- private static boolean checkMemorySegmentsSysprop() {
- try {
- return Optional.ofNullable(System.getProperty(ENABLE_MEMORY_SEGMENTS_SYSPROP))
- .map(Boolean::valueOf)
- .orElse(Boolean.TRUE);
- } catch (
- @SuppressWarnings("unused")
- SecurityException ignored) {
- LOG.warning(
- "Cannot read sysprop "
- + ENABLE_MEMORY_SEGMENTS_SYSPROP
- + ", so MemorySegments will be enabled by default, if possible.");
- return true;
- }
- }
-
private static MMapIndexInputProvider lookupProvider() {
- if (checkMemorySegmentsSysprop() == false) {
- return new MappedByteBufferIndexInputProvider();
- }
final var lookup = MethodHandles.lookup();
- final int runtimeVersion = Runtime.version().feature();
- if (runtimeVersion >= 19) {
+ try {
+ final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
+ // we use method handles, so we do not need to deal with setAccessible as we have private
+ // access through the lookup:
+ final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class));
try {
- final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
- // we use method handles, so we do not need to deal with setAccessible as we have private
- // access through the lookup:
- final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class));
- try {
- return (MMapIndexInputProvider) constr.invoke();
- } catch (RuntimeException | Error e) {
- throw e;
- } catch (Throwable th) {
- throw new AssertionError(th);
- }
- } catch (NoSuchMethodException | IllegalAccessException e) {
- throw new LinkageError(
- "MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
- } catch (ClassNotFoundException cnfe) {
- throw new LinkageError(
- "MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);
+ return (MMapIndexInputProvider) constr.invoke();
+ } catch (RuntimeException | Error e) {
+ throw e;
+ } catch (Throwable th) {
+ throw new AssertionError(th);
}
+ } catch (NoSuchMethodException | IllegalAccessException e) {
+ throw new LinkageError(
+ "MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
+ } catch (ClassNotFoundException cnfe) {
+ throw new LinkageError("MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);
}
- return new MappedByteBufferIndexInputProvider();
}
static {
- PROVIDER = doPrivileged(MMapDirectory::lookupProvider);
+ PROVIDER = lookupProvider();
DEFAULT_MAX_CHUNK_SIZE = PROVIDER.getDefaultMaxChunkSize();
- UNMAP_SUPPORTED = PROVIDER.isUnmapSupported();
- UNMAP_NOT_SUPPORTED_REASON = PROVIDER.getUnmapNotSupportedReason();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/MappedByteBufferIndexInputProvider.java b/lucene/core/src/java/org/apache/lucene/store/MappedByteBufferIndexInputProvider.java
deleted file mode 100644
index 9f631ada5c5..00000000000
--- a/lucene/core/src/java/org/apache/lucene/store/MappedByteBufferIndexInputProvider.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.store;
-
-import static java.lang.invoke.MethodHandles.lookup;
-import static java.lang.invoke.MethodType.methodType;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandle;
-import java.lang.invoke.MethodHandles.Lookup;
-import java.lang.reflect.Field;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileChannel.MapMode;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.logging.Logger;
-import org.apache.lucene.store.ByteBufferGuard.BufferCleaner;
-import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.SuppressForbidden;
-
-final class MappedByteBufferIndexInputProvider implements MMapDirectory.MMapIndexInputProvider {
-
- private static final Logger LOG =
- Logger.getLogger(MappedByteBufferIndexInputProvider.class.getName());
-
- private final BufferCleaner cleaner;
-
- private final boolean unmapSupported;
- private final String unmapNotSupportedReason;
-
- public MappedByteBufferIndexInputProvider() {
- final Object hack = unmapHackImpl();
- if (hack instanceof BufferCleaner) {
- cleaner = (BufferCleaner) hack;
- unmapSupported = true;
- unmapNotSupportedReason = null;
- } else {
- cleaner = null;
- unmapSupported = false;
- unmapNotSupportedReason = hack.toString();
- LOG.warning(unmapNotSupportedReason);
- }
- }
-
- @Override
- public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
- throws IOException {
- if (chunkSizePower > 30) {
- throw new IllegalArgumentException(
- "ByteBufferIndexInput cannot use a chunk size of >1 GiBytes.");
- }
-
- final String resourceDescription = "ByteBufferIndexInput(path=\"" + path.toString() + "\")";
-
- try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {
- final long fileSize = fc.size();
- return ByteBufferIndexInput.newInstance(
- resourceDescription,
- map(resourceDescription, fc, chunkSizePower, preload, fileSize),
- fileSize,
- chunkSizePower,
- new ByteBufferGuard(resourceDescription, cleaner));
- }
- }
-
- @Override
- public long getDefaultMaxChunkSize() {
- return Constants.JRE_IS_64BIT ? (1L << 30) : (1L << 28);
- }
-
- @Override
- public boolean isUnmapSupported() {
- return unmapSupported;
- }
-
- @Override
- public String getUnmapNotSupportedReason() {
- return unmapNotSupportedReason;
- }
-
- /** Maps a file into a set of buffers */
- final ByteBuffer[] map(
- String resourceDescription, FileChannel fc, int chunkSizePower, boolean preload, long length)
- throws IOException {
- if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
- throw new IllegalArgumentException(
- "RandomAccessFile too big for chunk size: " + resourceDescription);
-
- final long chunkSize = 1L << chunkSizePower;
-
- // we always allocate one more buffer, the last one may be a 0 byte one
- final int nrBuffers = (int) (length >>> chunkSizePower) + 1;
-
- final ByteBuffer[] buffers = new ByteBuffer[nrBuffers];
-
- long startOffset = 0L;
- for (int bufNr = 0; bufNr < nrBuffers; bufNr++) {
- final int bufSize =
- (int) ((length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset));
- final MappedByteBuffer buffer;
- try {
- buffer = fc.map(MapMode.READ_ONLY, startOffset, bufSize);
- buffer.order(ByteOrder.LITTLE_ENDIAN);
- } catch (IOException ioe) {
- throw convertMapFailedIOException(ioe, resourceDescription, bufSize);
- }
- if (preload) {
- buffer.load();
- }
- buffers[bufNr] = buffer;
- startOffset += bufSize;
- }
-
- return buffers;
- }
-
- private static boolean checkUnmapHackSysprop() {
- try {
- return Optional.ofNullable(System.getProperty(MMapDirectory.ENABLE_UNMAP_HACK_SYSPROP))
- .map(Boolean::valueOf)
- .orElse(Boolean.TRUE);
- } catch (
- @SuppressWarnings("unused")
- SecurityException ignored) {
- LOG.warning(
- "Cannot read sysprop "
- + MMapDirectory.ENABLE_UNMAP_HACK_SYSPROP
- + ", so buffer unmap hack will be enabled by default, if possible.");
- return true;
- }
- }
-
- @SuppressForbidden(reason = "Needs access to sun.misc.Unsafe to enable hack")
- private static Object unmapHackImpl() {
- if (checkUnmapHackSysprop() == false) {
- return "Unmapping was disabled by system property "
- + MMapDirectory.ENABLE_UNMAP_HACK_SYSPROP
- + "=false";
- }
- final Lookup lookup = lookup();
- try {
- // *** sun.misc.Unsafe unmapping (Java 9+) ***
- final Class<?> unsafeClass = lookup.findClass("sun.misc.Unsafe");
- // first check if Unsafe has the right method, otherwise we can give up
- // without doing any security critical stuff:
- final MethodHandle unmapper =
- lookup.findVirtual(
- unsafeClass, "invokeCleaner", methodType(void.class, ByteBuffer.class));
- // fetch the unsafe instance and bind it to the virtual MH:
- final Field f = unsafeClass.getDeclaredField("theUnsafe");
- f.setAccessible(true);
- final Object theUnsafe = f.get(null);
- return newBufferCleaner(unmapper.bindTo(theUnsafe));
- } catch (SecurityException se) {
- return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: "
- + se
- + " [Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\") "
- + " and ReflectPermission(\"suppressAccessChecks\")]";
- } catch (ReflectiveOperationException | RuntimeException e) {
- final Module module = MappedByteBufferIndexInputProvider.class.getModule();
- final ModuleLayer layer = module.getLayer();
- // classpath / unnamed module has no layer, so we need to check:
- if (layer != null
- && layer.findModule("jdk.unsupported").map(module::canRead).orElse(false) == false) {
- return "Unmapping is not supported, because Lucene cannot read 'jdk.unsupported' module "
- + "[please add 'jdk.unsupported' to modular application either by command line or its module descriptor]";
- }
- return "Unmapping is not supported on this platform, because internal Java APIs are not compatible with this Lucene version: "
- + e;
- }
- }
-
- private static BufferCleaner newBufferCleaner(final MethodHandle unmapper) {
- assert Objects.equals(methodType(void.class, ByteBuffer.class), unmapper.type());
- return (String resourceDescription, ByteBuffer buffer) -> {
- if (!buffer.isDirect()) {
- throw new IllegalArgumentException("unmapping only works with direct buffers");
- }
- try {
- unmapper.invokeExact(buffer);
- } catch (Throwable t) {
- throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, t);
- }
- };
- }
-}
diff --git a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
index d32498c1663..e5d93fb74ab 100644
--- a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
@@ -74,7 +74,7 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
ByteBuffersDataOutput::new,
(fileName, content) -> {
cacheSize.addAndGet(content.size());
- return ByteBuffersDirectory.OUTPUT_AS_MANY_BUFFERS_LUCENE.apply(fileName, content);
+ return ByteBuffersDirectory.OUTPUT_AS_MANY_BUFFERS.apply(fileName, content);
});
private final long maxMergeSizeBytes;
diff --git a/lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInput.java
deleted file mode 100644
index ff2ab999319..00000000000
--- a/lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInput.java
+++ /dev/null
@@ -1,683 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.store;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.lang.foreign.MemorySegment;
-import java.lang.foreign.MemorySession;
-import java.lang.foreign.ValueLayout;
-import java.nio.ByteOrder;
-import java.util.Arrays;
-import java.util.Objects;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.GroupVIntUtil;
-
-/**
- * Base IndexInput implementation that uses an array of MemorySegments to represent a file.
- *
- * <p>For efficiency, this class requires that the segment size are a power-of-two (<code>
- * chunkSizePower</code>).
- */
-@SuppressWarnings("preview")
-abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccessInput {
- static final ValueLayout.OfByte LAYOUT_BYTE = ValueLayout.JAVA_BYTE;
- static final ValueLayout.OfShort LAYOUT_LE_SHORT =
- ValueLayout.JAVA_SHORT.withOrder(ByteOrder.LITTLE_ENDIAN).withBitAlignment(8);
- static final ValueLayout.OfInt LAYOUT_LE_INT =
- ValueLayout.JAVA_INT.withOrder(ByteOrder.LITTLE_ENDIAN).withBitAlignment(8);
- static final ValueLayout.OfLong LAYOUT_LE_LONG =
- ValueLayout.JAVA_LONG.withOrder(ByteOrder.LITTLE_ENDIAN).withBitAlignment(8);
- static final ValueLayout.OfFloat LAYOUT_LE_FLOAT =
- ValueLayout.JAVA_FLOAT.withOrder(ByteOrder.LITTLE_ENDIAN).withBitAlignment(8);
-
- final long length;
- final long chunkSizeMask;
- final int chunkSizePower;
- final MemorySession session;
- final MemorySegment[] segments;
-
- int curSegmentIndex = -1;
- MemorySegment
- curSegment; // redundant for speed: segments[curSegmentIndex], also marker if closed!
- long curPosition; // relative to curSegment, not globally
-
- public static MemorySegmentIndexInput newInstance(
- String resourceDescription,
- MemorySession session,
- MemorySegment[] segments,
- long length,
- int chunkSizePower) {
- assert Arrays.stream(segments).map(MemorySegment::session).allMatch(session::equals);
- if (segments.length == 1) {
- return new SingleSegmentImpl(
- resourceDescription, session, segments[0], length, chunkSizePower);
- } else {
- return new MultiSegmentImpl(
- resourceDescription, session, segments, 0, length, chunkSizePower);
- }
- }
-
- private MemorySegmentIndexInput(
- String resourceDescription,
- MemorySession session,
- MemorySegment[] segments,
- long length,
- int chunkSizePower) {
- super(resourceDescription);
- this.session = session;
- this.segments = segments;
- this.length = length;
- this.chunkSizePower = chunkSizePower;
- this.chunkSizeMask = (1L << chunkSizePower) - 1L;
- this.curSegment = segments[0];
- }
-
- void ensureOpen() {
- if (curSegment == null) {
- throw alreadyClosed(null);
- }
- }
-
- // the unused parameter is just to silence javac about unused variables
- RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
- throws IOException {
- if (pos < 0L) {
- return new IllegalArgumentException(action + " negative position (pos=" + pos + "): " + this);
- } else {
- throw new EOFException(action + " past EOF (pos=" + pos + "): " + this);
- }
- }
-
- AlreadyClosedException alreadyClosed(RuntimeException e) {
- // we use NPE to signal if this input is closed (to not have checks everywhere). If NPE happens,
- // we check the "is closed" condition explicitly by checking that our "curSegment" is null.
- // Care must be taken to not leak the NPE to code outside MemorySegmentIndexInput!
- if (this.curSegment == null) {
- return new AlreadyClosedException("Already closed: " + this);
- }
- // ISE can be thrown by MemorySegment and contains "closed" in message:
- if (e instanceof IllegalStateException
- && e.getMessage() != null
- && e.getMessage().contains("closed")) {
- return new AlreadyClosedException("Already closed: " + this, e);
- }
- // otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to
- // the IndexInput method):
- throw e;
- }
-
- @Override
- public final byte readByte() throws IOException {
- try {
- final byte v = curSegment.get(LAYOUT_BYTE, curPosition);
- curPosition++;
- return v;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException e) {
- do {
- curSegmentIndex++;
- if (curSegmentIndex >= segments.length) {
- throw new EOFException("read past EOF: " + this);
- }
- curSegment = segments[curSegmentIndex];
- curPosition = 0L;
- } while (curSegment.byteSize() == 0L);
- final byte v = curSegment.get(LAYOUT_BYTE, curPosition);
- curPosition++;
- return v;
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final void readBytes(byte[] b, int offset, int len) throws IOException {
- try {
- MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len);
- curPosition += len;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException e) {
- readBytesBoundary(b, offset, len);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- private void readBytesBoundary(byte[] b, int offset, int len) throws IOException {
- try {
- long curAvail = curSegment.byteSize() - curPosition;
- while (len > curAvail) {
- MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, (int) curAvail);
- len -= curAvail;
- offset += curAvail;
- curSegmentIndex++;
- if (curSegmentIndex >= segments.length) {
- throw new EOFException("read past EOF: " + this);
- }
- curSegment = segments[curSegmentIndex];
- curPosition = 0L;
- curAvail = curSegment.byteSize();
- }
- MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len);
- curPosition += len;
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readInts(int[] dst, int offset, int length) throws IOException {
- try {
- MemorySegment.copy(curSegment, LAYOUT_LE_INT, curPosition, dst, offset, length);
- curPosition += Integer.BYTES * (long) length;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException iobe) {
- super.readInts(dst, offset, length);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readLongs(long[] dst, int offset, int length) throws IOException {
- try {
- MemorySegment.copy(curSegment, LAYOUT_LE_LONG, curPosition, dst, offset, length);
- curPosition += Long.BYTES * (long) length;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException iobe) {
- super.readLongs(dst, offset, length);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readFloats(float[] dst, int offset, int length) throws IOException {
- try {
- MemorySegment.copy(curSegment, LAYOUT_LE_FLOAT, curPosition, dst, offset, length);
- curPosition += Float.BYTES * (long) length;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException iobe) {
- super.readFloats(dst, offset, length);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final short readShort() throws IOException {
- try {
- final short v = curSegment.get(LAYOUT_LE_SHORT, curPosition);
- curPosition += Short.BYTES;
- return v;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException e) {
- return super.readShort();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final int readInt() throws IOException {
- try {
- final int v = curSegment.get(LAYOUT_LE_INT, curPosition);
- curPosition += Integer.BYTES;
- return v;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException e) {
- return super.readInt();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final int readVInt() throws IOException {
- // this can make JVM less confused (see LUCENE-10366)
- return super.readVInt();
- }
-
- @Override
- public final long readVLong() throws IOException {
- // this can make JVM less confused (see LUCENE-10366)
- return super.readVLong();
- }
-
- @Override
- public final long readLong() throws IOException {
- try {
- final long v = curSegment.get(LAYOUT_LE_LONG, curPosition);
- curPosition += Long.BYTES;
- return v;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException e) {
- return super.readLong();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public long getFilePointer() {
- ensureOpen();
- return (((long) curSegmentIndex) << chunkSizePower) + curPosition;
- }
-
- @Override
- public void seek(long pos) throws IOException {
- ensureOpen();
- // we use >> here to preserve negative, so we will catch AIOOBE,
- // in case pos + offset overflows.
- final int si = (int) (pos >> chunkSizePower);
- try {
- if (si != curSegmentIndex) {
- final MemorySegment seg = segments[si];
- // write values, on exception all is unchanged
- this.curSegmentIndex = si;
- this.curSegment = seg;
- }
- this.curPosition = Objects.checkIndex(pos & chunkSizeMask, curSegment.byteSize() + 1);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "seek", pos);
- }
- }
-
- @Override
- public byte readByte(long pos) throws IOException {
- try {
- final int si = (int) (pos >> chunkSizePower);
- return segments[si].get(LAYOUT_BYTE, pos & chunkSizeMask);
- } catch (IndexOutOfBoundsException ioobe) {
- throw handlePositionalIOOBE(ioobe, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- protected void readGroupVInt(long[] dst, int offset) throws IOException {
- try {
- final int len =
- GroupVIntUtil.readGroupVInt(
- this,
- curSegment.byteSize() - curPosition,
- p -> curSegment.get(LAYOUT_LE_INT, p),
- curPosition,
- dst,
- offset);
- curPosition += len;
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readBytes(long pos, byte[] b, int offset, int len) throws IOException {
- try {
- int si = (int) (pos >> chunkSizePower);
- pos = pos & chunkSizeMask;
- long curAvail = segments[si].byteSize() - pos;
- while (len > curAvail) {
- MemorySegment.copy(segments[si], LAYOUT_BYTE, pos, b, offset, (int) curAvail);
- len -= curAvail;
- offset += curAvail;
- si++;
- if (si >= segments.length) {
- throw new EOFException("read past EOF: " + this);
- }
- pos = 0L;
- curAvail = segments[si].byteSize();
- }
- MemorySegment.copy(segments[si], LAYOUT_BYTE, pos, b, offset, len);
- } catch (IndexOutOfBoundsException ioobe) {
- throw handlePositionalIOOBE(ioobe, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- // used only by random access methods to handle reads across boundaries
- private void setPos(long pos, int si) throws IOException {
- try {
- final MemorySegment seg = segments[si];
- // write values, on exception above all is unchanged
- this.curPosition = pos & chunkSizeMask;
- this.curSegmentIndex = si;
- this.curSegment = seg;
- } catch (IndexOutOfBoundsException ioobe) {
- throw handlePositionalIOOBE(ioobe, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- final int si = (int) (pos >> chunkSizePower);
- try {
- return segments[si].get(LAYOUT_LE_SHORT, pos & chunkSizeMask);
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException ioobe) {
- // either it's a boundary, or read past EOF, fall back:
- setPos(pos, si);
- return readShort();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- final int si = (int) (pos >> chunkSizePower);
- try {
- return segments[si].get(LAYOUT_LE_INT, pos & chunkSizeMask);
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException ioobe) {
- // either it's a boundary, or read past EOF, fall back:
- setPos(pos, si);
- return readInt();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- final int si = (int) (pos >> chunkSizePower);
- try {
- return segments[si].get(LAYOUT_LE_LONG, pos & chunkSizeMask);
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException ioobe) {
- // either it's a boundary, or read past EOF, fall back:
- setPos(pos, si);
- return readLong();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final long length() {
- return length;
- }
-
- @Override
- public final MemorySegmentIndexInput clone() {
- final MemorySegmentIndexInput clone = buildSlice((String) null, 0L, this.length);
- try {
- clone.seek(getFilePointer());
- } catch (IOException ioe) {
- throw new AssertionError(ioe);
- }
-
- return clone;
- }
-
- /**
- * Creates a slice of this index input, with the given description, offset, and length. The slice
- * is seeked to the beginning.
- */
- @Override
- public final MemorySegmentIndexInput slice(String sliceDescription, long offset, long length) {
- if (offset < 0 || length < 0 || offset + length > this.length) {
- throw new IllegalArgumentException(
- "slice() "
- + sliceDescription
- + " out of bounds: offset="
- + offset
- + ",length="
- + length
- + ",fileLength="
- + this.length
- + ": "
- + this);
- }
-
- return buildSlice(sliceDescription, offset, length);
- }
-
- /** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */
- MemorySegmentIndexInput buildSlice(String sliceDescription, long offset, long length) {
- ensureOpen();
-
- final long sliceEnd = offset + length;
- final int startIndex = (int) (offset >>> chunkSizePower);
- final int endIndex = (int) (sliceEnd >>> chunkSizePower);
-
- // we always allocate one more slice, the last one may be a 0 byte one after truncating with
- // asSlice():
- final MemorySegment slices[] = ArrayUtil.copyOfSubArray(segments, startIndex, endIndex + 1);
-
- // set the last segment's limit for the sliced view.
- slices[slices.length - 1] = slices[slices.length - 1].asSlice(0L, sliceEnd & chunkSizeMask);
-
- offset = offset & chunkSizeMask;
-
- final String newResourceDescription = getFullSliceDescription(sliceDescription);
- if (slices.length == 1) {
- return new SingleSegmentImpl(
- newResourceDescription,
- null, // clones don't have a MemorySession, as they can't close)
- slices[0].asSlice(offset, length),
- length,
- chunkSizePower);
- } else {
- return new MultiSegmentImpl(
- newResourceDescription,
- null, // clones don't have a MemorySession, as they can't close)
- slices,
- offset,
- length,
- chunkSizePower);
- }
- }
-
- @Override
- public final void close() throws IOException {
- if (curSegment == null) {
- return;
- }
-
- // the master IndexInput has a MemorySession and is able
- // to release all resources (unmap segments) - a
- // side effect is that other threads still using clones
- // will throw IllegalStateException
- if (session != null) {
- while (session.isAlive()) {
- try {
- session.close();
- break;
- } catch (
- @SuppressWarnings("unused")
- IllegalStateException e) {
- Thread.onSpinWait();
- }
- }
- }
-
- // make sure all accesses to this IndexInput instance throw NPE:
- curSegment = null;
- Arrays.fill(segments, null);
- }
-
- /** Optimization of MemorySegmentIndexInput for when there is only one segment. */
- static final class SingleSegmentImpl extends MemorySegmentIndexInput {
-
- SingleSegmentImpl(
- String resourceDescription,
- MemorySession session,
- MemorySegment segment,
- long length,
- int chunkSizePower) {
- super(resourceDescription, session, new MemorySegment[] {segment}, length, chunkSizePower);
- this.curSegmentIndex = 0;
- }
-
- @Override
- public void seek(long pos) throws IOException {
- ensureOpen();
- try {
- curPosition = Objects.checkIndex(pos, length + 1);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "seek", pos);
- }
- }
-
- @Override
- public long getFilePointer() {
- ensureOpen();
- return curPosition;
- }
-
- @Override
- public byte readByte(long pos) throws IOException {
- try {
- return curSegment.get(LAYOUT_BYTE, pos);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readBytes(long pos, byte[] bytes, int offset, int length) throws IOException {
- try {
- MemorySegment.copy(curSegment, LAYOUT_BYTE, pos, bytes, offset, length);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- try {
- return curSegment.get(LAYOUT_LE_SHORT, pos);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- try {
- return curSegment.get(LAYOUT_LE_INT, pos);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- try {
- return curSegment.get(LAYOUT_LE_LONG, pos);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
- }
-
- /** This class adds offset support to MemorySegmentIndexInput, which is needed for slices. */
- static final class MultiSegmentImpl extends MemorySegmentIndexInput {
- private final long offset;
-
- MultiSegmentImpl(
- String resourceDescription,
- MemorySession session,
- MemorySegment[] segments,
- long offset,
- long length,
- int chunkSizePower) {
- super(resourceDescription, session, segments, length, chunkSizePower);
- this.offset = offset;
- try {
- seek(0L);
- } catch (IOException ioe) {
- throw new AssertionError(ioe);
- }
- assert curSegment != null && curSegmentIndex >= 0;
- }
-
- @Override
- RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
- throws IOException {
- return super.handlePositionalIOOBE(unused, action, pos - offset);
- }
-
- @Override
- public void seek(long pos) throws IOException {
- assert pos >= 0L : "negative position";
- super.seek(pos + offset);
- }
-
- @Override
- public long getFilePointer() {
- return super.getFilePointer() - offset;
- }
-
- @Override
- public byte readByte(long pos) throws IOException {
- return super.readByte(pos + offset);
- }
-
- @Override
- public void readBytes(long pos, byte[] bytes, int offset, int length) throws IOException {
- super.readBytes(pos + this.offset, bytes, offset, length);
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- return super.readShort(pos + offset);
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- return super.readInt(pos + offset);
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- return super.readLong(pos + offset);
- }
-
- @Override
- MemorySegmentIndexInput buildSlice(String sliceDescription, long ofs, long length) {
- return super.buildSlice(sliceDescription, this.offset + ofs, length);
- }
- }
-}
diff --git a/lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
deleted file mode 100644
index deaca72ae41..00000000000
--- a/lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.store;
-
-import java.io.IOException;
-import java.lang.foreign.MemorySegment;
-import java.lang.foreign.MemorySession;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileChannel.MapMode;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.logging.Logger;
-import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.Unwrappable;
-
-@SuppressWarnings("preview")
-final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInputProvider {
-
- public MemorySegmentIndexInputProvider() {
- var log = Logger.getLogger(getClass().getName());
- log.info(
- "Using MemorySegmentIndexInput with Java 19; to disable start with -D"
- + MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP
- + "=false");
- }
-
- @Override
- public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
- throws IOException {
- final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")";
-
- // Work around for JDK-8259028: we need to unwrap our test-only file system layers
- path = Unwrappable.unwrapAll(path);
-
- boolean success = false;
- final MemorySession session = MemorySession.openShared();
- try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {
- final long fileSize = fc.size();
- final IndexInput in =
- MemorySegmentIndexInput.newInstance(
- resourceDescription,
- session,
- map(session, resourceDescription, fc, chunkSizePower, preload, fileSize),
- fileSize,
- chunkSizePower);
- success = true;
- return in;
- } finally {
- if (success == false) {
- session.close();
- }
- }
- }
-
- @Override
- public long getDefaultMaxChunkSize() {
- return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);
- }
-
- @Override
- public boolean isUnmapSupported() {
- return true;
- }
-
- @Override
- public String getUnmapNotSupportedReason() {
- return null;
- }
-
- private final MemorySegment[] map(
- MemorySession session,
- String resourceDescription,
- FileChannel fc,
- int chunkSizePower,
- boolean preload,
- long length)
- throws IOException {
- if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
- throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription);
-
- final long chunkSize = 1L << chunkSizePower;
-
- // we always allocate one more segments, the last one may be a 0 byte one
- final int nrSegments = (int) (length >>> chunkSizePower) + 1;
-
- final MemorySegment[] segments = new MemorySegment[nrSegments];
-
- long startOffset = 0L;
- for (int segNr = 0; segNr < nrSegments; segNr++) {
- final long segSize =
- (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset);
- final MemorySegment segment;
- try {
- segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, session);
- } catch (IOException ioe) {
- throw convertMapFailedIOException(ioe, resourceDescription, segSize);
- }
- if (preload) {
- segment.load();
- }
- segments[segNr] = segment;
- startOffset += segSize;
- }
- return segments;
- }
-}
diff --git a/lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInput.java
deleted file mode 100644
index ba890d8056c..00000000000
--- a/lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInput.java
+++ /dev/null
@@ -1,681 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.store;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.lang.foreign.Arena;
-import java.lang.foreign.MemorySegment;
-import java.lang.foreign.ValueLayout;
-import java.nio.ByteOrder;
-import java.util.Arrays;
-import java.util.Objects;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.GroupVIntUtil;
-
-/**
- * Base IndexInput implementation that uses an array of MemorySegments to represent a file.
- *
- * <p>For efficiency, this class requires that the segment size are a power-of-two (<code>
- * chunkSizePower</code>).
- */
-@SuppressWarnings("preview")
-abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccessInput {
- static final ValueLayout.OfByte LAYOUT_BYTE = ValueLayout.JAVA_BYTE;
- static final ValueLayout.OfShort LAYOUT_LE_SHORT =
- ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
- static final ValueLayout.OfInt LAYOUT_LE_INT =
- ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
- static final ValueLayout.OfLong LAYOUT_LE_LONG =
- ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
- static final ValueLayout.OfFloat LAYOUT_LE_FLOAT =
- ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
-
- final long length;
- final long chunkSizeMask;
- final int chunkSizePower;
- final Arena arena;
- final MemorySegment[] segments;
-
- int curSegmentIndex = -1;
- MemorySegment
- curSegment; // redundant for speed: segments[curSegmentIndex], also marker if closed!
- long curPosition; // relative to curSegment, not globally
-
- public static MemorySegmentIndexInput newInstance(
- String resourceDescription,
- Arena arena,
- MemorySegment[] segments,
- long length,
- int chunkSizePower) {
- assert Arrays.stream(segments).map(MemorySegment::scope).allMatch(arena.scope()::equals);
- if (segments.length == 1) {
- return new SingleSegmentImpl(resourceDescription, arena, segments[0], length, chunkSizePower);
- } else {
- return new MultiSegmentImpl(resourceDescription, arena, segments, 0, length, chunkSizePower);
- }
- }
-
- private MemorySegmentIndexInput(
- String resourceDescription,
- Arena arena,
- MemorySegment[] segments,
- long length,
- int chunkSizePower) {
- super(resourceDescription);
- this.arena = arena;
- this.segments = segments;
- this.length = length;
- this.chunkSizePower = chunkSizePower;
- this.chunkSizeMask = (1L << chunkSizePower) - 1L;
- this.curSegment = segments[0];
- }
-
- void ensureOpen() {
- if (curSegment == null) {
- throw alreadyClosed(null);
- }
- }
-
- // the unused parameter is just to silence javac about unused variables
- RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
- throws IOException {
- if (pos < 0L) {
- return new IllegalArgumentException(action + " negative position (pos=" + pos + "): " + this);
- } else {
- throw new EOFException(action + " past EOF (pos=" + pos + "): " + this);
- }
- }
-
- AlreadyClosedException alreadyClosed(RuntimeException e) {
- // we use NPE to signal if this input is closed (to not have checks everywhere). If NPE happens,
- // we check the "is closed" condition explicitly by checking that our "curSegment" is null.
- // Care must be taken to not leak the NPE to code outside MemorySegmentIndexInput!
- if (this.curSegment == null) {
- return new AlreadyClosedException("Already closed: " + this);
- }
- // ISE can be thrown by MemorySegment and contains "closed" in message:
- if (e instanceof IllegalStateException
- && e.getMessage() != null
- && e.getMessage().contains("closed")) {
- return new AlreadyClosedException("Already closed: " + this, e);
- }
- // otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to
- // the IndexInput method):
- throw e;
- }
-
- @Override
- public final byte readByte() throws IOException {
- try {
- final byte v = curSegment.get(LAYOUT_BYTE, curPosition);
- curPosition++;
- return v;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException e) {
- do {
- curSegmentIndex++;
- if (curSegmentIndex >= segments.length) {
- throw new EOFException("read past EOF: " + this);
- }
- curSegment = segments[curSegmentIndex];
- curPosition = 0L;
- } while (curSegment.byteSize() == 0L);
- final byte v = curSegment.get(LAYOUT_BYTE, curPosition);
- curPosition++;
- return v;
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final void readBytes(byte[] b, int offset, int len) throws IOException {
- try {
- MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len);
- curPosition += len;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException e) {
- readBytesBoundary(b, offset, len);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- private void readBytesBoundary(byte[] b, int offset, int len) throws IOException {
- try {
- long curAvail = curSegment.byteSize() - curPosition;
- while (len > curAvail) {
- MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, (int) curAvail);
- len -= curAvail;
- offset += curAvail;
- curSegmentIndex++;
- if (curSegmentIndex >= segments.length) {
- throw new EOFException("read past EOF: " + this);
- }
- curSegment = segments[curSegmentIndex];
- curPosition = 0L;
- curAvail = curSegment.byteSize();
- }
- MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len);
- curPosition += len;
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readInts(int[] dst, int offset, int length) throws IOException {
- try {
- MemorySegment.copy(curSegment, LAYOUT_LE_INT, curPosition, dst, offset, length);
- curPosition += Integer.BYTES * (long) length;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException iobe) {
- super.readInts(dst, offset, length);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readLongs(long[] dst, int offset, int length) throws IOException {
- try {
- MemorySegment.copy(curSegment, LAYOUT_LE_LONG, curPosition, dst, offset, length);
- curPosition += Long.BYTES * (long) length;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException iobe) {
- super.readLongs(dst, offset, length);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readFloats(float[] dst, int offset, int length) throws IOException {
- try {
- MemorySegment.copy(curSegment, LAYOUT_LE_FLOAT, curPosition, dst, offset, length);
- curPosition += Float.BYTES * (long) length;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException iobe) {
- super.readFloats(dst, offset, length);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final short readShort() throws IOException {
- try {
- final short v = curSegment.get(LAYOUT_LE_SHORT, curPosition);
- curPosition += Short.BYTES;
- return v;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException e) {
- return super.readShort();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final int readInt() throws IOException {
- try {
- final int v = curSegment.get(LAYOUT_LE_INT, curPosition);
- curPosition += Integer.BYTES;
- return v;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException e) {
- return super.readInt();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final int readVInt() throws IOException {
- // this can make JVM less confused (see LUCENE-10366)
- return super.readVInt();
- }
-
- @Override
- public final long readVLong() throws IOException {
- // this can make JVM less confused (see LUCENE-10366)
- return super.readVLong();
- }
-
- @Override
- public final long readLong() throws IOException {
- try {
- final long v = curSegment.get(LAYOUT_LE_LONG, curPosition);
- curPosition += Long.BYTES;
- return v;
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException e) {
- return super.readLong();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public long getFilePointer() {
- ensureOpen();
- return (((long) curSegmentIndex) << chunkSizePower) + curPosition;
- }
-
- @Override
- public void seek(long pos) throws IOException {
- ensureOpen();
- // we use >> here to preserve negative, so we will catch AIOOBE,
- // in case pos + offset overflows.
- final int si = (int) (pos >> chunkSizePower);
- try {
- if (si != curSegmentIndex) {
- final MemorySegment seg = segments[si];
- // write values, on exception all is unchanged
- this.curSegmentIndex = si;
- this.curSegment = seg;
- }
- this.curPosition = Objects.checkIndex(pos & chunkSizeMask, curSegment.byteSize() + 1);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "seek", pos);
- }
- }
-
- @Override
- public byte readByte(long pos) throws IOException {
- try {
- final int si = (int) (pos >> chunkSizePower);
- return segments[si].get(LAYOUT_BYTE, pos & chunkSizeMask);
- } catch (IndexOutOfBoundsException ioobe) {
- throw handlePositionalIOOBE(ioobe, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- protected void readGroupVInt(long[] dst, int offset) throws IOException {
- try {
- final int len =
- GroupVIntUtil.readGroupVInt(
- this,
- curSegment.byteSize() - curPosition,
- p -> curSegment.get(LAYOUT_LE_INT, p),
- curPosition,
- dst,
- offset);
- curPosition += len;
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readBytes(long pos, byte[] b, int offset, int len) throws IOException {
- try {
- int si = (int) (pos >> chunkSizePower);
- pos = pos & chunkSizeMask;
- long curAvail = segments[si].byteSize() - pos;
- while (len > curAvail) {
- MemorySegment.copy(segments[si], LAYOUT_BYTE, pos, b, offset, (int) curAvail);
- len -= curAvail;
- offset += curAvail;
- si++;
- if (si >= segments.length) {
- throw new EOFException("read past EOF: " + this);
- }
- pos = 0L;
- curAvail = segments[si].byteSize();
- }
- MemorySegment.copy(segments[si], LAYOUT_BYTE, pos, b, offset, len);
- } catch (IndexOutOfBoundsException ioobe) {
- throw handlePositionalIOOBE(ioobe, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- // used only by random access methods to handle reads across boundaries
- private void setPos(long pos, int si) throws IOException {
- try {
- final MemorySegment seg = segments[si];
- // write values, on exception above all is unchanged
- this.curPosition = pos & chunkSizeMask;
- this.curSegmentIndex = si;
- this.curSegment = seg;
- } catch (IndexOutOfBoundsException ioobe) {
- throw handlePositionalIOOBE(ioobe, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- final int si = (int) (pos >> chunkSizePower);
- try {
- return segments[si].get(LAYOUT_LE_SHORT, pos & chunkSizeMask);
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException ioobe) {
- // either it's a boundary, or read past EOF, fall back:
- setPos(pos, si);
- return readShort();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- final int si = (int) (pos >> chunkSizePower);
- try {
- return segments[si].get(LAYOUT_LE_INT, pos & chunkSizeMask);
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException ioobe) {
- // either it's a boundary, or read past EOF, fall back:
- setPos(pos, si);
- return readInt();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- final int si = (int) (pos >> chunkSizePower);
- try {
- return segments[si].get(LAYOUT_LE_LONG, pos & chunkSizeMask);
- } catch (
- @SuppressWarnings("unused")
- IndexOutOfBoundsException ioobe) {
- // either it's a boundary, or read past EOF, fall back:
- setPos(pos, si);
- return readLong();
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public final long length() {
- return length;
- }
-
- @Override
- public final MemorySegmentIndexInput clone() {
- final MemorySegmentIndexInput clone = buildSlice((String) null, 0L, this.length);
- try {
- clone.seek(getFilePointer());
- } catch (IOException ioe) {
- throw new AssertionError(ioe);
- }
-
- return clone;
- }
-
- /**
- * Creates a slice of this index input, with the given description, offset, and length. The slice
- * is seeked to the beginning.
- */
- @Override
- public final MemorySegmentIndexInput slice(String sliceDescription, long offset, long length) {
- if (offset < 0 || length < 0 || offset + length > this.length) {
- throw new IllegalArgumentException(
- "slice() "
- + sliceDescription
- + " out of bounds: offset="
- + offset
- + ",length="
- + length
- + ",fileLength="
- + this.length
- + ": "
- + this);
- }
-
- return buildSlice(sliceDescription, offset, length);
- }
-
- /** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */
- MemorySegmentIndexInput buildSlice(String sliceDescription, long offset, long length) {
- ensureOpen();
-
- final long sliceEnd = offset + length;
- final int startIndex = (int) (offset >>> chunkSizePower);
- final int endIndex = (int) (sliceEnd >>> chunkSizePower);
-
- // we always allocate one more slice, the last one may be a 0 byte one after truncating with
- // asSlice():
- final MemorySegment slices[] = ArrayUtil.copyOfSubArray(segments, startIndex, endIndex + 1);
-
- // set the last segment's limit for the sliced view.
- slices[slices.length - 1] = slices[slices.length - 1].asSlice(0L, sliceEnd & chunkSizeMask);
-
- offset = offset & chunkSizeMask;
-
- final String newResourceDescription = getFullSliceDescription(sliceDescription);
- if (slices.length == 1) {
- return new SingleSegmentImpl(
- newResourceDescription,
- null, // clones don't have an Arena, as they can't close)
- slices[0].asSlice(offset, length),
- length,
- chunkSizePower);
- } else {
- return new MultiSegmentImpl(
- newResourceDescription,
- null, // clones don't have an Arena, as they can't close)
- slices,
- offset,
- length,
- chunkSizePower);
- }
- }
-
- @Override
- public final void close() throws IOException {
- if (curSegment == null) {
- return;
- }
-
- // the master IndexInput has an Arena and is able
- // to release all resources (unmap segments) - a
- // side effect is that other threads still using clones
- // will throw IllegalStateException
- if (arena != null) {
- while (arena.scope().isAlive()) {
- try {
- arena.close();
- break;
- } catch (
- @SuppressWarnings("unused")
- IllegalStateException e) {
- Thread.onSpinWait();
- }
- }
- }
-
- // make sure all accesses to this IndexInput instance throw NPE:
- curSegment = null;
- Arrays.fill(segments, null);
- }
-
- /** Optimization of MemorySegmentIndexInput for when there is only one segment. */
- static final class SingleSegmentImpl extends MemorySegmentIndexInput {
-
- SingleSegmentImpl(
- String resourceDescription,
- Arena arena,
- MemorySegment segment,
- long length,
- int chunkSizePower) {
- super(resourceDescription, arena, new MemorySegment[] {segment}, length, chunkSizePower);
- this.curSegmentIndex = 0;
- }
-
- @Override
- public void seek(long pos) throws IOException {
- ensureOpen();
- try {
- curPosition = Objects.checkIndex(pos, length + 1);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "seek", pos);
- }
- }
-
- @Override
- public long getFilePointer() {
- ensureOpen();
- return curPosition;
- }
-
- @Override
- public byte readByte(long pos) throws IOException {
- try {
- return curSegment.get(LAYOUT_BYTE, pos);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public void readBytes(long pos, byte[] bytes, int offset, int length) throws IOException {
- try {
- MemorySegment.copy(curSegment, LAYOUT_BYTE, pos, bytes, offset, length);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- try {
- return curSegment.get(LAYOUT_LE_SHORT, pos);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- try {
- return curSegment.get(LAYOUT_LE_INT, pos);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- try {
- return curSegment.get(LAYOUT_LE_LONG, pos);
- } catch (IndexOutOfBoundsException e) {
- throw handlePositionalIOOBE(e, "read", pos);
- } catch (NullPointerException | IllegalStateException e) {
- throw alreadyClosed(e);
- }
- }
- }
-
- /** This class adds offset support to MemorySegmentIndexInput, which is needed for slices. */
- static final class MultiSegmentImpl extends MemorySegmentIndexInput {
- private final long offset;
-
- MultiSegmentImpl(
- String resourceDescription,
- Arena arena,
- MemorySegment[] segments,
- long offset,
- long length,
- int chunkSizePower) {
- super(resourceDescription, arena, segments, length, chunkSizePower);
- this.offset = offset;
- try {
- seek(0L);
- } catch (IOException ioe) {
- throw new AssertionError(ioe);
- }
- assert curSegment != null && curSegmentIndex >= 0;
- }
-
- @Override
- RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
- throws IOException {
- return super.handlePositionalIOOBE(unused, action, pos - offset);
- }
-
- @Override
- public void seek(long pos) throws IOException {
- assert pos >= 0L : "negative position";
- super.seek(pos + offset);
- }
-
- @Override
- public long getFilePointer() {
- return super.getFilePointer() - offset;
- }
-
- @Override
- public byte readByte(long pos) throws IOException {
- return super.readByte(pos + offset);
- }
-
- @Override
- public void readBytes(long pos, byte[] bytes, int offset, int length) throws IOException {
- super.readBytes(pos + this.offset, bytes, offset, length);
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- return super.readShort(pos + offset);
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- return super.readInt(pos + offset);
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- return super.readLong(pos + offset);
- }
-
- @Override
- MemorySegmentIndexInput buildSlice(String sliceDescription, long ofs, long length) {
- return super.buildSlice(sliceDescription, this.offset + ofs, length);
- }
- }
-}
diff --git a/lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
deleted file mode 100644
index 5fc2533af28..00000000000
--- a/lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.store;
-
-import java.io.IOException;
-import java.lang.foreign.Arena;
-import java.lang.foreign.MemorySegment;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileChannel.MapMode;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.logging.Logger;
-import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.Unwrappable;
-
-@SuppressWarnings("preview")
-final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInputProvider {
-
- public MemorySegmentIndexInputProvider() {
- var log = Logger.getLogger(getClass().getName());
- log.info(
- "Using MemorySegmentIndexInput with Java 20; to disable start with -D"
- + MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP
- + "=false");
- }
-
- @Override
- public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
- throws IOException {
- final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")";
-
- // Work around for JDK-8259028: we need to unwrap our test-only file system layers
- path = Unwrappable.unwrapAll(path);
-
- boolean success = false;
- final Arena arena = Arena.openShared();
- try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {
- final long fileSize = fc.size();
- final IndexInput in =
- MemorySegmentIndexInput.newInstance(
- resourceDescription,
- arena,
- map(arena, resourceDescription, fc, chunkSizePower, preload, fileSize),
- fileSize,
- chunkSizePower);
- success = true;
- return in;
- } finally {
- if (success == false) {
- arena.close();
- }
- }
- }
-
- @Override
- public long getDefaultMaxChunkSize() {
- return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);
- }
-
- @Override
- public boolean isUnmapSupported() {
- return true;
- }
-
- @Override
- public String getUnmapNotSupportedReason() {
- return null;
- }
-
- private final MemorySegment[] map(
- Arena arena,
- String resourceDescription,
- FileChannel fc,
- int chunkSizePower,
- boolean preload,
- long length)
- throws IOException {
- if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
- throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription);
-
- final long chunkSize = 1L << chunkSizePower;
-
- // we always allocate one more segments, the last one may be a 0 byte one
- final int nrSegments = (int) (length >>> chunkSizePower) + 1;
-
- final MemorySegment[] segments = new MemorySegment[nrSegments];
-
- long startOffset = 0L;
- for (int segNr = 0; segNr < nrSegments; segNr++) {
- final long segSize =
- (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset);
- final MemorySegment segment;
- try {
- segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena.scope());
- } catch (IOException ioe) {
- throw convertMapFailedIOException(ioe, resourceDescription, segSize);
- }
- if (preload) {
- segment.load();
- }
- segments[segNr] = segment;
- startOffset += segSize;
- }
- return segments;
- }
-}
diff --git a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
similarity index 100%
rename from lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
rename to lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
diff --git a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
similarity index 100%
rename from lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
rename to lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
index 57c9b5275f0..7ccd89ff2bb 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
@@ -23,21 +23,12 @@ import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
-import java.util.logging.Logger;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.Unwrappable;
@SuppressWarnings("preview")
final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInputProvider {
- public MemorySegmentIndexInputProvider() {
- var log = Logger.getLogger(getClass().getName());
- log.info(
- "Using MemorySegmentIndexInput with Java 21 or later; to disable start with -D"
- + MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP
- + "=false");
- }
-
@Override
public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
throws IOException {
@@ -71,16 +62,6 @@ final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexIn
return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);
}
- @Override
- public boolean isUnmapSupported() {
- return true;
- }
-
- @Override
- public String getUnmapNotSupportedReason() {
- return null;
- }
-
private final MemorySegment[] map(
Arena arena,
String resourceDescription,
diff --git a/lucene/core/src/java21/org/apache/lucene/util/VectorUtilPanamaProvider.txt b/lucene/core/src/java21/org/apache/lucene/util/VectorUtilPanamaProvider.txt
deleted file mode 100644
index db75951206e..00000000000
--- a/lucene/core/src/java21/org/apache/lucene/util/VectorUtilPanamaProvider.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-The version of VectorUtilPanamaProvider for Java 21 is identical to that of Java 20.
-As such, there is no specific 21 version - the Java 20 version will be loaded from the MRJAR.
\ No newline at end of file
diff --git a/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java b/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java
index f03024a9860..c75ed932876 100644
--- a/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java
+++ b/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java
@@ -38,8 +38,6 @@ import org.apache.lucene.util.BytesRef;
public class Test4GBStoredFields extends LuceneTestCase {
public void test() throws Exception {
- assumeWorkingMMapOnWindows();
-
MockDirectoryWrapper dir =
new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("4GBStoredFields")));
dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 813212cf81d..61e5f7e361d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -1234,7 +1234,6 @@ public class TestIndexWriter extends LuceneTestCase {
public void testDeleteUnusedFiles() throws Exception {
assumeFalse("test relies on exact filenames", Codec.getDefault() instanceof SimpleTextCodec);
- assumeWorkingMMapOnWindows();
for (int iter = 0; iter < 2; iter++) {
// relies on Windows semantics
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDirectory.java
index 87cfee6e7c4..3717f490cfd 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDirectory.java
@@ -84,15 +84,6 @@ public class TestByteBuffersDirectory extends BaseDirectoryTestCase {
ByteBuffersDirectory.OUTPUT_AS_ONE_BUFFER),
"one buffer (heap)"
},
- {
- (Supplier<ByteBuffersDirectory>)
- () ->
- new ByteBuffersDirectory(
- new SingleInstanceLockFactory(),
- ByteBuffersDataOutput::new,
- ByteBuffersDirectory.OUTPUT_AS_MANY_BUFFERS_LUCENE),
- "lucene's buffers (heap)"
- },
{
(Supplier<ByteBuffersDirectory>)
() ->
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
index bd954f19d3f..3f601e4388d 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
@@ -43,9 +43,7 @@ public class TestDirectory extends LuceneTestCase {
final List<FSDirectory> dirs0 = new ArrayList<>();
dirs0.add(new NIOFSDirectory(path));
- if (hasWorkingMMapOnWindows()) {
- dirs0.add(new MMapDirectory(path));
- }
+ dirs0.add(new MMapDirectory(path));
final FSDirectory[] dirs = dirs0.toArray(FSDirectory[]::new);
for (int i = 0; i < dirs.length; i++) {
@@ -63,9 +61,6 @@ public class TestDirectory extends LuceneTestCase {
assertTrue(slowFileExists(d2, fname));
assertEquals(1 + largeBuffer.length, d2.fileLength(fname));
- // don't do read tests if unmapping is not supported!
- if (d2 instanceof MMapDirectory && !MMapDirectory.UNMAP_SUPPORTED) continue;
-
IndexInput input = d2.openInput(fname, newIOContext(random()));
assertEquals((byte) i, input.readByte());
// read array with buffering enabled
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
index 6be7e570a62..611dd6b1c82 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
@@ -18,11 +18,9 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Path;
-import java.util.Objects;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import org.apache.lucene.tests.store.BaseDirectoryTestCase;
-import org.junit.BeforeClass;
/** Tests MMapDirectory */
// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows
@@ -36,30 +34,7 @@ public class TestMMapDirectory extends BaseDirectoryTestCase {
return m;
}
- @BeforeClass
- public static void beforeClass() throws Exception {
- assertTrue(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, MMapDirectory.UNMAP_SUPPORTED);
- }
-
- private static boolean isMemorySegmentImpl() {
- return Objects.equals(
- "MemorySegmentIndexInputProvider", MMapDirectory.PROVIDER.getClass().getSimpleName());
- }
-
- public void testCorrectImplementation() {
- final int runtimeVersion = Runtime.version().feature();
- if (runtimeVersion >= 19) {
- assertTrue(
- "on Java 19 or later we should use MemorySegmentIndexInputProvider to create mmap IndexInputs",
- isMemorySegmentImpl());
- } else {
- assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass());
- }
- }
-
public void testAceWithThreads() throws Exception {
- assumeTrue("Test requires MemorySegmentIndexInput", isMemorySegmentImpl());
-
final int nInts = 8 * 1024 * 1024;
try (Directory dir = getDirectory(createTempDir("testAceWithThreads"))) {
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java b/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
index a0c5a29ccf8..3b2ef1b664e 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Locale;
import org.apache.lucene.tests.store.BaseChunkedDirectoryTestCase;
import org.apache.lucene.util.BytesRef;
-import org.junit.BeforeClass;
/**
* Tests MMapDirectory's MultiMMapIndexInput
@@ -38,11 +37,6 @@ public class TestMultiMMap extends BaseChunkedDirectoryTestCase {
return new MMapDirectory(path, maxChunkSize);
}
- @BeforeClass
- public static void beforeClass() throws Exception {
- assertTrue(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, MMapDirectory.UNMAP_SUPPORTED);
- }
-
public void testSeekingExceptions() throws IOException {
final int sliceSize = 128;
try (Directory dir = getDirectory(createTempDir(), sliceSize)) {
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
index 1cf72208a93..6ba979beb53 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
@@ -42,8 +42,6 @@ public class Test2BFST extends LuceneTestCase {
private static long LIMIT = 3L * 1024 * 1024 * 1024;
public void test() throws Exception {
- assumeWorkingMMapOnWindows();
-
int[] ints = new int[7];
IntsRef input = new IntsRef(ints, 0, ints.length);
long seed = random().nextLong();
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFSTOffHeap.java b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFSTOffHeap.java
index 1773c47f432..42fd4e1128a 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFSTOffHeap.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFSTOffHeap.java
@@ -44,8 +44,6 @@ public class Test2BFSTOffHeap extends LuceneTestCase {
private static long LIMIT = 3L * 1024 * 1024 * 1024;
public void test() throws Exception {
- assumeWorkingMMapOnWindows();
-
int[] ints = new int[7];
IntsRef input = new IntsRef(ints, 0, ints.length);
long seed = random().nextLong();
diff --git a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java
index 577f0f9e36c..084a044092f 100644
--- a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java
+++ b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java
@@ -207,7 +207,7 @@ public class TestModularLayer extends AbstractLuceneDistributionTest {
ClassLoader loader = layer.findLoader(coreModuleId);
- final Set<Integer> jarVersions = Set.of(19, 20, 21);
+ final Set<Integer> jarVersions = Set.of(21);
for (var v : jarVersions) {
Assertions.assertThat(
loader.getResource(
@@ -217,12 +217,9 @@ public class TestModularLayer extends AbstractLuceneDistributionTest {
.isNotNull();
}
- final int runtimeVersion = Runtime.version().feature();
- if (jarVersions.contains(runtimeVersion)) {
- Assertions.assertThat(
- loader.loadClass("org.apache.lucene.store.MemorySegmentIndexInput"))
- .isNotNull();
- }
+ Assertions.assertThat(
+ loader.loadClass("org.apache.lucene.store.MemorySegmentIndexInput"))
+ .isNotNull();
});
}
diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
index 732c9553a64..d62e4230bb8 100644
--- a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
+++ b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
@@ -97,7 +97,7 @@ public final class JavascriptCompiler {
private static final Lookup LOOKUP = MethodHandles.lookup();
- private static final int CLASSFILE_VERSION = Opcodes.V17;
+ private static final int CLASSFILE_VERSION = Opcodes.V21;
private static final MethodType MT_EXPRESSION_CTOR_LOOKUP =
MethodType.methodType(void.class, String.class, String[].class);
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java
index 4f6936ee205..b059d3e7356 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java
@@ -27,7 +27,6 @@ import org.apache.lucene.search.suggest.analyzing.FSTUtil;
import org.apache.lucene.search.suggest.document.CompletionPostingsFormat.FSTLoadMode;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
-import org.apache.lucene.store.ByteBufferIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
@@ -327,8 +326,7 @@ public final class NRTSuggester implements Accountable {
case AUTO:
// TODO: Make this less hacky to maybe expose "off-heap" feature using a marker interface on
// the IndexInput
- return input instanceof ByteBufferIndexInput
- || input.getClass().getName().contains(".MemorySegmentIndexInput");
+ return input.getClass().getName().contains(".MemorySegmentIndexInput");
default:
throw new IllegalStateException("unknown enum constant: " + fstLoadMode);
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseStoredFieldsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseStoredFieldsFormatTestCase.java
index efd351c21fe..63aa50bf713 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseStoredFieldsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseStoredFieldsFormatTestCase.java
@@ -723,8 +723,6 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
@Nightly
public void testBigDocuments() throws IOException {
- assumeWorkingMMapOnWindows();
-
// "big" as "much bigger than the chunk size"
// for this test we force an FS dir
// we can't just use newFSDirectory, because this test doesn't really index anything.
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
index 492bf9423fd..f1b9a644178 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
@@ -166,7 +166,6 @@ import org.apache.lucene.store.FileSwitchDirectory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.LockFactory;
-import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
@@ -190,7 +189,6 @@ import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CommandLineUtil;
-import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.NamedThreadFactory;
@@ -498,31 +496,9 @@ public abstract class LuceneTestCase extends Assert {
LEAVE_TEMPORARY = defaultValue;
}
- /**
- * Returns true, if MMapDirectory supports unmapping on this platform (required for Windows), or
- * if we are not on Windows.
- */
- public static boolean hasWorkingMMapOnWindows() {
- return !Constants.WINDOWS || MMapDirectory.UNMAP_SUPPORTED;
- }
-
- /**
- * Assumes that the current MMapDirectory implementation supports unmapping, so the test will not
- * fail on Windows.
- *
- * @see #hasWorkingMMapOnWindows()
- */
- public static void assumeWorkingMMapOnWindows() {
- assumeTrue(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, hasWorkingMMapOnWindows());
- }
-
/** Filesystem-based {@link Directory} implementations. */
private static final List<String> FS_DIRECTORIES =
- Arrays.asList(
- "NIOFSDirectory",
- // NIOFSDirectory as replacement for MMapDirectory if unmapping is not supported on
- // Windows (to make randomization stable):
- hasWorkingMMapOnWindows() ? "MMapDirectory" : "NIOFSDirectory");
+ Arrays.asList("NIOFSDirectory", "MMapDirectory");
/** All {@link Directory} implementations. */
private static final List<String> CORE_DIRECTORIES;