Apache Lucene open-source search software
Redo #12707: Do not rely on isAlive() status of MemorySegment#Scope and make sure IndexInput#close() does not throw IllegalStateException and waits instead (#12785)
Uwe Schindler5b43384367bf
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex db102c52562..470666c822b 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -208,8 +208,11 @@ Improvements* GITHUB#12586: Remove over-counting of deleted terms. (Guo Feng)-* GITHUB#12705, GITHUB#12705: Improve handling of NullPointerException and IllegalStateException- in MMapDirectory's IndexInputs. (Uwe Schindler, Michael Sokolov)+* GITHUB#12705, GITHUB#12705, GITHUB#12785: Improve handling of NullPointerException and+ IllegalStateException in MMapDirectory's IndexInputs. Also makes sure to close master+ MemorySegmentIndexInput while not throwing IllegalStateException (retry in spin loop).+ Also improve TestMmapDirectory.testAceWithThreads to run faster and use less resources.+ (Uwe Schindler, Maurizio Cimadamore, Michael Sokolov)* GITHUB#12689: TaskExecutor to cancel all tasks on exception to avoid needless computation. (Luca Cavanna)diff --git a/lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInput.javaindex ee102a12d4e..44ecff57cb1 100644--- a/lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInput.java+++ b/lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInput.java@@ -105,12 +105,15 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccesAlreadyClosedException alreadyClosed(RuntimeException e) {// we use NPE to signal if this input is closed (to not have checks everywhere). If NPE happens,// we check the "is closed" condition explicitly by checking that our "curSegment" is null.+ // Care must be taken to not leak the NPE to code outside MemorySegmentIndexInput!if (this.curSegment == null) {return new AlreadyClosedException("Already closed: " + this);}- // we also check if the session of all segments is still alive:- if (Arrays.stream(segments).allMatch(s -> s.session().isAlive()) == false) {- return new AlreadyClosedException("Already closed: " + this);+ // ISE can be thrown by MemorySegment and contains "closed" in message:+ if (e instanceof IllegalStateException+ && e.getMessage() != null+ && e.getMessage().contains("closed")) {+ return new AlreadyClosedException("Already closed: " + this, e);}// otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to// the IndexInput method):@@ -472,17 +475,26 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccesreturn;}- // make sure all accesses to this IndexInput instance throw NPE:- curSegment = null;- Arrays.fill(segments, null);-// the master IndexInput has a MemorySession and is able// to release all resources (unmap segments) - a// side effect is that other threads still using clones// will throw IllegalStateExceptionif (session != null) {- session.close();+ while (session.isAlive()) {+ try {+ session.close();+ break;+ } catch (+ @SuppressWarnings("unused")+ IllegalStateException e) {+ Thread.onSpinWait();+ }+ }}++ // make sure all accesses to this IndexInput instance throw NPE:+ curSegment = null;+ Arrays.fill(segments, null);}/** Optimization of MemorySegmentIndexInput for when there is only one segment. */diff --git a/lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInput.javaindex 627c5edbd5a..0f2fdccac90 100644--- a/lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInput.java+++ b/lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInput.java@@ -103,12 +103,15 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccesAlreadyClosedException alreadyClosed(RuntimeException e) {// we use NPE to signal if this input is closed (to not have checks everywhere). If NPE happens,// we check the "is closed" condition explicitly by checking that our "curSegment" is null.+ // Care must be taken to not leak the NPE to code outside MemorySegmentIndexInput!if (this.curSegment == null) {return new AlreadyClosedException("Already closed: " + this);}- // we also check if the scope of all segments is still alive:- if (Arrays.stream(segments).allMatch(s -> s.scope().isAlive()) == false) {- return new AlreadyClosedException("Already closed: " + this);+ // ISE can be thrown by MemorySegment and contains "closed" in message:+ if (e instanceof IllegalStateException+ && e.getMessage() != null+ && e.getMessage().contains("closed")) {+ return new AlreadyClosedException("Already closed: " + this, e);}// otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to// the IndexInput method):@@ -470,17 +473,26 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccesreturn;}- // make sure all accesses to this IndexInput instance throw NPE:- curSegment = null;- Arrays.fill(segments, null);-// the master IndexInput has an Arena and is able// to release all resources (unmap segments) - a// side effect is that other threads still using clones// will throw IllegalStateExceptionif (arena != null) {- arena.close();+ while (arena.scope().isAlive()) {+ try {+ arena.close();+ break;+ } catch (+ @SuppressWarnings("unused")+ IllegalStateException e) {+ Thread.onSpinWait();+ }+ }}++ // make sure all accesses to this IndexInput instance throw NPE:+ curSegment = null;+ Arrays.fill(segments, null);}/** Optimization of MemorySegmentIndexInput for when there is only one segment. */diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.javaindex 627c5edbd5a..0f2fdccac90 100644--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java@@ -103,12 +103,15 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccesAlreadyClosedException alreadyClosed(RuntimeException e) {// we use NPE to signal if this input is closed (to not have checks everywhere). If NPE happens,// we check the "is closed" condition explicitly by checking that our "curSegment" is null.+ // Care must be taken to not leak the NPE to code outside MemorySegmentIndexInput!if (this.curSegment == null) {return new AlreadyClosedException("Already closed: " + this);}- // we also check if the scope of all segments is still alive:- if (Arrays.stream(segments).allMatch(s -> s.scope().isAlive()) == false) {- return new AlreadyClosedException("Already closed: " + this);+ // ISE can be thrown by MemorySegment and contains "closed" in message:+ if (e instanceof IllegalStateException+ && e.getMessage() != null+ && e.getMessage().contains("closed")) {+ return new AlreadyClosedException("Already closed: " + this, e);}// otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to// the IndexInput method):@@ -470,17 +473,26 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccesreturn;}- // make sure all accesses to this IndexInput instance throw NPE:- curSegment = null;- Arrays.fill(segments, null);-// the master IndexInput has an Arena and is able// to release all resources (unmap segments) - a// side effect is that other threads still using clones// will throw IllegalStateExceptionif (arena != null) {- arena.close();+ while (arena.scope().isAlive()) {+ try {+ arena.close();+ break;+ } catch (+ @SuppressWarnings("unused")+ IllegalStateException e) {+ Thread.onSpinWait();+ }+ }}++ // make sure all accesses to this IndexInput instance throw NPE:+ curSegment = null;+ Arrays.fill(segments, null);}/** Optimization of MemorySegmentIndexInput for when there is only one segment. */diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.javaindex a7156383aab..a0b013d0608 100644--- a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java+++ b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java@@ -60,48 +60,45 @@ public class TestMmapDirectory extends BaseDirectoryTestCase {public void testAceWithThreads() throws Exception {assumeTrue("Test requires MemorySegmentIndexInput", isMemorySegmentImpl());- final int iters = RANDOM_MULTIPLIER * (TEST_NIGHTLY ? 50 : 10);- for (int iter = 0; iter < iters; iter++) {- Directory dir = getDirectory(createTempDir("testAceWithThreads"));- IndexOutput out = dir.createOutput("test", IOContext.DEFAULT);- Random random = random();- for (int i = 0; i < 8 * 1024 * 1024; i++) {- out.writeInt(random.nextInt());+ final int nInts = 8 * 1024 * 1024;++ try (Directory dir = getDirectory(createTempDir("testAceWithThreads"))) {+ try (IndexOutput out = dir.createOutput("test", IOContext.DEFAULT)) {+ final Random random = random();+ for (int i = 0; i < nInts; i++) {+ out.writeInt(random.nextInt());+ }}- out.close();- IndexInput in = dir.openInput("test", IOContext.DEFAULT);- IndexInput clone = in.clone();- final byte[] accum = new byte[32 * 1024 * 1024];- final CountDownLatch shotgun = new CountDownLatch(1);- Thread t1 =- new Thread(- () -> {- try {- shotgun.await();- for (int i = 0; i < 10; i++) {- clone.seek(0);- clone.readBytes(accum, 0, accum.length);++ final int iters = RANDOM_MULTIPLIER * (TEST_NIGHTLY ? 50 : 10);+ for (int iter = 0; iter < iters; iter++) {+ final IndexInput in = dir.openInput("test", IOContext.DEFAULT);+ final IndexInput clone = in.clone();+ final byte[] accum = new byte[nInts * Integer.BYTES];+ final CountDownLatch shotgun = new CountDownLatch(1);+ final Thread t1 =+ new Thread(+ () -> {+ try {+ shotgun.await();+ for (int i = 0; i < 10; i++) {+ clone.seek(0);+ clone.readBytes(accum, 0, accum.length);+ }+ } catch (+ @SuppressWarnings("unused")+ AlreadyClosedException ok) {+ // OK+ } catch (InterruptedException | IOException e) {+ throw new RuntimeException(e);}- } catch (@SuppressWarnings("unused") IOException | AlreadyClosedException ok) {- // OK- } catch (InterruptedException e) {- throw new RuntimeException(e);- }- });- t1.start();- shotgun.countDown();- try {+ });+ t1.start();+ shotgun.countDown();+ // this triggers "bad behaviour": closing input while other threads are runningin.close();- } catch (- @SuppressWarnings("unused")- IllegalStateException ise) {- // this may also happen and is a valid exception, informing our user that, e.g., a query is- // running!- // "java.lang.IllegalStateException: Cannot close while another thread is accessing the- // segment"+ t1.join();}- t1.join();- dir.close();}}