Apache Lucene open-source search software
LUCENE-10370: pass proper classpath/module arguments for forking jvms from within tests. (#909)
Dawid Weiss63b66e06cdf6
diff --git a/gradle/java/modules.gradle b/gradle/java/modules.gradleindex fa751167586..164b2a431e8 100644--- a/gradle/java/modules.gradle+++ b/gradle/java/modules.gradle@@ -15,6 +15,9 @@* limitations under the License.*/++import java.nio.charset.StandardCharsets+import java.nio.file.Filesimport java.nio.file.Path// Configure miscellaneous aspects required for supporting the java module system layer.@@ -172,6 +175,8 @@ allprojects {// have the need for it.Closure<Void> configureTestTaskForSourceSet = { Test task, SourceSet sourceSet ->task.configure {+ def forkProperties = file("${task.temporaryDir}/jvm-forking.properties")+ModularPathsExtension modularPaths = sourceSet.modularPathsdependsOn modularPaths@@ -185,6 +190,30 @@ allprojects {doFirst {modularPaths.logRuntimePaths(logger)}++ // Pass all the required properties for tests which fork the JVM. We don't use+ // regular system properties here because this could affect task up-to-date checks.+ jvmArgumentProviders.add(new CommandLineArgumentProvider() {+ @Override+ Iterable<String> asArguments() {+ return ["-Dtests.jvmForkArgsFile=" + forkProperties.absolutePath]+ }+ })+ doFirst {+ List<String> args = new ArrayList<>(modularPaths.runtimeArguments.asArguments().collect())+ def cp = modularPaths.runtimeClasspath.asPath+ if (!cp.isBlank()) {+ args.addAll(["-cp", cp])+ }++ // Sanity check.+ args.forEach(s -> {+ if (s.contains("\n")) {+ throw new RuntimeException("LF in forked jvm property?: " + s)+ }+ })+ Files.writeString(forkProperties.toPath(), String.join("\n", args), StandardCharsets.UTF_8)+ }}}diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 2792af7352d..88ebf205167 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -88,7 +88,9 @@ Bug FixesOther----------------------(No changes)++* LUCENE-10370: pass proper classpath/module arguments for forking jvms from within tests. (Dawid Weiss)+======================= Lucene 9.2.0 =======================diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestCodecLoadingDeadlock.java b/lucene/core/src/test/org/apache/lucene/codecs/TestCodecLoadingDeadlock.javaindex 925c8a941b9..16dea0e0d2e 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/TestCodecLoadingDeadlock.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/TestCodecLoadingDeadlock.java@@ -21,6 +21,7 @@ import com.carrotsearch.randomizedtesting.RandomizedRunner;import java.nio.file.Paths;import java.util.ArrayList;import java.util.Arrays;+import java.util.List;import java.util.Locale;import java.util.Random;import java.util.Set;@@ -31,6 +32,7 @@ import java.util.concurrent.Executors;import java.util.concurrent.TimeUnit;import java.util.stream.Collectors;import java.util.stream.IntStream;+import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.util.NamedThreadFactory;import org.junit.Assert;import org.junit.Test;@@ -60,18 +62,13 @@ public class TestCodecLoadingDeadlock extends Assert {System.out.println(String.format(Locale.ROOT, "codec: %s, pf: %s, dvf: %s", codecName, pfName, dvfName));+ List<String> args = new ArrayList<>();+ args.add(Paths.get(System.getProperty("java.home"), "bin", "java").toString());+ args.addAll(LuceneTestCase.getJvmForkArguments());+ args.addAll(List.of(getClass().getName(), codecName, pfName, dvfName));+// Fork a separate JVM to reinitialize classes.- final Process p =- new ProcessBuilder(- Paths.get(System.getProperty("java.home"), "bin", "java").toString(),- "-cp",- System.getProperty("java.class.path"),- getClass().getName(),- codecName,- pfName,- dvfName)- .inheritIO()- .start();+ final Process p = new ProcessBuilder(args).inheritIO().start();if (p.waitFor(MAX_TIME_SECONDS * 2, TimeUnit.SECONDS)) {assertEquals("Process died abnormally?", 0, p.waitFor());} else {diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.javaindex fca6b065c0c..7b084734bb6 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java@@ -104,8 +104,7 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {cmd.add("-DtempDir=" + tempDir);cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(random().nextLong()));cmd.add("-ea");- cmd.add("-cp");- cmd.add(System.getProperty("java.class.path"));+ cmd.addAll(getJvmForkArguments());cmd.add("org.junit.runner.JUnitCore");cmd.add(getClass().getName());ProcessBuilder pb =diff --git a/lucene/core/src/test/org/apache/lucene/store/TestStressLockFactories.java b/lucene/core/src/test/org/apache/lucene/store/TestStressLockFactories.javaindex 0ff235389f0..3a3c5b1cb54 100644--- a/lucene/core/src/test/org/apache/lucene/store/TestStressLockFactories.java+++ b/lucene/core/src/test/org/apache/lucene/store/TestStressLockFactories.java@@ -57,24 +57,22 @@ public class TestStressLockFactories extends LuceneTestCase {// spawn clients as separate Java processesfor (int i = 0; i < clients; i++) {try {- processes.add(- applyRedirection(- new ProcessBuilder(- Paths.get(System.getProperty("java.home"), "bin", "java").toString(),- "-Xmx32M",- "-cp",- System.getProperty("java.class.path"),- LockStressTest.class.getName(),- Integer.toString(i),- addr.getHostString(),- Integer.toString(addr.getPort()),- impl.getName(),- dir.toString(),- Integer.toString(delay),- Integer.toString(rounds)),- i,- dir)- .start());+ List<String> args = new ArrayList<>();+ args.add(Paths.get(System.getProperty("java.home"), "bin", "java").toString());+ args.addAll(getJvmForkArguments());+ args.addAll(+ List.of(+ "-Xmx32M",+ LockStressTest.class.getName(),+ Integer.toString(i),+ addr.getHostString(),+ Integer.toString(addr.getPort()),+ impl.getName(),+ dir.toString(),+ Integer.toString(delay),+ Integer.toString(rounds)));++ processes.add(applyRedirection(new ProcessBuilder(args), i, dir).start());} catch (IOException ioe) {throw new AssertionError("Failed to start child process.", ioe);}diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestNRTReplication.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestNRTReplication.javaindex 176a39c4568..27926c40b82 100644--- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestNRTReplication.java+++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestNRTReplication.java@@ -35,7 +35,6 @@ import org.apache.lucene.store.FSDirectory;import org.apache.lucene.store.LockObtainFailedException;import org.apache.lucene.tests.util.LineFileDocs;import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks;import org.apache.lucene.tests.util.TestRuleIgnoreTestSuites;@@ -44,7 +43,6 @@ import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.SuppressForbidden;// MockRandom's .sd file has no index header/footer:-@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-10370")@SuppressCodecs({"MockRandom", "Direct", "SimpleText"})@SuppressSysoutChecks(bugUrl = "Stuff gets printed, important stuff for debugging a failure")public class TestNRTReplication extends LuceneTestCase {@@ -105,8 +103,9 @@ public class TestNRTReplication extends LuceneTestCase {long seed = random().nextLong() * nodeStartCounter.incrementAndGet();cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(seed));cmd.add("-ea");- cmd.add("-cp");- cmd.add(System.getProperty("java.class.path"));++ cmd.addAll(getJvmForkArguments());+cmd.add("org.junit.runner.JUnitCore");cmd.add(TestSimpleServer.class.getName());@@ -355,7 +354,6 @@ public class TestNRTReplication extends LuceneTestCase {// Start up, index 10 docs, replicate, but crash and restart the replica without committing it:@Nightlypublic void testReplicaCrashNoCommit() throws Exception {-Path primaryPath = createTempDir("primary");NodeProcess primary = startNode(-1, 0, primaryPath, -1, false);diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestStressNRTReplication.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestStressNRTReplication.javaindex a2dd0802e0c..ed1bb97af93 100644--- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestStressNRTReplication.java+++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestStressNRTReplication.java@@ -27,6 +27,7 @@ import java.io.Writer;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;+import java.nio.file.Paths;import java.nio.file.StandardOpenOption;import java.util.ArrayList;import java.util.Arrays;@@ -47,7 +48,6 @@ import org.apache.lucene.document.Field;import org.apache.lucene.tests.store.MockDirectoryWrapper;import org.apache.lucene.tests.util.LineFileDocs;import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks;import org.apache.lucene.tests.util.TestRuleIgnoreTestSuites;@@ -115,7 +115,6 @@ import org.apache.lucene.util.ThreadInterruptedException;** <p>Slow network is simulated with a RateLimiter.*/-@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-10370")// MockRandom's .sd file has no index header/footer:@SuppressCodecs({"MockRandom", "Direct", "SimpleText"})@SuppressSysoutChecks(bugUrl = "Stuff gets printed, important stuff for debugging a failure")@@ -314,14 +313,9 @@ public class TestStressNRTReplication extends LuceneTestCase {}message(- "PG="- + (primary == null ? "X" : primaryGen)- + " "- + liveCount- + " (of "- + nodes.length- + ") nodes running: "- + sb);+ ("PG=" + (primary == null ? "X" : primaryGen))+ + (" " + liveCount)+ + (" (of " + nodes.length + ") nodes running: " + sb));// Commit a random node, primary or replica@@ -560,12 +554,7 @@ public class TestStressNRTReplication extends LuceneTestCase {NodeProcess curPrimary = primary;- cmd.add(- System.getProperty("java.home")- + System.getProperty("file.separator")- + "bin"- + System.getProperty("file.separator")- + "java");+ cmd.add(Paths.get(System.getProperty("java.home"), "bin", "java").toString());cmd.add("-Xmx512m");if (curPrimary != null) {@@ -618,8 +607,7 @@ public class TestStressNRTReplication extends LuceneTestCase {long seed = random().nextLong() * nodeStartCounter.incrementAndGet();cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(seed));cmd.add("-ea");- cmd.add("-cp");- cmd.add(System.getProperty("java.class.path"));+ cmd.addAll(getJvmForkArguments());cmd.add("org.junit.runner.JUnitCore");cmd.add(TestSimpleServer.class.getName());diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.javaindex b87b0193bcc..43af8c6769c 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java@@ -63,6 +63,7 @@ import java.lang.reflect.Constructor;import java.lang.reflect.Method;import java.net.URISyntaxException;import java.nio.charset.StandardCharsets;+import java.nio.file.Files;import java.nio.file.NoSuchFileException;import java.nio.file.Path;import java.nio.file.Paths;@@ -106,8 +107,46 @@ import org.apache.lucene.document.Field.Store;import org.apache.lucene.document.FieldType;import org.apache.lucene.document.StringField;import org.apache.lucene.document.TextField;-import org.apache.lucene.index.*;+import org.apache.lucene.index.BinaryDocValues;+import org.apache.lucene.index.CodecReader;+import org.apache.lucene.index.CompositeReader;+import org.apache.lucene.index.ConcurrentMergeScheduler;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.DocValuesType;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;+import org.apache.lucene.index.Fields;+import org.apache.lucene.index.IndexFileNames;+import org.apache.lucene.index.IndexOptions;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.IndexableField;+import org.apache.lucene.index.LeafReader;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.LiveIndexWriterConfig;+import org.apache.lucene.index.LogByteSizeMergePolicy;+import org.apache.lucene.index.LogDocMergePolicy;+import org.apache.lucene.index.LogMergePolicy;+import org.apache.lucene.index.MergePolicy;+import org.apache.lucene.index.MergeScheduler;+import org.apache.lucene.index.MultiBits;+import org.apache.lucene.index.MultiDocValues;+import org.apache.lucene.index.MultiTerms;+import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.ParallelCompositeReader;+import org.apache.lucene.index.ParallelLeafReader;+import org.apache.lucene.index.PointValues;+import org.apache.lucene.index.PostingsEnum;+import org.apache.lucene.index.SerialMergeScheduler;+import org.apache.lucene.index.SimpleMergedSegmentWarmer;+import org.apache.lucene.index.SortedDocValues;+import org.apache.lucene.index.SortedNumericDocValues;+import org.apache.lucene.index.SortedSetDocValues;+import org.apache.lucene.index.Terms;+import org.apache.lucene.index.TermsEnum;import org.apache.lucene.index.TermsEnum.SeekStatus;+import org.apache.lucene.index.TieredMergePolicy;import org.apache.lucene.internal.tests.IndexPackageAccess;import org.apache.lucene.internal.tests.TestSecrets;import org.apache.lucene.search.DocIdSetIterator;@@ -910,7 +949,7 @@ public abstract class LuceneTestCase extends Assert {/*** Convenience method for logging an array. Wraps the array in an iterator and delegates*- * @see #dumpIterator(String,Iterator,PrintStream)+ * @see #dumpIterator(String, Iterator, PrintStream)*/public static void dumpArray(String label, Object[] objs, PrintStream stream) {Iterator<?> iter = (null == objs) ? null : Arrays.asList(objs).iterator();@@ -3011,6 +3050,25 @@ public abstract class LuceneTestCase extends Assert {return createTempFile("tempFile", ".tmp");}+ /**+ * Returns a set of JVM arguments to fork a JVM with the same class or module path (including any+ * associated JVM options). The returned value may be empty. This method may throw an assertion+ * error if fork options cannot be reliably acquired (at the moment they are collected and passed+ * as an external file in gradle scripts).+ *+ * <p><b>JVM forking is strongly discouraged as it makes test slower and more resource-hungry.+ * Consider all alternatives first.</b>+ */+ public static List<String> getJvmForkArguments() throws IOException {+ String forkArgsFile = System.getProperty("tests.jvmForkArgsFile");+ Path forkArgsPath;+ if (forkArgsFile == null || !Files.isRegularFile(forkArgsPath = Paths.get(forkArgsFile))) {+ throw new AssertionError("JVM fork arguments are not present.");+ }++ return Files.readAllLines(forkArgsPath, StandardCharsets.UTF_8);+ }+/*** Runs a code part with restricted permissions (be sure to add all required permissions, because* it would start with empty permissions). You cannot grant more permissions than our policy file