Skip to content

Apache Lucene open-source search software

An eye-gouging way to limit suppressAccessChecks to just the three JARs that need them. (#13164)

Dawid Weiss1c77e2315c41
diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradle
index 4f24d7f412f..44ae964cab0 100644
--- a/gradle/testing/randomization.gradle
+++ b/gradle/testing/randomization.gradle
@@ -119,6 +119,18 @@ allprojects {
// Resolve test option values after all evaluation is complete.
allprojects {
plugins.withType(JavaPlugin) {
+ configurations {
+ secManagerExclusions
+ }
+ dependencies {
+ secManagerExclusions ( "com.carrotsearch.randomizedtesting:randomizedtesting-runner", {
+ exclude group: "junit"
+ })
+ secManagerExclusions ( "junit:junit", {
+ exclude group: "org.hamcrest"
+ })
+ }
+
afterEvaluate {
ext.testOptionsResolved = testOptions.findAll { opt ->
propertyOrDefault(opt.propName, opt.value) != null
@@ -155,7 +167,7 @@ allprojects {
}
// Append resolved test properties to the test task.
- tasks.withType(Test) { task ->
+ tasks.withType(Test) { Test task ->
// TODO: we could remove some options that are only relevant to the build environment
// and not the test JVM itself.
systemProperties testOptionsResolved
@@ -172,13 +184,13 @@ allprojects {
// Enable security manager, if requested. We could move the selection of security manager and security policy
// to each project's build/ configuration but it seems compact enough to keep it here for now.
- def securityArgumentProvider = new SecurityArgumentProvider(commonDir: project(":lucene").layout.projectDirectory)
+ def securityArgumentProvider = new SecurityArgumentProvider()
+ securityArgumentProvider.commonDir = project(":lucene").layout.projectDirectory
+ securityArgumentProvider.otherProperties = project.objects.mapProperty(String, String)
+
if (Boolean.parseBoolean(testOptionsResolved["tests.useSecurityManager"])) {
if (project.path.endsWith(".tests")) {
// LUCENE-10301: for now, do not use the security manager for modular tests (test framework is not available).
- } else if (project.path == ":lucene:replicator") {
- systemProperty 'java.security.manager', "org.apache.lucene.tests.util.TestSecurityManager"
- securityArgumentProvider.javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/replicator-tests.policy")
} else if (project.path.startsWith(":lucene")) {
systemProperty 'java.security.manager', "org.apache.lucene.tests.util.TestSecurityManager"
securityArgumentProvider.javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/tests.policy")
@@ -189,6 +201,14 @@ allprojects {
systemProperty 'gradle.lib.dir', Paths.get(project.class.location.toURI()).parent.toAbsolutePath().toString().replace('\\', '/')
systemProperty 'gradle.worker.jar', Paths.get("${gradleUserHome}/caches/${gradle.gradleVersion}/workerMain/gradle-worker.jar").toAbsolutePath().toString()
systemProperty 'gradle.user.home', gradleUserHome.toPath().toAbsolutePath().toString()
+
+ securityArgumentProvider.otherProperties.put("randomizedtesting.jar", project.provider { ->
+ return configurations.secManagerExclusions.resolve().find { it.name.startsWith("randomizedtesting-runner-") }.absolutePath
+ })
+ securityArgumentProvider.otherProperties.put("junit.jar", project.provider { ->
+ return configurations.secManagerExclusions.resolve().find { it.name.startsWith("junit-") }.absolutePath
+ })
+ securityArgumentProvider.otherProperties.put("lucene.test.framework", project(":lucene:test-framework").layout.buildDirectory.get().asFile.absolutePath)
}
doFirst {
@@ -242,12 +262,20 @@ class SecurityArgumentProvider implements CommandLineArgumentProvider {
@PathSensitive(PathSensitivity.RELATIVE)
RegularFile javaSecurityPolicy
+ @Input
+ MapProperty<String, String> otherProperties
+
@Override
Iterable<String> asArguments() {
def args = ["-Dcommon.dir=${commonDir.getAsFile()}"]
if (javaSecurityPolicy) {
args.add("-Djava.security.policy=${javaSecurityPolicy.getAsFile()}")
}
+
+ otherProperties.getOrElse(Map.of()).forEach { key, value ->
+ args.add("-D${key}=${value}")
+ }
+
return args
}
}
diff --git a/gradle/testing/randomization/policies/replicator-tests.policy b/gradle/testing/randomization/policies/replicator-tests.policy
deleted file mode 100644
index 4d8fcd83f24..00000000000
--- a/gradle/testing/randomization/policies/replicator-tests.policy
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Policy file for :lucene:replicator tests. Please keep minimal and avoid wildcards.
-// this differs from the standard lucene policy in that it must allow read-write access
-// to all system properties, because of jetty calls to System.getProperties()
-
-grant {
- // jetty-specific:
- permission java.lang.RuntimePermission "getenv.JETTY_AVAILABLE_PROCESSORS";
- permission java.lang.RuntimePermission "getenv.JETTY_WORKER_INSTANCE";
- // allow TestNRTReplication fork its jvm
- permission java.io.FilePermission "${java.home}${/}-", "read,execute";
- // read/write access to all system properties (required by jetty in these tests)
- permission java.util.PropertyPermission "*", "read,write";
- // all possibilities of accepting/binding/connecting on localhost with ports >= 1024:
- permission java.net.SocketPermission "localhost:1024-", "accept,connect,resolve";
- permission java.net.SocketPermission "127.0.0.1:1024-", "accept,connect,resolve";
- permission java.net.SocketPermission "[::1]:1024-", "accept,connect,resolve";
-
- // test-files/ resources
- permission java.io.FilePermission "${common.dir}${/}-", "read";
-
- // write only to sandbox
- permission java.io.FilePermission "${tests.linedocsfile}", "read";
- permission java.io.FilePermission "${java.io.tmpdir}", "read,write";
- permission java.io.FilePermission "${java.io.tmpdir}${/}-", "read,write,delete";
-
- // needed by randomizedtesting runner to identify test methods.
- permission java.lang.RuntimePermission "accessDeclaredMembers";
- // needed by certain tests to redirect sysout/syserr:
- permission java.lang.RuntimePermission "setIO";
- // needed by randomized runner to catch failures from other threads:
- permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler";
- // needed by randomized runner getTopThreadGroup:
- permission java.lang.RuntimePermission "modifyThreadGroup";
- // needed by tests e.g. shutting down executors:
- permission java.lang.RuntimePermission "modifyThread";
- // needed for tons of test hacks etc
- permission java.lang.RuntimePermission "getStackTrace";
- // needed for mock filesystems in tests
- permission java.lang.RuntimePermission "fileSystemProvider";
- // test runner seems to require supression of access checks:
- permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
-};
-
-// Permissions to support ant build
-grant {
- permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read";
- permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";
- permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";
-};
-
-// Permissions for jacoco code coverage
-grant {
- // permission to write the per-jvm code coverage output
- permission java.io.FilePermission "${user.dir}${/}jacoco.exec", "write";
- // needed by jacoco to dump coverage on shutdown
- permission java.lang.RuntimePermission "shutdownHooks";
- // needed by jacoco to instrument classes
- permission java.lang.RuntimePermission "defineClass";
- // needed by jacoco for God knows what.
- permission java.lang.RuntimePermission "createClassLoader";
-};
-
-// Grant all permissions to Gradle test runner classes.
-grant codeBase "file:${gradle.lib.dir}${/}-" {
- permission java.security.AllPermission;
-};
-
-grant codeBase "file:${gradle.worker.jar}" {
- permission java.security.AllPermission;
-};
-
-grant {
- // Allow reading gradle worker JAR.
- permission java.io.FilePermission "${gradle.worker.jar}", "read";
- // Allow reading from classpath JARs (resources).
- permission java.io.FilePermission "${gradle.user.home}${/}-", "read";
-};
diff --git a/gradle/testing/randomization/policies/tests.policy b/gradle/testing/randomization/policies/tests.policy
index ffb6d0b52d0..f8e09ba0366 100644
--- a/gradle/testing/randomization/policies/tests.policy
+++ b/gradle/testing/randomization/policies/tests.policy
@@ -33,34 +33,29 @@ grant {
// misc HardlinkCopyDirectoryWrapper needs this to test if hardlinks can be created
permission java.nio.file.LinkPermission "hard";
- // needed by SSD detection tests in TestIOUtils (creates symlinks)
- permission java.nio.file.LinkPermission "symbolic";
- // needed by randomizedtesting runner to identify test methods.
+ // needed by RamUsageEstimator, and randomizedtesting runner to identify test methods.
permission java.lang.RuntimePermission "accessDeclaredMembers";
// needed by certain tests to capture sysout/syserr:
permission java.lang.RuntimePermission "setIO";
- // needed by randomized runner to catch failures from other threads:
+ // needed by luke and randomized runner to catch failures from other threads:
permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler";
- // needed by randomized runner getTopThreadGroup:
- permission java.lang.RuntimePermission "modifyThreadGroup";
// needed by tests e.g. shutting down executors:
permission java.lang.RuntimePermission "modifyThread";
// needed for tons of test hacks etc
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
- // test runner seems to require supression of access checks:
- permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
// needed by cyberneko usage by benchmarks on J9
permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util";
- // Needed for loading native library (lucene:misc:native) in lucene:misc
+ // Needed for DirectIODirectory to retrieve block size
permission java.lang.RuntimePermission "getFileStoreAttributes";
- permission java.lang.RuntimePermission "writeFileDescriptor";
// TestLockFactoriesMultiJVM opens a random port on 127.0.0.1 (port 0 = ephemeral port range):
permission java.net.SocketPermission "127.0.0.1:0", "accept,listen,resolve";
+ // Replicator tests connect to ephemeral ports
+ permission java.net.SocketPermission "127.0.0.1:1024-", "connect,resolve";
// read access to all system properties:
permission java.util.PropertyPermission "*", "read";
@@ -87,13 +82,6 @@ grant {
permission java.io.FilePermission "${hunspell.dictionaries}${/}-", "read";
};
-// Permissions to support ant build
-grant {
- permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read";
- permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";
- permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";
-};
-
// Permissions for jacoco code coverage
grant {
// permission to write the per-jvm code coverage output
@@ -122,3 +110,14 @@ grant {
permission java.io.FilePermission "${gradle.user.home}${/}-", "read";
};
+// Grant permissions to certain test-related JARs (https://github.com/apache/lucene/pull/13146)
+grant codeBase "file:${randomizedtesting.jar}" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "modifyThreadGroup";
+};
+grant codeBase "file:${junit.jar}" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+};
+grant codeBase "file:${lucene.test.framework}${/}-" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+};
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
index 296a39af15d..60de20d22bb 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
@@ -22,8 +22,6 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.lang.ProcessBuilder.Redirect;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -31,10 +29,10 @@ import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.tests.store.BaseDirectoryWrapper;
import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.Constants;
import org.apache.lucene.util.SuppressForbidden;
/**
@@ -57,7 +55,8 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {
if (System.getProperty("tests.crashmode") == null) {
// try up to 10 times to create an index
for (int i = 0; i < 10; i++) {
- forkTest();
+ final int crashDelayMs = TestUtil.nextInt(random(), 3000, 4000);
+ forkTest(crashDelayMs);
// if we succeeded in finding an index, we are done.
if (checkIndexes(tempDir)) return;
}
@@ -68,34 +67,17 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {
// assumeFalse("does not support PreFlex, see LUCENE-3992",
// Codec.getDefault().getName().equals("Lucene4x"));
- // we are the fork, setup a crashing thread
- final int crashTime = TestUtil.nextInt(random(), 3000, 4000);
- Thread t =
- new Thread() {
- @SuppressForbidden(reason = "Thread sleep")
- @Override
- public void run() {
- try {
- Thread.sleep(crashTime);
- } catch (
- @SuppressWarnings("unused")
- InterruptedException e) {
- }
- crashJRE();
- }
- };
- t.setPriority(Thread.MAX_PRIORITY);
- t.start();
- // run the test until we crash.
- for (int i = 0; i < 1000; i++) {
+ // Effectively an endless loop until the process is killed from the outside.
+ for (int i = 0; i < Integer.MAX_VALUE; i++) {
super.testNRTThreads();
}
}
}
/** fork ourselves in a new jvm. sets -Dtests.crashmode=true */
- @SuppressForbidden(reason = "ProcessBuilder requires java.io.File for CWD")
- public void forkTest() throws Exception {
+ @SuppressForbidden(
+ reason = "ProcessBuilder requires java.io.File for setting ProcessBuilder.directory")
+ public void forkTest(int crashDelayMs) throws Exception {
List<String> cmd = new ArrayList<>();
cmd.add(Paths.get(System.getProperty("java.home"), "bin", "java").toString());
cmd.add("-Xmx512m");
@@ -114,12 +96,21 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {
.redirectInput(Redirect.INHERIT)
.redirectErrorStream(true);
Process p = pb.start();
-
// We pump everything to stderr.
PrintStream childOut = System.err;
Thread stdoutPumper = ThreadPumper.start(p.getInputStream(), childOut);
- if (VERBOSE) childOut.println(">>> Begin subprocess output");
- p.waitFor();
+ try {
+ if (VERBOSE) childOut.println(">>> Begin subprocess output");
+ if (p.waitFor(crashDelayMs, TimeUnit.MILLISECONDS)) {
+ // This means the process has exited before the timeout. This is odd because it should
+ // run in an endless loop?
+ throw new AssertionError("Subprocess has exited unexpectedly.");
+ }
+ } finally {
+ // Try to kill the process and wait until it terminates. destroyForcibly is a no-op in case
+ // the process is not alive, so no need for additional checks.
+ p.destroyForcibly().waitFor();
+ }
stdoutPumper.join();
if (VERBOSE) childOut.println("<<< End subprocess output");
}
@@ -140,7 +131,7 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {
}
}
} catch (IOException e) {
- System.err.println("Couldn't pipe from the forked process: " + e.toString());
+ System.err.println("Couldn't pipe from the forked process: " + e);
}
}
};
@@ -186,37 +177,4 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {
});
return found.get();
}
-
- /** currently, this only works/tested on Sun and IBM. */
- @SuppressForbidden(reason = "We need Unsafe to actually crush :-)")
- public void crashJRE() {
- final String vendor = Constants.JAVA_VENDOR;
- final boolean supportsUnsafeNpeDereference =
- vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple");
-
- try {
- if (supportsUnsafeNpeDereference) {
- try {
- Class<?> clazz = Class.forName("sun.misc.Unsafe");
- Field field = clazz.getDeclaredField("theUnsafe");
- field.setAccessible(true);
- Object o = field.get(null);
- Method m = clazz.getMethod("putAddress", long.class, long.class);
- m.invoke(o, 0L, 0L);
- } catch (Throwable e) {
- System.out.println("Couldn't kill the JVM via Unsafe.");
- e.printStackTrace(System.out);
- }
- }
-
- // Fallback attempt to Runtime.halt();
- Runtime.getRuntime().halt(-1);
- } catch (Exception e) {
- System.out.println("Couldn't kill the JVM.");
- e.printStackTrace(System.out);
- }
-
- // We couldn't get the JVM to crash for some reason.
- fail();
- }
}
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java
index c715bd57936..f21ac11b884 100644
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java
+++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java
@@ -21,7 +21,6 @@ import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
@@ -45,7 +44,6 @@ import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks;
import org.apache.lucene.tests.util.TestRuleIgnoreTestSuites;
import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.SuppressForbidden;
import org.junit.Assume;
@@ -58,7 +56,6 @@ import org.junit.BeforeClass;
*/
@SuppressCodecs({"MockRandom", "Direct", "SimpleText"})
@SuppressSysoutChecks(bugUrl = "Stuff gets printed, important stuff for debugging a failure")
-@SuppressForbidden(reason = "We need Unsafe to actually crush :-)")
public class TestSimpleServer extends LuceneTestCase {
static final Set<Thread> clientThreads = Collections.synchronizedSet(new HashSet<>());
@@ -135,39 +132,12 @@ public class TestSimpleServer extends LuceneTestCase {
}
}
- /** currently, this only works/tested on Sun and IBM. */
+ /** forcibly halt the JVM: similar to crashing */
// poached from TestIndexWriterOnJRECrash ... should we factor out to TestUtil? seems dangerous
// to give it such "publicity"?
private static void crashJRE() {
- final String vendor = Constants.JAVA_VENDOR;
- final boolean supportsUnsafeNpeDereference =
- vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple");
-
- try {
- if (supportsUnsafeNpeDereference) {
- try {
- Class<?> clazz = Class.forName("sun.misc.Unsafe");
- java.lang.reflect.Field field = clazz.getDeclaredField("theUnsafe");
- field.setAccessible(true);
- Object o = field.get(null);
- Method m = clazz.getMethod("putAddress", long.class, long.class);
- m.invoke(o, 0L, 0L);
- } catch (Throwable e) {
- System.out.println("Couldn't kill the JVM via Unsafe.");
- e.printStackTrace(System.out);
- }
- }
-
- // Fallback attempt to Runtime.halt();
- Runtime.getRuntime().halt(-1);
- } catch (Exception e) {
- System.out.println("Couldn't kill the JVM.");
- e.printStackTrace(System.out);
- }
-
- // We couldn't get the JVM to crash for some reason.
- throw new RuntimeException("JVM refuses to die!");
+ Runtime.getRuntime().halt(1);
}
static void writeFilesMetaData(DataOutput out, Map<String, FileMetaData> files)