Apache Lucene open-source search software
An eye-gouging way to limit suppressAccessChecks to just the three JARs that need them. (#13164)
Dawid Weiss1c77e2315c41
diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradleindex 4f24d7f412f..44ae964cab0 100644--- a/gradle/testing/randomization.gradle+++ b/gradle/testing/randomization.gradle@@ -119,6 +119,18 @@ allprojects {// Resolve test option values after all evaluation is complete.allprojects {plugins.withType(JavaPlugin) {+ configurations {+ secManagerExclusions+ }+ dependencies {+ secManagerExclusions ( "com.carrotsearch.randomizedtesting:randomizedtesting-runner", {+ exclude group: "junit"+ })+ secManagerExclusions ( "junit:junit", {+ exclude group: "org.hamcrest"+ })+ }+afterEvaluate {ext.testOptionsResolved = testOptions.findAll { opt ->propertyOrDefault(opt.propName, opt.value) != null@@ -155,7 +167,7 @@ allprojects {}// Append resolved test properties to the test task.- tasks.withType(Test) { task ->+ tasks.withType(Test) { Test task ->// TODO: we could remove some options that are only relevant to the build environment// and not the test JVM itself.systemProperties testOptionsResolved@@ -172,13 +184,13 @@ allprojects {// Enable security manager, if requested. We could move the selection of security manager and security policy// to each project's build/ configuration but it seems compact enough to keep it here for now.- def securityArgumentProvider = new SecurityArgumentProvider(commonDir: project(":lucene").layout.projectDirectory)+ def securityArgumentProvider = new SecurityArgumentProvider()+ securityArgumentProvider.commonDir = project(":lucene").layout.projectDirectory+ securityArgumentProvider.otherProperties = project.objects.mapProperty(String, String)+if (Boolean.parseBoolean(testOptionsResolved["tests.useSecurityManager"])) {if (project.path.endsWith(".tests")) {// LUCENE-10301: for now, do not use the security manager for modular tests (test framework is not available).- } else if (project.path == ":lucene:replicator") {- systemProperty 'java.security.manager', "org.apache.lucene.tests.util.TestSecurityManager"- securityArgumentProvider.javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/replicator-tests.policy")} else if (project.path.startsWith(":lucene")) {systemProperty 'java.security.manager', "org.apache.lucene.tests.util.TestSecurityManager"securityArgumentProvider.javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/tests.policy")@@ -189,6 +201,14 @@ allprojects {systemProperty 'gradle.lib.dir', Paths.get(project.class.location.toURI()).parent.toAbsolutePath().toString().replace('\\', '/')systemProperty 'gradle.worker.jar', Paths.get("${gradleUserHome}/caches/${gradle.gradleVersion}/workerMain/gradle-worker.jar").toAbsolutePath().toString()systemProperty 'gradle.user.home', gradleUserHome.toPath().toAbsolutePath().toString()++ securityArgumentProvider.otherProperties.put("randomizedtesting.jar", project.provider { ->+ return configurations.secManagerExclusions.resolve().find { it.name.startsWith("randomizedtesting-runner-") }.absolutePath+ })+ securityArgumentProvider.otherProperties.put("junit.jar", project.provider { ->+ return configurations.secManagerExclusions.resolve().find { it.name.startsWith("junit-") }.absolutePath+ })+ securityArgumentProvider.otherProperties.put("lucene.test.framework", project(":lucene:test-framework").layout.buildDirectory.get().asFile.absolutePath)}doFirst {@@ -242,12 +262,20 @@ class SecurityArgumentProvider implements CommandLineArgumentProvider {@PathSensitive(PathSensitivity.RELATIVE)RegularFile javaSecurityPolicy+ @Input+ MapProperty<String, String> otherProperties+@OverrideIterable<String> asArguments() {def args = ["-Dcommon.dir=${commonDir.getAsFile()}"]if (javaSecurityPolicy) {args.add("-Djava.security.policy=${javaSecurityPolicy.getAsFile()}")}++ otherProperties.getOrElse(Map.of()).forEach { key, value ->+ args.add("-D${key}=${value}")+ }+return args}}diff --git a/gradle/testing/randomization/policies/replicator-tests.policy b/gradle/testing/randomization/policies/replicator-tests.policydeleted file mode 100644index 4d8fcd83f24..00000000000--- a/gradle/testing/randomization/policies/replicator-tests.policy+++ /dev/null@@ -1,94 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--// Policy file for :lucene:replicator tests. Please keep minimal and avoid wildcards.-// this differs from the standard lucene policy in that it must allow read-write access-// to all system properties, because of jetty calls to System.getProperties()--grant {- // jetty-specific:- permission java.lang.RuntimePermission "getenv.JETTY_AVAILABLE_PROCESSORS";- permission java.lang.RuntimePermission "getenv.JETTY_WORKER_INSTANCE";- // allow TestNRTReplication fork its jvm- permission java.io.FilePermission "${java.home}${/}-", "read,execute";- // read/write access to all system properties (required by jetty in these tests)- permission java.util.PropertyPermission "*", "read,write";- // all possibilities of accepting/binding/connecting on localhost with ports >= 1024:- permission java.net.SocketPermission "localhost:1024-", "accept,connect,resolve";- permission java.net.SocketPermission "127.0.0.1:1024-", "accept,connect,resolve";- permission java.net.SocketPermission "[::1]:1024-", "accept,connect,resolve";-- // test-files/ resources- permission java.io.FilePermission "${common.dir}${/}-", "read";-- // write only to sandbox- permission java.io.FilePermission "${tests.linedocsfile}", "read";- permission java.io.FilePermission "${java.io.tmpdir}", "read,write";- permission java.io.FilePermission "${java.io.tmpdir}${/}-", "read,write,delete";-- // needed by randomizedtesting runner to identify test methods.- permission java.lang.RuntimePermission "accessDeclaredMembers";- // needed by certain tests to redirect sysout/syserr:- permission java.lang.RuntimePermission "setIO";- // needed by randomized runner to catch failures from other threads:- permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler";- // needed by randomized runner getTopThreadGroup:- permission java.lang.RuntimePermission "modifyThreadGroup";- // needed by tests e.g. shutting down executors:- permission java.lang.RuntimePermission "modifyThread";- // needed for tons of test hacks etc- permission java.lang.RuntimePermission "getStackTrace";- // needed for mock filesystems in tests- permission java.lang.RuntimePermission "fileSystemProvider";- // test runner seems to require supression of access checks:- permission java.lang.reflect.ReflectPermission "suppressAccessChecks";-};--// Permissions to support ant build-grant {- permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read";- permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";- permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";-};--// Permissions for jacoco code coverage-grant {- // permission to write the per-jvm code coverage output- permission java.io.FilePermission "${user.dir}${/}jacoco.exec", "write";- // needed by jacoco to dump coverage on shutdown- permission java.lang.RuntimePermission "shutdownHooks";- // needed by jacoco to instrument classes- permission java.lang.RuntimePermission "defineClass";- // needed by jacoco for God knows what.- permission java.lang.RuntimePermission "createClassLoader";-};--// Grant all permissions to Gradle test runner classes.-grant codeBase "file:${gradle.lib.dir}${/}-" {- permission java.security.AllPermission;-};--grant codeBase "file:${gradle.worker.jar}" {- permission java.security.AllPermission;-};--grant {- // Allow reading gradle worker JAR.- permission java.io.FilePermission "${gradle.worker.jar}", "read";- // Allow reading from classpath JARs (resources).- permission java.io.FilePermission "${gradle.user.home}${/}-", "read";-};diff --git a/gradle/testing/randomization/policies/tests.policy b/gradle/testing/randomization/policies/tests.policyindex ffb6d0b52d0..f8e09ba0366 100644--- a/gradle/testing/randomization/policies/tests.policy+++ b/gradle/testing/randomization/policies/tests.policy@@ -33,34 +33,29 @@ grant {// misc HardlinkCopyDirectoryWrapper needs this to test if hardlinks can be createdpermission java.nio.file.LinkPermission "hard";- // needed by SSD detection tests in TestIOUtils (creates symlinks)- permission java.nio.file.LinkPermission "symbolic";- // needed by randomizedtesting runner to identify test methods.+ // needed by RamUsageEstimator, and randomizedtesting runner to identify test methods.permission java.lang.RuntimePermission "accessDeclaredMembers";// needed by certain tests to capture sysout/syserr:permission java.lang.RuntimePermission "setIO";- // needed by randomized runner to catch failures from other threads:+ // needed by luke and randomized runner to catch failures from other threads:permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler";- // needed by randomized runner getTopThreadGroup:- permission java.lang.RuntimePermission "modifyThreadGroup";// needed by tests e.g. shutting down executors:permission java.lang.RuntimePermission "modifyThread";// needed for tons of test hacks etcpermission java.lang.RuntimePermission "getStackTrace";// needed for mock filesystems in testspermission java.lang.RuntimePermission "fileSystemProvider";- // test runner seems to require supression of access checks:- permission java.lang.reflect.ReflectPermission "suppressAccessChecks";// needed by cyberneko usage by benchmarks on J9permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util";- // Needed for loading native library (lucene:misc:native) in lucene:misc+ // Needed for DirectIODirectory to retrieve block sizepermission java.lang.RuntimePermission "getFileStoreAttributes";- permission java.lang.RuntimePermission "writeFileDescriptor";// TestLockFactoriesMultiJVM opens a random port on 127.0.0.1 (port 0 = ephemeral port range):permission java.net.SocketPermission "127.0.0.1:0", "accept,listen,resolve";+ // Replicator tests connect to ephemeral ports+ permission java.net.SocketPermission "127.0.0.1:1024-", "connect,resolve";// read access to all system properties:permission java.util.PropertyPermission "*", "read";@@ -87,13 +82,6 @@ grant {permission java.io.FilePermission "${hunspell.dictionaries}${/}-", "read";};-// Permissions to support ant build-grant {- permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read";- permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";- permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";-};-// Permissions for jacoco code coveragegrant {// permission to write the per-jvm code coverage output@@ -122,3 +110,14 @@ grant {permission java.io.FilePermission "${gradle.user.home}${/}-", "read";};+// Grant permissions to certain test-related JARs (https://github.com/apache/lucene/pull/13146)+grant codeBase "file:${randomizedtesting.jar}" {+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";+ permission java.lang.RuntimePermission "modifyThreadGroup";+};+grant codeBase "file:${junit.jar}" {+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";+};+grant codeBase "file:${lucene.test.framework}${/}-" {+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";+};diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.javaindex 296a39af15d..60de20d22bb 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java@@ -22,8 +22,6 @@ import java.io.InputStream;import java.io.OutputStream;import java.io.PrintStream;import java.lang.ProcessBuilder.Redirect;-import java.lang.reflect.Field;-import java.lang.reflect.Method;import java.nio.file.FileVisitResult;import java.nio.file.Files;import java.nio.file.Path;@@ -31,10 +29,10 @@ import java.nio.file.Paths;import java.nio.file.SimpleFileVisitor;import java.util.ArrayList;import java.util.List;+import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicBoolean;import org.apache.lucene.tests.store.BaseDirectoryWrapper;import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.Constants;import org.apache.lucene.util.SuppressForbidden;/**@@ -57,7 +55,8 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {if (System.getProperty("tests.crashmode") == null) {// try up to 10 times to create an indexfor (int i = 0; i < 10; i++) {- forkTest();+ final int crashDelayMs = TestUtil.nextInt(random(), 3000, 4000);+ forkTest(crashDelayMs);// if we succeeded in finding an index, we are done.if (checkIndexes(tempDir)) return;}@@ -68,34 +67,17 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {// assumeFalse("does not support PreFlex, see LUCENE-3992",// Codec.getDefault().getName().equals("Lucene4x"));- // we are the fork, setup a crashing thread- final int crashTime = TestUtil.nextInt(random(), 3000, 4000);- Thread t =- new Thread() {- @SuppressForbidden(reason = "Thread sleep")- @Override- public void run() {- try {- Thread.sleep(crashTime);- } catch (- @SuppressWarnings("unused")- InterruptedException e) {- }- crashJRE();- }- };- t.setPriority(Thread.MAX_PRIORITY);- t.start();- // run the test until we crash.- for (int i = 0; i < 1000; i++) {+ // Effectively an endless loop until the process is killed from the outside.+ for (int i = 0; i < Integer.MAX_VALUE; i++) {super.testNRTThreads();}}}/** fork ourselves in a new jvm. sets -Dtests.crashmode=true */- @SuppressForbidden(reason = "ProcessBuilder requires java.io.File for CWD")- public void forkTest() throws Exception {+ @SuppressForbidden(+ reason = "ProcessBuilder requires java.io.File for setting ProcessBuilder.directory")+ public void forkTest(int crashDelayMs) throws Exception {List<String> cmd = new ArrayList<>();cmd.add(Paths.get(System.getProperty("java.home"), "bin", "java").toString());cmd.add("-Xmx512m");@@ -114,12 +96,21 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {.redirectInput(Redirect.INHERIT).redirectErrorStream(true);Process p = pb.start();-// We pump everything to stderr.PrintStream childOut = System.err;Thread stdoutPumper = ThreadPumper.start(p.getInputStream(), childOut);- if (VERBOSE) childOut.println(">>> Begin subprocess output");- p.waitFor();+ try {+ if (VERBOSE) childOut.println(">>> Begin subprocess output");+ if (p.waitFor(crashDelayMs, TimeUnit.MILLISECONDS)) {+ // This means the process has exited before the timeout. This is odd because it should+ // run in an endless loop?+ throw new AssertionError("Subprocess has exited unexpectedly.");+ }+ } finally {+ // Try to kill the process and wait until it terminates. destroyForcibly is a no-op in case+ // the process is not alive, so no need for additional checks.+ p.destroyForcibly().waitFor();+ }stdoutPumper.join();if (VERBOSE) childOut.println("<<< End subprocess output");}@@ -140,7 +131,7 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {}}} catch (IOException e) {- System.err.println("Couldn't pipe from the forked process: " + e.toString());+ System.err.println("Couldn't pipe from the forked process: " + e);}}};@@ -186,37 +177,4 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {});return found.get();}-- /** currently, this only works/tested on Sun and IBM. */- @SuppressForbidden(reason = "We need Unsafe to actually crush :-)")- public void crashJRE() {- final String vendor = Constants.JAVA_VENDOR;- final boolean supportsUnsafeNpeDereference =- vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple");-- try {- if (supportsUnsafeNpeDereference) {- try {- Class<?> clazz = Class.forName("sun.misc.Unsafe");- Field field = clazz.getDeclaredField("theUnsafe");- field.setAccessible(true);- Object o = field.get(null);- Method m = clazz.getMethod("putAddress", long.class, long.class);- m.invoke(o, 0L, 0L);- } catch (Throwable e) {- System.out.println("Couldn't kill the JVM via Unsafe.");- e.printStackTrace(System.out);- }- }-- // Fallback attempt to Runtime.halt();- Runtime.getRuntime().halt(-1);- } catch (Exception e) {- System.out.println("Couldn't kill the JVM.");- e.printStackTrace(System.out);- }-- // We couldn't get the JVM to crash for some reason.- fail();- }}diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.javaindex c715bd57936..f21ac11b884 100644--- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java+++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java@@ -21,7 +21,6 @@ import java.io.BufferedInputStream;import java.io.BufferedOutputStream;import java.io.IOException;import java.io.InputStream;-import java.lang.reflect.Method;import java.net.InetAddress;import java.net.InetSocketAddress;import java.net.ServerSocket;@@ -45,7 +44,6 @@ import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks;import org.apache.lucene.tests.util.TestRuleIgnoreTestSuites;import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.Constants;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.SuppressForbidden;import org.junit.Assume;@@ -58,7 +56,6 @@ import org.junit.BeforeClass;*/@SuppressCodecs({"MockRandom", "Direct", "SimpleText"})@SuppressSysoutChecks(bugUrl = "Stuff gets printed, important stuff for debugging a failure")-@SuppressForbidden(reason = "We need Unsafe to actually crush :-)")public class TestSimpleServer extends LuceneTestCase {static final Set<Thread> clientThreads = Collections.synchronizedSet(new HashSet<>());@@ -135,39 +132,12 @@ public class TestSimpleServer extends LuceneTestCase {}}- /** currently, this only works/tested on Sun and IBM. */+ /** forcibly halt the JVM: similar to crashing */// poached from TestIndexWriterOnJRECrash ... should we factor out to TestUtil? seems dangerous// to give it such "publicity"?private static void crashJRE() {- final String vendor = Constants.JAVA_VENDOR;- final boolean supportsUnsafeNpeDereference =- vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple");-- try {- if (supportsUnsafeNpeDereference) {- try {- Class<?> clazz = Class.forName("sun.misc.Unsafe");- java.lang.reflect.Field field = clazz.getDeclaredField("theUnsafe");- field.setAccessible(true);- Object o = field.get(null);- Method m = clazz.getMethod("putAddress", long.class, long.class);- m.invoke(o, 0L, 0L);- } catch (Throwable e) {- System.out.println("Couldn't kill the JVM via Unsafe.");- e.printStackTrace(System.out);- }- }-- // Fallback attempt to Runtime.halt();- Runtime.getRuntime().halt(-1);- } catch (Exception e) {- System.out.println("Couldn't kill the JVM.");- e.printStackTrace(System.out);- }-- // We couldn't get the JVM to crash for some reason.- throw new RuntimeException("JVM refuses to die!");+ Runtime.getRuntime().halt(1);}static void writeFilesMetaData(DataOutput out, Map<String, FileMetaData> files)