Apache Lucene open-source search software
Refactor access to VM options and move some VM options to oal.util.Constants (#12754)
Uwe Schindlera35573eed960
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 7467b990080..5fae1e4b88f 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -210,6 +210,9 @@ Improvements* GITHUB#12689: TaskExecutor to cancel all tasks on exception to avoid needless computation. (Luca Cavanna)+* GITHUB#12754: Refactor lookup of Hotspot VM options and do not initialize constants with NULL+ if SecurityManager prevents access. (Uwe Schindler)+Optimizations---------------------* GITHUB#12183: Make TermStates#build concurrent. (Shubham Chaudhary)diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.javaindex ed4066a94ac..3d565b650a9 100644--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java@@ -21,8 +21,6 @@ import java.lang.Runtime.Version;import java.lang.StackWalker.StackFrame;import java.lang.invoke.MethodHandles;import java.lang.invoke.MethodType;-import java.security.AccessController;-import java.security.PrivilegedAction;import java.util.Locale;import java.util.Objects;import java.util.Optional;@@ -31,7 +29,7 @@ import java.util.Set;import java.util.function.Predicate;import java.util.logging.Logger;import java.util.stream.Stream;-import org.apache.lucene.util.SuppressForbidden;+import org.apache.lucene.util.Constants;import org.apache.lucene.util.VectorUtil;/**@@ -129,7 +127,7 @@ public abstract class VectorizationProvider {"Vector bitsize and/or integer vectors enforcement; using default vectorization provider outside of testMode");return new DefaultVectorizationProvider();}- if (isClientVM()) {+ if (Constants.IS_CLIENT_VM) {LOG.warning("C2 compiler is disabled; Java vector incubator API can't be enabled");return new DefaultVectorizationProvider();}@@ -188,23 +186,6 @@ public abstract class VectorizationProvider {&& !Objects.equals("I", "i".toUpperCase(Locale.getDefault()));}- @SuppressWarnings("removal")- @SuppressForbidden(reason = "security manager")- private static boolean isClientVM() {- try {- final PrivilegedAction<Boolean> action =- () -> System.getProperty("java.vm.info", "").contains("emulated-client");- return AccessController.doPrivileged(action);- } catch (- @SuppressWarnings("unused")- SecurityException e) {- LOG.warning(- "SecurityManager denies permission to 'java.vm.info' system property, so state of C2 compiler can't be detected. "- + "In case of performance issues allow access to this property.");- return false;- }- }-// add all possible callers here as FQCN:private static final Set<String> VALID_CALLERS = Set.of("org.apache.lucene.util.VectorUtil");diff --git a/lucene/core/src/java/org/apache/lucene/util/Constants.java b/lucene/core/src/java/org/apache/lucene/util/Constants.javaindex 090472e6736..3ef12986bb2 100644--- a/lucene/core/src/java/org/apache/lucene/util/Constants.java+++ b/lucene/core/src/java/org/apache/lucene/util/Constants.java@@ -16,18 +16,25 @@*/package org.apache.lucene.util;+import java.security.AccessController;+import java.security.PrivilegedAction;+import java.util.Objects;+import java.util.logging.Logger;+/** Some useful constants. */public final class Constants {private Constants() {} // can't construct+ private static final String UNKNOWN = "Unknown";+/** JVM vendor info. */- public static final String JVM_VENDOR = System.getProperty("java.vm.vendor");+ public static final String JVM_VENDOR = getSysProp("java.vm.vendor", UNKNOWN);/** JVM vendor name. */- public static final String JVM_NAME = System.getProperty("java.vm.name");+ public static final String JVM_NAME = getSysProp("java.vm.name", UNKNOWN);/** The value of <code>System.getProperty("os.name")</code>. * */- public static final String OS_NAME = System.getProperty("os.name");+ public static final String OS_NAME = getSysProp("os.name", UNKNOWN);/** True iff running on Linux. */public static final boolean LINUX = OS_NAME.startsWith("Linux");@@ -45,36 +52,67 @@ public final class Constants {public static final boolean FREE_BSD = OS_NAME.startsWith("FreeBSD");/** The value of <code>System.getProperty("os.arch")</code>. */- public static final String OS_ARCH = System.getProperty("os.arch");+ public static final String OS_ARCH = getSysProp("os.arch", UNKNOWN);/** The value of <code>System.getProperty("os.version")</code>. */- public static final String OS_VERSION = System.getProperty("os.version");+ public static final String OS_VERSION = getSysProp("os.version", UNKNOWN);/** The value of <code>System.getProperty("java.vendor")</code>. */- public static final String JAVA_VENDOR = System.getProperty("java.vendor");+ public static final String JAVA_VENDOR = getSysProp("java.vendor", UNKNOWN);++ /** True iff the Java runtime is a client runtime and C2 compiler is not enabled */+ public static final boolean IS_CLIENT_VM =+ getSysProp("java.vm.info", "").contains("emulated-client");/** True iff running on a 64bit JVM */- public static final boolean JRE_IS_64BIT;+ public static final boolean JRE_IS_64BIT = is64Bit();++ /** true iff we know fast FMA is supported, to deliver less error */+ public static final boolean HAS_FAST_FMA =+ (IS_CLIENT_VM == false)+ && Objects.equals(OS_ARCH, "amd64")+ && HotspotVMOptions.get("UseFMA").map(Boolean::valueOf).orElse(false);- static {- boolean is64Bit = false;- String datamodel = null;+ private static boolean is64Bit() {+ final String datamodel = getSysProp("sun.arch.data.model");+ if (datamodel != null) {+ return datamodel.contains("64");+ } else {+ return (OS_ARCH != null && OS_ARCH.contains("64"));+ }+ }++ private static String getSysProp(String property) {try {- datamodel = System.getProperty("sun.arch.data.model");- if (datamodel != null) {- is64Bit = datamodel.contains("64");- }+ return doPrivileged(() -> System.getProperty(property));} catch (@SuppressWarnings("unused")- SecurityException ex) {+ SecurityException se) {+ logSecurityWarning(property);+ return null;}- if (datamodel == null) {- if (OS_ARCH != null && OS_ARCH.contains("64")) {- is64Bit = true;- } else {- is64Bit = false;- }+ }++ private static String getSysProp(String property, String def) {+ try {+ return doPrivileged(() -> System.getProperty(property, def));+ } catch (+ @SuppressWarnings("unused")+ SecurityException se) {+ logSecurityWarning(property);+ return def;}- JRE_IS_64BIT = is64Bit;+ }++ private static void logSecurityWarning(String property) {+ var log = Logger.getLogger(Constants.class.getName());+ log.warning("SecurityManager prevented access to system property: " + property);+ }++ // Extracted to a method to be able to apply the SuppressForbidden annotation+ @SuppressWarnings("removal")+ @SuppressForbidden(reason = "security manager")+ private static <T> T doPrivileged(PrivilegedAction<T> action) {+ return AccessController.doPrivileged(action);}}diff --git a/lucene/core/src/java/org/apache/lucene/util/HotspotVMOptions.java b/lucene/core/src/java/org/apache/lucene/util/HotspotVMOptions.javanew file mode 100644index 00000000000..70f963e1b37--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/util/HotspotVMOptions.java@@ -0,0 +1,90 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.util;++import java.lang.reflect.Method;+import java.util.Objects;+import java.util.Optional;+import java.util.function.Function;+import java.util.logging.Logger;++/** Accessor to get Hotspot VM Options (if available). */+final class HotspotVMOptions {+ private HotspotVMOptions() {} // can't construct++ /** True if the Java VM is based on Hotspot and has the Hotspot MX bean readable by Lucene */+ public static final boolean IS_HOTSPOT;++ /**+ * Returns an optional with the value of a Hotspot VM option. If the VM option does not exist or+ * is not readable, returns an empty optional.+ */+ public static Optional<String> get(String name) {+ return ACCESSOR.apply(Objects.requireNonNull(name, "name"));+ }++ private static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory";+ private static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean";+ private static final Function<String, Optional<String>> ACCESSOR;++ static {+ boolean isHotspot = false;+ Function<String, Optional<String>> accessor = name -> Optional.empty();+ try {+ final Class<?> beanClazz = Class.forName(HOTSPOT_BEAN_CLASS);+ // we use reflection for this, because the management factory is not part+ // of java.base module:+ final Object hotSpotBean =+ Class.forName(MANAGEMENT_FACTORY_CLASS)+ .getMethod("getPlatformMXBean", Class.class)+ .invoke(null, beanClazz);+ if (hotSpotBean != null) {+ final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);+ final Method getValueMethod = getVMOptionMethod.getReturnType().getMethod("getValue");+ isHotspot = true;+ accessor =+ name -> {+ try {+ final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, name);+ return Optional.of(getValueMethod.invoke(vmOption).toString());+ } catch (@SuppressWarnings("unused")+ ReflectiveOperationException+ | RuntimeException e) {+ return Optional.empty();+ }+ };+ }+ } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {+ isHotspot = false;+ final Logger log = Logger.getLogger(HotspotVMOptions.class.getName());+ final Module module = HotspotVMOptions.class.getModule();+ final ModuleLayer layer = module.getLayer();+ // classpath / unnamed module has no layer, so we need to check:+ if (layer != null+ && layer.findModule("jdk.management").map(module::canRead).orElse(false) == false) {+ log.warning(+ "Lucene cannot access JVM internals to optimize algorithms or calculate object sizes, unless the 'jdk.management' Java module "+ + "is readable [please add 'jdk.management' to modular application either by command line or its module descriptor].");+ } else {+ log.warning(+ "Lucene cannot optimize algorithms or calculate object sizes for JVMs that are not based on Hotspot or a compatible implementation.");+ }+ }+ IS_HOTSPOT = isHotspot;+ ACCESSOR = accessor;+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java b/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.javaindex 1d363170a94..7e0bdfdee84 100644--- a/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java+++ b/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java@@ -18,7 +18,6 @@ package org.apache.lucene.util;import java.lang.reflect.Array;import java.lang.reflect.Field;-import java.lang.reflect.Method;import java.lang.reflect.Modifier;import java.security.AccessControlException;import java.security.AccessController;@@ -30,7 +29,6 @@ import java.util.Collections;import java.util.IdentityHashMap;import java.util.Locale;import java.util.Map;-import java.util.logging.Logger;import org.apache.lucene.index.Term;import org.apache.lucene.search.BooleanClause;import org.apache.lucene.search.Query;@@ -112,64 +110,16 @@ public final class RamUsageEstimator {/** For testing only */static final boolean JVM_IS_HOTSPOT_64BIT;- static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory";- static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean";-/** Initialize constants and try to collect information about the JVM internals. */static {- if (Constants.JRE_IS_64BIT) {+ if (Constants.JRE_IS_64BIT && HotspotVMOptions.IS_HOTSPOT) {// Try to get compressed oops and object alignment (the default seems to be 8 on Hotspot);// (this only works on 64 bit, on 32 bits the alignment and reference size is fixed):- boolean compressedOops = false;- int objectAlignment = 8;- boolean isHotspot = false;- try {- final Class<?> beanClazz = Class.forName(HOTSPOT_BEAN_CLASS);- // we use reflection for this, because the management factory is not part- // of Java 8's compact profile:- final Object hotSpotBean =- Class.forName(MANAGEMENT_FACTORY_CLASS)- .getMethod("getPlatformMXBean", Class.class)- .invoke(null, beanClazz);- if (hotSpotBean != null) {- isHotspot = true;- final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);- try {- final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "UseCompressedOops");- compressedOops =- Boolean.parseBoolean(- vmOption.getClass().getMethod("getValue").invoke(vmOption).toString());- } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {- isHotspot = false;- }- try {- final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes");- objectAlignment =- Integer.parseInt(- vmOption.getClass().getMethod("getValue").invoke(vmOption).toString());- } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {- isHotspot = false;- }- }- } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {- isHotspot = false;- final Logger log = Logger.getLogger(RamUsageEstimator.class.getName());- final Module module = RamUsageEstimator.class.getModule();- final ModuleLayer layer = module.getLayer();- // classpath / unnamed module has no layer, so we need to check:- if (layer != null- && layer.findModule("jdk.management").map(module::canRead).orElse(false) == false) {- log.warning(- "Lucene cannot correctly calculate object sizes on 64bit JVMs, unless the 'jdk.management' Java module "- + "is readable [please add 'jdk.management' to modular application either by command line or its module descriptor]");- } else {- log.warning(- "Lucene cannot correctly calculate object sizes on 64bit JVMs that are not based on Hotspot or a compatible implementation.");- }- }- JVM_IS_HOTSPOT_64BIT = isHotspot;- COMPRESSED_REFS_ENABLED = compressedOops;- NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment;+ JVM_IS_HOTSPOT_64BIT = true;+ COMPRESSED_REFS_ENABLED =+ HotspotVMOptions.get("UseCompressedOops").map(Boolean::valueOf).orElse(false);+ NUM_BYTES_OBJECT_ALIGNMENT =+ HotspotVMOptions.get("ObjectAlignmentInBytes").map(Integer::valueOf).orElse(8);// reference size is 4, if we have compressed oops:NUM_BYTES_OBJECT_REF = COMPRESSED_REFS_ENABLED ? 4 : 8;// "best guess" based on reference size:diff --git a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.javaindex 5b382c4c7c2..d4e8a50ef8f 100644--- a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java+++ b/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java@@ -77,41 +77,9 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {VectorizationProvider.TESTS_FORCE_INTEGER_VECTORS || (isAMD64withoutAVX2 == false);}- private static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory";- private static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean";-- // best effort to see if FMA is fast (this is architecture-independent option)- private static boolean hasFastFMA() {- // on ARM cpus, FMA works fine but is a slight slowdown: don't use it.- if (Constants.OS_ARCH.equals("amd64") == false) {- return false;- }- try {- final Class<?> beanClazz = Class.forName(HOTSPOT_BEAN_CLASS);- // we use reflection for this, because the management factory is not part- // of Java 8's compact profile:- final Object hotSpotBean =- Class.forName(MANAGEMENT_FACTORY_CLASS)- .getMethod("getPlatformMXBean", Class.class)- .invoke(null, beanClazz);- if (hotSpotBean != null) {- final var getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);- final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "UseFMA");- return Boolean.parseBoolean(- vmOption.getClass().getMethod("getValue").invoke(vmOption).toString());- }- return false;- } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {- return false;- }- }-- // true if we know FMA is supported, to deliver less error- static final boolean HAS_FAST_FMA = hasFastFMA();-// the way FMA should work! if available use it, otherwise fall back to mul/addprivate static FloatVector fma(FloatVector a, FloatVector b, FloatVector c) {- if (HAS_FAST_FMA) {+ if (Constants.HAS_FAST_FMA) {return a.fma(b, c);} else {return a.mul(b).add(c);diff --git a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java b/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.javaindex fc303a687a0..ffd18df1a27 100644--- a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java+++ b/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java@@ -21,6 +21,7 @@ import java.security.PrivilegedAction;import java.util.Locale;import java.util.logging.Logger;import jdk.incubator.vector.FloatVector;+import org.apache.lucene.util.Constants;import org.apache.lucene.util.SuppressForbidden;/** A vectorization provider that leverages the Panama Vector API. */@@ -62,7 +63,7 @@ final class PanamaVectorizationProvider extends VectorizationProvider {Locale.ENGLISH,"Java vector incubator API enabled; uses preferredBitSize=%d%s%s",PanamaVectorUtilSupport.VECTOR_BITSIZE,- PanamaVectorUtilSupport.HAS_FAST_FMA ? "; FMA enabled" : "",+ Constants.HAS_FAST_FMA ? "; FMA enabled" : "",PanamaVectorUtilSupport.HAS_FAST_INTEGER_VECTORS? "": "; floating-point vectors only"));