Apache Solr open-source search software
SOLR-16809: Converge logic for hidden sysProps
Houston Putman98c198810f2c
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 5e8640a2f4f..4a5cb6cb0d4 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -269,6 +269,8 @@ Bug Fixes* SOLR-16619: Fix solr scripts running on IBM i (Jesse Gorzinski via Eric Pugh)+* SOLR-16809: The configuration for hiding sensitive sysProp information has been joined under `-Dsolr.hiddenSysProps` and `SOLR_HIDDEN_SYS_PROPS`. (Houston Putman, David Smiley)+Dependency Upgrades---------------------* PR#1494: Upgrade forbiddenapis to 3.5 (Uwe Schindler)diff --git a/solr/core/src/java/org/apache/solr/core/MetricsConfig.java b/solr/core/src/java/org/apache/solr/core/MetricsConfig.javaindex 08f039927d2..75c98707f77 100644--- a/solr/core/src/java/org/apache/solr/core/MetricsConfig.java+++ b/solr/core/src/java/org/apache/solr/core/MetricsConfig.java@@ -17,14 +17,11 @@package org.apache.solr.core;import java.util.Collections;-import java.util.HashSet;-import java.util.Set;/** */public class MetricsConfig {private final PluginInfo[] metricReporters;- private final Set<String> hiddenSysProps;private final PluginInfo counterSupplier;private final PluginInfo meterSupplier;private final PluginInfo timerSupplier;@@ -39,7 +36,6 @@ public class MetricsConfig {private MetricsConfig(boolean enabled,PluginInfo[] metricReporters,- Set<String> hiddenSysProps,PluginInfo counterSupplier,PluginInfo meterSupplier,PluginInfo timerSupplier,@@ -51,7 +47,6 @@ public class MetricsConfig {CacheConfig cacheConfig) {this.enabled = enabled;this.metricReporters = metricReporters;- this.hiddenSysProps = hiddenSysProps;this.counterSupplier = counterSupplier;this.meterSupplier = meterSupplier;this.timerSupplier = timerSupplier;@@ -97,14 +92,6 @@ public class MetricsConfig {return nullObject;}- public Set<String> getHiddenSysProps() {- if (enabled) {- return hiddenSysProps;- } else {- return Collections.emptySet();- }- }-/** Symbolic name to use as plugin class name for no-op implementations. */public static final String NOOP_IMPL_CLASS = "__noop__";@@ -145,7 +132,6 @@ public class MetricsConfig {public static class MetricsConfigBuilder {private PluginInfo[] metricReporterPlugins = new PluginInfo[0];- private Set<String> hiddenSysProps = new HashSet<>();private PluginInfo counterSupplier;private PluginInfo meterSupplier;private PluginInfo timerSupplier;@@ -170,14 +156,6 @@ public class MetricsConfig {return this;}- public MetricsConfigBuilder setHiddenSysProps(Set<String> hiddenSysProps) {- if (hiddenSysProps != null && !hiddenSysProps.isEmpty()) {- this.hiddenSysProps.clear();- this.hiddenSysProps.addAll(hiddenSysProps);- }- return this;- }-public MetricsConfigBuilder setMetricReporterPlugins(PluginInfo[] metricReporterPlugins) {this.metricReporterPlugins =metricReporterPlugins != null ? metricReporterPlugins : new PluginInfo[0];@@ -228,7 +206,6 @@ public class MetricsConfig {return new MetricsConfig(enabled,metricReporterPlugins,- hiddenSysProps,counterSupplier,meterSupplier,timerSupplier,diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java b/solr/core/src/java/org/apache/solr/core/NodeConfig.javaindex 169d4b4643f..8117aad4523 100644--- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java+++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java@@ -25,13 +25,15 @@ import java.nio.file.Path;import java.nio.file.Paths;import java.util.Arrays;import java.util.Collections;-import java.util.HashSet;import java.util.LinkedHashSet;import java.util.List;import java.util.Map;import java.util.Properties;import java.util.Set;import java.util.concurrent.TimeUnit;+import java.util.function.Predicate;+import java.util.regex.Pattern;+import java.util.stream.Collectors;import org.apache.lucene.search.IndexSearcher;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;@@ -68,6 +70,9 @@ public class NodeConfig {private final String modules;+ private final Set<String> hiddenSysProps;+ private final Predicate<String> hiddenSysPropPattern;+private final PluginInfo shardHandlerFactoryConfig;private final UpdateShardHandlerConfig updateShardHandlerConfig;@@ -145,7 +150,8 @@ public class NodeConfig {Set<Path> allowPaths,List<String> allowUrls,String configSetServiceClass,- String modules) {+ String modules,+ Set<String> hiddenSysProps) {// all Path params here are absolute and normalized.this.nodeName = nodeName;this.coreRootDirectory = coreRootDirectory;@@ -180,6 +186,10 @@ public class NodeConfig {this.allowUrls = allowUrls;this.configSetServiceClass = configSetServiceClass;this.modules = modules;+ this.hiddenSysProps = hiddenSysProps;+ this.hiddenSysPropPattern =+ Pattern.compile("^(" + String.join("|", hiddenSysProps) + ")$", Pattern.CASE_INSENSITIVE)+ .asMatchPredicate();if (this.cloudConfig != null && this.getCoreLoadThreadCount(false) < 2) {throw new SolrException(@@ -464,6 +474,25 @@ public class NodeConfig {return modules;}+ /** Returns the list of hidden system properties. The list values are regex expressions */+ public Set<String> getHiddenSysProps() {+ return hiddenSysProps;+ }++ /** Returns whether a given system property is hidden */+ public boolean isSysPropHidden(String sysPropName) {+ return hiddenSysPropPattern.test(sysPropName);+ }++ public static final String REDACTED_SYS_PROP_VALUE = "--REDACTED--";++ /** Returns the a system property value, or "--REDACTED--" if the system property is hidden */+ public String getRedactedSysPropValue(String sysPropName) {+ return hiddenSysPropPattern.test(sysPropName)+ ? REDACTED_SYS_PROP_VALUE+ : System.getProperty(sysPropName);+ }+// Finds every jar in each folder and adds it to shardLib, then reloads Lucene SPIprivate void addFoldersToSharedLib(Set<String> libDirs) {boolean modified = false;@@ -551,6 +580,7 @@ public class NodeConfig {private Path configSetBaseDirectory;private String sharedLibDirectory;private String modules;+ private String hiddenSysProps;private PluginInfo shardHandlerFactoryConfig;private UpdateShardHandlerConfig updateShardHandlerConfig = UpdateShardHandlerConfig.DEFAULT;private String configSetServiceClass;@@ -595,16 +625,17 @@ public class NodeConfig {"org.apache.solr.handler.admin.ConfigSetsHandler";public static final Set<String> DEFAULT_HIDDEN_SYS_PROPS =- new HashSet<>(- Arrays.asList(- "javax.net.ssl.keyStorePassword",- "javax.net.ssl.trustStorePassword",- "basicauth",- "zkDigestPassword",- "zkDigestReadonlyPassword",- "aws.secretKey", // AWS SDK v1- "aws.secretAccessKey", // AWS SDK v2- "http.proxyPassword"));+ Set.of(+ "javax\\.net\\.ssl\\.keyStorePassword",+ "javax\\.net\\.ssl\\.trustStorePassword",+ "basicauth",+ "zkDigestPassword",+ "zkDigestReadonlyPassword",+ "aws\\.secretKey", // AWS SDK v1+ "aws\\.secretAccessKey", // AWS SDK v2+ "http\\.proxyPassword",+ ".*password.*",+ ".*secret.*");public NodeConfigBuilder(String nodeName, Path solrHome) {this.nodeName = nodeName;@@ -779,6 +810,44 @@ public class NodeConfig {return this;}+ public NodeConfigBuilder setHiddenSysProps(String hiddenSysProps) {+ this.hiddenSysProps = hiddenSysProps;+ return this;+ }++ /**+ * Finds list of hiddenSysProps requested by system property or environment variable or the+ * default+ *+ * @return set of raw hidden sysProps, may be regex+ */+ private Set<String> resolveHiddenSysPropsFromSysPropOrEnvOrDefault(String hiddenSysProps) {+ // Fall back to sysprop and env.var if nothing configured through solr.xml+ if (!StrUtils.isNotNullOrEmpty(hiddenSysProps)) {+ String fromProps = System.getProperty("solr.hiddenSysProps");+ // Back-compat for solr 9x+ // DEPRECATED: Remove in 10.0+ if (StrUtils.isNotNullOrEmpty(fromProps)) {+ fromProps = System.getProperty("solr.redaction.system.pattern");+ }+ String fromEnv = System.getenv("SOLR_HIDDEN_SYS_PROPS");+ if (StrUtils.isNotNullOrEmpty(fromProps)) {+ hiddenSysProps = fromProps;+ } else if (StrUtils.isNotNullOrEmpty(fromEnv)) {+ hiddenSysProps = fromEnv;+ }+ }+ Set<String> hiddenSysPropSet = Collections.emptySet();+ if (hiddenSysProps != null) {+ hiddenSysPropSet =+ StrUtils.splitSmart(hiddenSysProps, ',').stream()+ .map(String::trim)+ .filter(s -> !s.isEmpty())+ .collect(Collectors.toSet());+ }+ return hiddenSysPropSet.isEmpty() ? DEFAULT_HIDDEN_SYS_PROPS : hiddenSysPropSet;+ }+public NodeConfig build() {// if some things weren't set then set them now. Simple primitives are set on the field// declaration@@ -818,7 +887,8 @@ public class NodeConfig {allowPaths,allowUrls,configSetServiceClass,- modules);+ modules,+ resolveHiddenSysPropsFromSysPropOrEnvOrDefault(hiddenSysProps));}public NodeConfigBuilder setSolrResourceLoader(SolrResourceLoader resourceLoader) {diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.javaindex 935f6bfa2fc..676026c82ed 100644--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java@@ -172,6 +172,10 @@ public class SolrXmlConfig {if (cloudConfig != null) configBuilder.setCloudConfig(cloudConfig);configBuilder.setBackupRepositoryPlugins(getBackupRepositoryPluginInfos(root.get("backup").getAll("repository")));+ // <metrics><hiddenSysProps></metrics> will be removed in Solr 10, but until then, use it if a+ // <hiddenSysProps> is not provided under <solr>.+ // Remove this line in 10.0+ configBuilder.setHiddenSysProps(getHiddenSysProps(root.get("metrics")));configBuilder.setMetricsConfig(getMetricsConfig(root.get("metrics")));configBuilder.setFromZookeeper(fromZookeeper);configBuilder.setDefaultZkHost(defaultZkHost);@@ -360,6 +364,9 @@ public class SolrXmlConfig {case "modules":builder.setModules(it.txt());break;+ case "hiddenSysProps":+ builder.setHiddenSysProps(it.txt());+ break;case "allowPaths":builder.setAllowPaths(separatePaths(it.txt()));break;@@ -404,6 +411,13 @@ public class SolrXmlConfig {return Arrays.asList(COMMA_SEPARATED_PATTERN.split(commaSeparatedString));}+ private static Set<String> separateStringsToSet(String commaSeparatedString) {+ if (StrUtils.isNullOrEmpty(commaSeparatedString)) {+ return Collections.emptySet();+ }+ return Set.of(COMMA_SEPARATED_PATTERN.split(commaSeparatedString));+ }+private static Set<Path> separatePaths(String commaSeparatedString) {if (StrUtils.isNullOrEmpty(commaSeparatedString)) {return Collections.emptySet();@@ -673,11 +687,7 @@ public class SolrXmlConfig {}PluginInfo[] reporterPlugins = getMetricReporterPluginInfos(metrics);- Set<String> hiddenSysProps = getHiddenSysProps(metrics);- return builder- .setMetricReporterPlugins(reporterPlugins)- .setHiddenSysProps(hiddenSysProps)- .build();+ return builder.setMetricReporterPlugins(reporterPlugins).build();}private static Object decodeNullValue(Object o) {@@ -721,20 +731,24 @@ public class SolrXmlConfig {return configs.toArray(new PluginInfo[configs.size()]);}- private static Set<String> getHiddenSysProps(ConfigNode metrics) {+ /**+ * Deprecated as of 9.3, will be removed in 10.0+ *+ * @param metrics configNode for the metrics+ * @return a comma-separated list of hidden Sys Props+ */+ @Deprecated(forRemoval = true, since = "9.3")+ private static String getHiddenSysProps(ConfigNode metrics) {ConfigNode p = metrics.get("hiddenSysProps");- if (!p.exists()) return NodeConfig.NodeConfigBuilder.DEFAULT_HIDDEN_SYS_PROPS;+ if (!p.exists()) return null;Set<String> props = new HashSet<>();p.forEachChild(it -> {- if (it.name().equals("str") && StrUtils.isNotNullOrEmpty(it.txt())) props.add(it.txt());+ if (it.name().equals("str") && StrUtils.isNotNullOrEmpty(it.txt()))+ props.add(Pattern.quote(it.txt()));return Boolean.TRUE;});- if (props.isEmpty()) {- return NodeConfig.NodeConfigBuilder.DEFAULT_HIDDEN_SYS_PROPS;- } else {- return props;- }+ return String.join(",", props);}private static PluginInfo getPluginInfo(ConfigNode cfg) {diff --git a/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.javaindex b982b468935..455bd361247 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java@@ -48,7 +48,7 @@ public class InfoHandler extends RequestHandlerBase {public InfoHandler(final CoreContainer coreContainer) {this.coreContainer = coreContainer;handlers.put("threads", new ThreadDumpHandler());- handlers.put("properties", new PropertiesRequestHandler());+ handlers.put("properties", new PropertiesRequestHandler(coreContainer));handlers.put("logging", new LoggingHandler(coreContainer));handlers.put("system", new SystemInfoHandler(coreContainer));if (coreContainer.getHealthCheckHandler() == null) {diff --git a/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.javaindex 2a65e6a5c62..8658adf3c52 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java@@ -25,45 +25,49 @@ import org.apache.solr.api.AnnotatedApi;import org.apache.solr.api.Api;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.NodeConfig;import org.apache.solr.handler.RequestHandlerBase;import org.apache.solr.handler.admin.api.NodePropertiesAPI;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.security.AuthorizationContext;-import org.apache.solr.util.RedactionUtils;/*** @since solr 1.2*/public class PropertiesRequestHandler extends RequestHandlerBase {- public static final String REDACT_STRING = RedactionUtils.getRedactString();+ private CoreContainer cc;++ public PropertiesRequestHandler() {+ this(null);+ }++ public PropertiesRequestHandler(CoreContainer cc) {+ super();+ this.cc = cc;+ }@Overridepublic void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {NamedList<String> props = new SimpleOrderedMap<>();String name = req.getParams().get(NAME);+ NodeConfig nodeConfig = getCoreContainer(req).getNodeConfig();if (name != null) {- String property = getSecuredPropertyValue(name);+ String property = nodeConfig.getRedactedSysPropValue(name);props.add(name, property);} else {Enumeration<?> enumeration = System.getProperties().propertyNames();while (enumeration.hasMoreElements()) {name = (String) enumeration.nextElement();- props.add(name, getSecuredPropertyValue(name));+ props.add(name, nodeConfig.getRedactedSysPropValue(name));}}rsp.add("system.properties", props);rsp.setHttpCaching(false);}- private String getSecuredPropertyValue(String name) {- if (RedactionUtils.isSystemPropertySensitive(name)) {- return REDACT_STRING;- }- return System.getProperty(name);- }-//////////////////////// SolrInfoMBeans methods //////////////////////@Override@@ -90,4 +94,9 @@ public class PropertiesRequestHandler extends RequestHandlerBase {public Name getPermissionName(AuthorizationContext request) {return Name.CONFIG_READ_PERM;}++ private CoreContainer getCoreContainer(SolrQueryRequest req) {+ CoreContainer coreContainer = req.getCoreContainer();+ return coreContainer == null ? cc : coreContainer;+ }}diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.javaindex e9c5061c7e1..6f30e06b340 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java@@ -42,6 +42,7 @@ import org.apache.solr.api.Api;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.NodeConfig;import org.apache.solr.core.SolrCore;import org.apache.solr.handler.RequestHandlerBase;import org.apache.solr.handler.admin.api.NodeSystemInfoAPI;@@ -53,7 +54,6 @@ import org.apache.solr.security.AuthorizationPlugin;import org.apache.solr.security.PKIAuthenticationPlugin;import org.apache.solr.security.RuleBasedAuthorizationPluginBase;import org.apache.solr.util.RTimer;-import org.apache.solr.util.RedactionUtils;import org.apache.solr.util.stats.MetricUtils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -66,8 +66,6 @@ import org.slf4j.LoggerFactory;public class SystemInfoHandler extends RequestHandlerBase {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- public static String REDACT_STRING = RedactionUtils.getRedactString();-/*** Undocumented expert level system property to prevent doing a reverse lookup of our hostname.* This property will be logged as a suggested workaround if any problems are noticed when doing@@ -152,7 +150,8 @@ public class SystemInfoHandler extends RequestHandlerBase {}rsp.add("lucene", getLuceneInfo());- rsp.add("jvm", getJvmInfo());+ NodeConfig nodeConfig = getCoreContainer(req).getNodeConfig();+ rsp.add("jvm", getJvmInfo(nodeConfig));rsp.add("security", getSecurityInfo(req));rsp.add("system", getSystemInfo());if (solrCloudMode) {@@ -235,7 +234,7 @@ public class SystemInfoHandler extends RequestHandlerBase {}/** Get JVM Info - including memory info */- public static SimpleOrderedMap<Object> getJvmInfo() {+ public static SimpleOrderedMap<Object> getJvmInfo(NodeConfig nodeConfig) {SimpleOrderedMap<Object> jvm = new SimpleOrderedMap<>();final String javaVersion = System.getProperty("java.specification.version", "unknown");@@ -304,7 +303,7 @@ public class SystemInfoHandler extends RequestHandlerBase {// the input arguments passed to the Java virtual machine// which does not include the arguments to the main method.- jmx.add("commandLineArgs", getInputArgumentsRedacted(mx));+ jmx.add("commandLineArgs", getInputArgumentsRedacted(nodeConfig, mx));jmx.add("startTime", new Date(mx.getStartTime()));jmx.add("upTimeMS", mx.getUptime());@@ -408,14 +407,18 @@ public class SystemInfoHandler extends RequestHandlerBase {return newSizeAndUnits;}- private static List<String> getInputArgumentsRedacted(RuntimeMXBean mx) {+ private static List<String> getInputArgumentsRedacted(NodeConfig nodeConfig, RuntimeMXBean mx) {List<String> list = new ArrayList<>();for (String arg : mx.getInputArguments()) {if (arg.startsWith("-D")&& arg.contains("=")- && RedactionUtils.isSystemPropertySensitive(arg.substring(2, arg.indexOf('=')))) {+ && nodeConfig.isSysPropHidden(arg.substring(2, arg.indexOf('=')))) {list.add(- String.format(Locale.ROOT, "%s=%s", arg.substring(0, arg.indexOf('=')), REDACT_STRING));+ String.format(+ Locale.ROOT,+ "%s=%s",+ arg.substring(0, arg.indexOf('=')),+ NodeConfig.REDACTED_SYS_PROP_VALUE));} else {list.add(arg);}diff --git a/solr/core/src/java/org/apache/solr/servlet/CoreContainerProvider.java b/solr/core/src/java/org/apache/solr/servlet/CoreContainerProvider.javaindex 47eb8198e82..d5988f01990 100644--- a/solr/core/src/java/org/apache/solr/servlet/CoreContainerProvider.java+++ b/solr/core/src/java/org/apache/solr/servlet/CoreContainerProvider.java@@ -42,7 +42,6 @@ import java.util.Map;import java.util.Objects;import java.util.Optional;import java.util.Properties;-import java.util.Set;import java.util.WeakHashMap;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;@@ -415,7 +414,7 @@ public class CoreContainerProvider implements ServletContextListener {private void setupJvmMetrics(CoreContainer coresInit, MetricsConfig config) {metricManager = coresInit.getMetricManager();registryName = SolrMetricManager.getRegistryName(Group.jvm);- final Set<String> hiddenSysProps = coresInit.getConfig().getMetricsConfig().getHiddenSysProps();+ final NodeConfig nodeConfig = coresInit.getConfig();try {metricManager.registerAll(registryName, new AltBufferPoolMetricSet(), ResolutionStrategy.IGNORE, "buffers");@@ -455,8 +454,7 @@ public class CoreContainerProvider implements ServletContextListener {System.getProperties().forEach((k, v) -> {- //noinspection SuspiciousMethodCalls- if (!hiddenSysProps.contains(k)) {+ if (!nodeConfig.isSysPropHidden(String.valueOf(k))) {map.putNoEx(String.valueOf(k), v);}}));diff --git a/solr/core/src/java/org/apache/solr/util/RedactionUtils.java b/solr/core/src/java/org/apache/solr/util/RedactionUtils.javadeleted file mode 100644index d6e866d53f5..00000000000--- a/solr/core/src/java/org/apache/solr/util/RedactionUtils.java+++ /dev/null@@ -1,77 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.util;--import java.util.Comparator;-import java.util.Map;-import java.util.TreeMap;-import java.util.regex.Pattern;--public class RedactionUtils {- public static final String SOLR_REDACTION_SYSTEM_PATTERN_PROP = "solr.redaction.system.pattern";- private static Pattern pattern =- Pattern.compile(- System.getProperty(SOLR_REDACTION_SYSTEM_PATTERN_PROP, ".*password.*"),- Pattern.CASE_INSENSITIVE);- private static final String REDACT_STRING = "--REDACTED--";- public static final String NODE_REDACTION_PREFIX = "N_";- public static final String COLL_REDACTION_PREFIX = "COLL_";-- private static boolean redactSystemProperty =- Boolean.parseBoolean(System.getProperty("solr.redaction.system.enabled", "true"));-- /**- * Returns if the given system property should be redacted.- *- * @param name The system property that is being checked.- * @return true if property should be redacted.- */- public static boolean isSystemPropertySensitive(String name) {- return redactSystemProperty && pattern.matcher(name).matches();- }-- /**- * @return redaction string to be used instead of the value.- */- public static String getRedactString() {- return REDACT_STRING;- }-- public static void setRedactSystemProperty(boolean redactSystemProperty) {- RedactionUtils.redactSystemProperty = redactSystemProperty;- }-- /**- * Replace actual names found in a string with redacted names.- *- * @param redactions a map of original to redacted names- * @param data string to redact- * @return redacted string where all actual names have been replaced.- */- public static String redactNames(Map<String, String> redactions, String data) {- // replace the longest first to avoid partial replacements- Map<String, String> sorted =- new TreeMap<>(- Comparator.comparing(String::length).reversed().thenComparing(String::compareTo));- sorted.putAll(redactions);- for (Map.Entry<String, String> entry : sorted.entrySet()) {- data = data.replaceAll("\\Q" + entry.getKey() + "\\E", entry.getValue());- }- return data;- }-}diff --git a/solr/core/src/test-files/solr/solr-hiddensysprops.xml b/solr/core/src/test-files/solr/solr-hiddensysprops.xmlindex 20e5aec2175..eb638daf819 100644--- a/solr/core/src/test-files/solr/solr-hiddensysprops.xml+++ b/solr/core/src/test-files/solr/solr-hiddensysprops.xml@@ -17,12 +17,8 @@--><solr>+ <str name="hiddenSysProps">foo,bar,baz</str><metrics>- <hiddenSysProps>- <str>foo</str>- <str>bar</str>- <str>baz</str>- </hiddenSysProps><!-- this reporter doesn't specify 'group' or 'registry', it will be instantiated for any group. --><reporter name="universal" class="org.apache.solr.metrics.reporters.MockMetricReporter"><str name="configurable">configured</str>diff --git a/solr/core/src/test-files/solr/solr-metricsconfig.xml b/solr/core/src/test-files/solr/solr-metricsconfig.xmlindex 3e656826406..acab241f3f1 100644--- a/solr/core/src/test-files/solr/solr-metricsconfig.xml+++ b/solr/core/src/test-files/solr/solr-metricsconfig.xml@@ -17,6 +17,7 @@--><solr>+ <str name="hiddenSysProps">foo,bar,baz</str><metrics enabled="${metricsEnabled:true}"><suppliers><counter class="${counter.class:}">@@ -48,11 +49,6 @@<long name="window">${histogram.window:-1}</long></histogram></suppliers>- <hiddenSysProps>- <str>foo</str>- <str>bar</str>- <str>baz</str>- </hiddenSysProps><!-- this reporter doesn't specify 'group' or 'registry', it will be instantiated for any group. --><reporter name="universal" class="org.apache.solr.metrics.reporters.MockMetricReporter"><str name="configurable">configured</str>diff --git a/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.javaindex 7332d8525a5..c9120121586 100644--- a/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java+++ b/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java@@ -22,14 +22,13 @@ import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;import org.apache.solr.client.solrj.request.GenericSolrRequest;import org.apache.solr.common.util.NamedList;-import org.apache.solr.util.RedactionUtils;+import org.apache.solr.core.NodeConfig;import org.junit.BeforeClass;import org.junit.Test;public class PropertiesRequestHandlerTest extends SolrTestCaseJ4 {public static final String PASSWORD = "secret123";- public static final String REDACT_STRING = RedactionUtils.getRedactString();@BeforeClasspublic static void beforeClass() throws Exception {@@ -38,23 +37,17 @@ public class PropertiesRequestHandlerTest extends SolrTestCaseJ4 {@Testpublic void testRedaction() throws Exception {- RedactionUtils.setRedactSystemProperty(true);- for (String propName : new String[] {"some.password", "javax.net.ssl.trustStorePassword"}) {+ for (String propName :+ new String[] {+ "some.password", "javax.net.ssl.trustStorePassword", "basicauth", "some.Secret"+ }) {System.setProperty(propName, PASSWORD);NamedList<Object> properties = readProperties();- assertEquals("Failed to redact " + propName, REDACT_STRING, properties.get(propName));- }- }-- @Test- public void testDisabledRedaction() throws Exception {- RedactionUtils.setRedactSystemProperty(false);- for (String propName : new String[] {"some.password", "javax.net.ssl.trustStorePassword"}) {- System.setProperty(propName, PASSWORD);- NamedList<Object> properties = readProperties();-- assertEquals("Failed to *not* redact " + propName, PASSWORD, properties.get(propName));+ assertEquals(+ "Failed to redact " + propName,+ NodeConfig.REDACTED_SYS_PROP_VALUE,+ properties.get(propName));}}diff --git a/solr/core/src/test/org/apache/solr/metrics/JvmMetricsTest.java b/solr/core/src/test/org/apache/solr/metrics/JvmMetricsTest.javaindex ec1207e63f5..970818a27d4 100644--- a/solr/core/src/test/org/apache/solr/metrics/JvmMetricsTest.java+++ b/solr/core/src/test/org/apache/solr/metrics/JvmMetricsTest.java@@ -115,13 +115,12 @@ public class JvmMetricsTest extends SolrJettyTestBase {String solrXml = Files.readString(home.resolve("solr.xml"), StandardCharsets.UTF_8);NodeConfig config = SolrXmlConfig.fromString(home, solrXml);NodeConfig.NodeConfigBuilder.DEFAULT_HIDDEN_SYS_PROPS.forEach(- s -> assertTrue(s, config.getMetricsConfig().getHiddenSysProps().contains(s)));+ s -> assertTrue(s, config.isSysPropHidden(s)));// custom configsolrXml = Files.readString(home.resolve("solr-hiddensysprops.xml"), StandardCharsets.UTF_8);NodeConfig config2 = SolrXmlConfig.fromString(home, solrXml);- Arrays.asList("foo", "bar", "baz")- .forEach(s -> assertTrue(s, config2.getMetricsConfig().getHiddenSysProps().contains(s)));+ Arrays.asList("foo", "bar", "baz").forEach(s -> assertTrue(s, config2.isSysPropHidden(s)));}@Testdiff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/configuring-solr-xml.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/configuring-solr-xml.adocindex 5006d6eab8e..fe6348c4c0a 100644--- a/solr/solr-ref-guide/modules/configuration-guide/pages/configuring-solr-xml.adoc+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/configuring-solr-xml.adoc@@ -306,6 +306,22 @@ This is the same system property used in the `_default` configset for the xref:c<maxBooleanClauses>${solr.max.booleanClauses:1024}</maxBooleanClauses>----+[#hiddenSysProps]+`hiddenSysProps`::+++[%autowidth,frame=none]+|===+|Optional |Default: _see description_+|===+++Comma-separated list of regex patterns to match sysProps that should be redacted to hide sensitive information.+++The allow-list can also be configured with the `solr.hiddenSysProps` system property,+or via the `SOLR_HIDDEN_SYS_PROPS` environment variable.+++By default, Solr will hide all basicAuth, AWS, ZK or SSL secret sysProps. It will also hide any sysProp that contains+"password" or "secret" in it.+=== The <solrcloud> ElementThis element defines several parameters that relate so SolrCloud.diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adocindex 9b3b8d84df4..33e386b6b46 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc@@ -32,6 +32,13 @@ Starting in 10, the Maven POM for SolrJ does not refer to SolrJ modules like Zoo=== Deprecation removals-The `jaegertracer-configurator` module, which was deprecated in 9.2, is removed. Users should migrate to the `opentelemetry` module.+* The `jaegertracer-configurator` module, which was deprecated in 9.2, is removed. Users should migrate to the `opentelemetry` module.-The `analytics` module, which was deprecated in 9.2, is removed.+* The `analytics` module, which was deprecated in 9.2, is removed.++* The sysProp `-Dsolr.redaction.system.pattern`, which allows users to provide a pattern to match sysProps that should be redacted for sensitive information,+has been removed. Please use `-Dsolr.hiddenSysProps` or the envVar `SOLR_HIDDEN_SYS_PROPS` instead.++* The `<hiddenSysProps>` solr.xml element under `<metrics>` has been removed. Instead use the `<hiddenSysProps>` tag under `<solr>`, which accepts a comma-separated string.+Please see `-Dsolr.redaction.system.pattern`, which allows users to provide a pattern to match sysProps that should be redacted for sensitive information,+has been removed. Please use `-Dsolr.hiddenSysProps` or the envVar `SOLR_HIDDEN_SYS_PROPS` instead.diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adocindex d4e8ef3d5bb..e2a94642d2e 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc@@ -81,6 +81,17 @@ Please refer to the https://solr.apache.org/downloads.html[Solr Downloads] siteOlder configuration now does nothing.Instead, set an env var: SOLR_ENABLE_REMOTE_STREAMING or SOLR_ENABLE_STREAM_BODY or system property equivalents.+* The method for specifying sysProps that contain sensitive information has been streamlined.+Now the sysProp `-Dsolr.hiddenSysProps` or the envVar `SOLR_HIDDEN_SYS_PROPS` are available to provide a comma-separated+list of patterns to match sysProps that should be hidden or redacted.+Please see the xref:configuration-guide:configuring-solr-xml.adoc#hiddenSysProps[hiddenSysProps section] for more information.+++The sysProp `-Dsolr.redaction.system.pattern` has been deprecated, use the above options instead.+++The `<hiddenSysProps>` solr.xml element under `<metrics>` has been deprecated.+Instead, use the xref:configuration-guide:configuring-solr-xml.adoc#hiddenSysProps[<hiddenSysProps>] tag under `<solr>`, which accepts a comma-separated string.++=== Official Docker Image* The customization of the Official Solr Dockerfile has been changed.The customization options `SOLR_DOWNLOAD_URL`, `SOLR_CLOSER_URL`, `SOLR_DIST_URL` and `SOLR_ARCHIVE_URL`, have been removed.