Apache Solr open-source search software
SOLR-15809 Get rid of blacklist/whitelist terminology (#424)
Jan Høydahl46b988012e13
diff --git a/dev-tools/test-patch/lucene-solr-yetus-personality.sh b/dev-tools/test-patch/lucene-solr-yetus-personality.shindex ee457f6a976..b74346f0a51 100644--- a/dev-tools/test-patch/lucene-solr-yetus-personality.sh+++ b/dev-tools/test-patch/lucene-solr-yetus-personality.sh@@ -113,11 +113,11 @@ function personality_modulescase ${moduleType} insubmodules)for module in "${CHANGED_MODULES[@]}"; do- if [[ ! "${module}" =~ ^lucene/(licenses|site) ]]; then # blacklist lucene/ dirs that aren't modules+ if [[ ! "${module}" =~ ^lucene/(licenses|site) ]]; then # disallow lucene/ dirs that aren't modulesif [[ "${module}" =~ ^(lucene/(analysis/[^/]+|[^/]+)) ]]; thenlocal lucene_module=${BASH_REMATCH[0]}personality_enqueue_module "${lucene_module}" "${extra}"- elif [[ "${module}" =~ ^solr/(core|solrj|test-framework|solr-ref-guide|contrib/[^.]+) ]]; then # whitelist solr/ modules+ elif [[ "${module}" =~ ^solr/(core|solrj|test-framework|solr-ref-guide|contrib/[^.]+) ]]; then # allow solr/ moduleslocal solr_module=${BASH_REMATCH[0]}# In solr-ref-guide module, do not execute "compile" or "unit" pluginsif [[ ! "${solr_module}" == solr/solr-ref-guide || ! ${testtype} =~ ^(compile|unit)$ ]]; thendiff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex c5efe7957a0..35c6534f54d 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -409,6 +409,11 @@ Other Changes* SOLR-15808: [Docker] The jattach tool is now installed with apt (janhoy)+* SOLR-15809: Get rid of blacklist/whitelist terminology+ JWTAuthPlugin parameter 'algWhitelist' is now 'algAllowlist'+ Environment variables SOLR_IP_WHITELIST and SOLR_IP_BLACKLIST are no longer supported, but replaced with SOLR_IP_ALLOWLIST and SOLR_IP_DENYLIST.+ (janhoy)+* SOLR-15807: New LogListener class for tests to use to make assertions about what Log messages should or should not be produced by a test (hossman)Bug Fixesdiff --git a/solr/bin/solr b/solr/bin/solrindex b8e68f0285f..20b337b3fa7 100755--- a/solr/bin/solr+++ b/solr/bin/solr@@ -2074,9 +2074,15 @@ elsefifi+# Exit if old syntax found+if [ -n "${SOLR_IP_BLACKLIST}" ] || [ -n "${SOLR_IP_WHITELIST}" ]; then+ echo "ERROR: SOLR_IP_BLACKLIST and SOLR_IP_WHITELIST are no longer supported. Please use SOLR_IP_ALLOWLIST and SOLR_IP_DENYLIST instead."+ exit 1+fi+# IP-based access control-IP_ACL_OPTS=("-Dsolr.jetty.inetaccess.includes=${SOLR_IP_WHITELIST}" \- "-Dsolr.jetty.inetaccess.excludes=${SOLR_IP_BLACKLIST}")+IP_ACL_OPTS=("-Dsolr.jetty.inetaccess.includes=${SOLR_IP_ALLOWLIST}" \+ "-Dsolr.jetty.inetaccess.excludes=${SOLR_IP_DENYLIST}")# These are useful for attaching remote profilers like VisualVM/JConsoleif [ "$ENABLE_REMOTE_JMX_OPTS" == "true" ]; thendiff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmdindex 5de6db54a66..bcd5eec336d 100755--- a/solr/bin/solr.cmd+++ b/solr/bin/solr.cmd@@ -1167,9 +1167,19 @@ IF "%SOLR_MODE%"=="solrcloud" ())+REM Exit if old syntax found+IF DEFINED SOLR_IP_BLACKLIST (+ set SCRIPT_ERROR=SOLR_IP_BLACKLIST and SOLR_IP_WHITELIST are no longer supported. Please use SOLR_IP_ALLOWLIST and SOLR_IP_DENYLIST instead.+ goto err+)+IF DEFINED SOLR_IP_WHITELIST (+ set SCRIPT_ERROR=SOLR_IP_BLACKLIST and SOLR_IP_WHITELIST are no longer supported. Please use SOLR_IP_ALLOWLIST and SOLR_IP_DENYLIST instead.+ goto err+)+REM IP-based access control-set IP_ACL_OPTS=-Dsolr.jetty.inetaccess.includes="%SOLR_IP_WHITELIST%" ^--Dsolr.jetty.inetaccess.excludes="%SOLR_IP_BLACKLIST%"+set IP_ACL_OPTS=-Dsolr.jetty.inetaccess.includes="%SOLR_IP_ALLOWLIST%" ^+-Dsolr.jetty.inetaccess.excludes="%SOLR_IP_DENYLIST%"REM These are useful for attaching remove profilers like VisualVM/JConsoleIF "%ENABLE_REMOTE_JMX_OPTS%"=="true" (diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmdindex b344879232e..4eb073eb9a3 100755--- a/solr/bin/solr.in.cmd+++ b/solr/bin/solr.in.cmd@@ -118,12 +118,12 @@ REM set SOLR_JETTY_HOST=127.0.0.1REM Restrict access to solr by IP address.REM Specify a comma-separated list of addresses or networks, for example:REM 127.0.0.1, 192.168.0.0/24, [::1], [2000:123:4:5::]/64-REM set SOLR_IP_WHITELIST=+REM set SOLR_IP_ALLOWLIST=REM Block access to solr from specific IP addresses.REM Specify a comma-separated list of addresses or networks, for example:REM 127.0.0.1, 192.168.0.0/24, [::1], [2000:123:4:5::]/64-REM set SOLR_IP_BLACKLIST=+REM set SOLR_IP_DENYLIST=REM Enables HTTPS. It is implictly true if you set SOLR_SSL_KEY_STORE. Use this configREM to enable https module with custom jetty configuration.diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.shindex e8ef93cb0a8..c0493cffafb 100644--- a/solr/bin/solr.in.sh+++ b/solr/bin/solr.in.sh@@ -134,12 +134,12 @@# Restrict access to solr by IP address.# Specify a comma-separated list of addresses or networks, for example:# 127.0.0.1, 192.168.0.0/24, [::1], [2000:123:4:5::]/64-#SOLR_IP_WHITELIST=+#SOLR_IP_ALLOWLIST=# Block access to solr from specific IP addresses.# Specify a comma-separated list of addresses or networks, for example:# 127.0.0.1, 192.168.0.0/24, [::1], [2000:123:4:5::]/64-#SOLR_IP_BLACKLIST=+#SOLR_IP_DENYLIST=# Sets the network interface the Solr binds to. To prevent administrators from# accidentally exposing Solr more widely than intended, this defaults to 127.0.0.1.diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.javaindex 54f05b0897c..7361b98cd26 100644--- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java+++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java@@ -210,7 +210,7 @@ public class JoinQParserPlugin extends QParserPlugin {};}- private static final EnumSet<Method> JOIN_METHOD_WHITELIST = EnumSet.of(Method.index, Method.topLevelDV, Method.dvWithScore);+ private static final EnumSet<Method> JOIN_METHOD_ALLOWLIST = EnumSet.of(Method.index, Method.topLevelDV, Method.dvWithScore);/*** A helper method for other plugins to create (non-scoring) JoinQueries wrapped around arbitrary queries against the same core.*@@ -228,7 +228,7 @@ public class JoinQParserPlugin extends QParserPlugin {final Method joinMethod = parseMethodString(method);- if (! JOIN_METHOD_WHITELIST.contains(joinMethod)) {+ if (! JOIN_METHOD_ALLOWLIST.contains(joinMethod)) {// TODO Throw something that the callers here (FacetRequest) can catch and produce a more domain-appropriate error message for?throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"Join method " + method + " not supported for non-scoring, same-core joins");diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.javaindex c10447df0b5..99531b66e44 100644--- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java+++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java@@ -84,7 +84,7 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements SpecProvider,private static final String PARAM_PRINCIPAL_CLAIM = "principalClaim";private static final String PARAM_ROLES_CLAIM = "rolesClaim";private static final String PARAM_REQUIRE_EXPIRATIONTIME = "requireExp";- private static final String PARAM_ALG_WHITELIST = "algWhitelist";+ private static final String PARAM_ALG_ALLOWLIST = "algAllowlist";private static final String PARAM_JWK_CACHE_DURATION = "jwkCacheDur";private static final String PARAM_CLAIMS_MATCH = "claimsMatch";private static final String PARAM_SCOPE = "scope";@@ -101,8 +101,11 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements SpecProvider,private static final long DEFAULT_REFRESH_REPRIEVE_THRESHOLD = 5000;static final String PRIMARY_ISSUER = "PRIMARY";+ @Deprecated(since = "9.0") // Remove in 10.0+ private static final String PARAM_ALG_WHITELIST = "algWhitelist";+private static final Set<String> PROPS = ImmutableSet.of(PARAM_BLOCK_UNKNOWN,- PARAM_PRINCIPAL_CLAIM, PARAM_REQUIRE_EXPIRATIONTIME, PARAM_ALG_WHITELIST,+ PARAM_PRINCIPAL_CLAIM, PARAM_REQUIRE_EXPIRATIONTIME, PARAM_ALG_ALLOWLIST,PARAM_JWK_CACHE_DURATION, PARAM_CLAIMS_MATCH, PARAM_SCOPE, PARAM_REALM, PARAM_ROLES_CLAIM,PARAM_ADMINUI_SCOPE, PARAM_REDIRECT_URIS, PARAM_REQUIRE_ISSUER, PARAM_ISSUERS,PARAM_TRUSTED_CERTS_FILE, PARAM_TRUSTED_CERTS,@@ -113,7 +116,7 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements SpecProvider,private JwtConsumer jwtConsumer;private boolean requireExpirationTime;- private List<String> algWhitelist;+ private List<String> algAllowlist;private String principalClaim;private String rolesClaim;private HashMap<String, Pattern> claimsMatchCompiled;@@ -159,7 +162,12 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements SpecProvider,principalClaim = (String) pluginConfig.getOrDefault(PARAM_PRINCIPAL_CLAIM, "sub");rolesClaim = (String) pluginConfig.get(PARAM_ROLES_CLAIM);- algWhitelist = (List<String>) pluginConfig.get(PARAM_ALG_WHITELIST);+ algAllowlist = (List<String>) pluginConfig.get(PARAM_ALG_ALLOWLIST);+ // TODO: Remove deprecated warning in Solr 10.0+ if ((algAllowlist == null || algAllowlist.isEmpty()) && pluginConfig.containsKey(PARAM_ALG_WHITELIST)) {+ log.warn("Found use of deprecated parameter algWhitelist. Please use {} instead.", PARAM_ALG_ALLOWLIST);+ algAllowlist = (List<String>) pluginConfig.get(PARAM_ALG_WHITELIST);+ }realm = (String) pluginConfig.getOrDefault(PARAM_REALM, DEFAULT_AUTH_REALM);Map<String, String> claimsMatch = (Map<String, String>) pluginConfig.get(PARAM_CLAIMS_MATCH);@@ -545,9 +553,9 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements SpecProvider,}if (requireExpirationTime)jwtConsumerBuilder.setRequireExpirationTime();- if (algWhitelist != null)+ if (algAllowlist != null)jwtConsumerBuilder.setJwsAlgorithmConstraints( // only allow the expected signature algorithm(s) in the given context- new AlgorithmConstraints(AlgorithmConstraints.ConstraintType.WHITELIST, algWhitelist.toArray(new String[0])));+ new AlgorithmConstraints(AlgorithmConstraints.ConstraintType.PERMIT, algAllowlist.toArray(new String[0])));jwtConsumerBuilder.setVerificationKeyResolver(verificationKeyResolver);jwtConsumer = jwtConsumerBuilder.build(); // create the JwtConsumer instance}diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.javaindex a58c9b7fcb8..3735d0a17fe 100644--- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java+++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java@@ -321,8 +321,8 @@ public class JWTAuthPluginTest extends SolrTestCaseJ4 {}@Test- public void algWhitelist() {- testConfig.put("algWhitelist", Arrays.asList("PS384", "PS512"));+ public void algAllowlist() {+ testConfig.put("algAllowlist", Arrays.asList("PS384", "PS512"));plugin.init(testConfig);JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader);assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode());diff --git a/solr/core/src/test/org/apache/solr/security/TestAuthorizationFramework.java b/solr/core/src/test/org/apache/solr/security/TestAuthorizationFramework.javaindex 33e0f4527eb..dd5548c60b6 100644--- a/solr/core/src/test/org/apache/solr/security/TestAuthorizationFramework.java+++ b/solr/core/src/test/org/apache/solr/security/TestAuthorizationFramework.java@@ -71,7 +71,7 @@ public class TestAuthorizationFramework extends AbstractFullDistribZkTestBase {cloudClient.query(params);MockAuthorizationPlugin.protectedResources.add("/select");- // This user is blacklisted in the mock. The request should return a 403.+ // This user is disallowed in the mock. The request should return a 403.params.add("uname", "user1");expectThrows(Exception.class, () -> cloudClient.query(params));log.info("Ending test");diff --git a/solr/solr-ref-guide/src/filters.adoc b/solr/solr-ref-guide/src/filters.adocindex 95c56ae38fe..4e0e158932b 100644--- a/solr/solr-ref-guide/src/filters.adoc+++ b/solr/solr-ref-guide/src/filters.adoc@@ -3428,7 +3428,7 @@ With the example below, for a token "example.com" with type `<URL>`, the token e== Type Token Filter-This filter blacklists or whitelists a specified list of token types, assuming the tokens have type metadata associated with them.+This filter denies or allows a specified list of token types, assuming the tokens have type metadata associated with them.For example, the <<tokenizers.adoc#uax29-url-email-tokenizer,UAX29 URL Email Tokenizer>> emits "<URL>" and "<EMAIL>" typed tokens, as well as other types.This filter would allow you to pull out only e-mail addresses from text as tokens, if you wish.@@ -3453,7 +3453,7 @@ Defines the path to a file of types to filter.|===+If `true`, the file defined in `types` should be used as include list.-If `false`, or undefined, the file defined in `types` is used as a blacklist.+If `false`, or undefined, the file defined in `types` is used as a denylist.*Example:*diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adocindex 4623b3dcb66..bcf3d7213de 100644--- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc+++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc@@ -52,7 +52,7 @@ realm ; Name of the authentication realm to echo back in HTTP 401scope ; Whitespace separated list of valid scopes. If configured, the JWT access token MUST contain a `scope` claim with at least one of the listed scopes. Example: `solr:read solr:admin` ;requireIss ; Fails requests that lacks an `iss` (issuer) claim ; `true`requireExp ; Fails requests that lacks an `exp` (expiry time) claim ; `true`-algWhitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; Default is to allow all algorithms+algAllowlist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; Default is to allow all algorithmsjwkCacheDur ; Duration of JWK cache in seconds ; `3600` (1 hour)principalClaim ; What claim id to pull principal from ; `sub`rolesClaim ; What claim id to pull user roles from. The claim must then either contain a space separated list of roles or a JSON array. The roles can then be used to define fine-grained access in an Authorization plugin ; By default the scopes from `scope` claim are passed on as user roles@@ -132,7 +132,7 @@ Let's look at a more complex configuration, this time with two issuers configure"principalClaim": "solruid", <3>"claimsMatch": { "foo" : "A|B", "dept" : "IT" }, <4>"scope": "solr:read solr:write solr:admin", <5>- "algWhitelist" : [ "RS256", "RS384", "RS512" ], <6>+ "algAllowlist" : [ "RS256", "RS384", "RS512" ], <6>"issuers": [ <7>{"name": "example1-static", <8>diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adocindex cd8c41e69ab..5ec3cd3ebeb 100644--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc@@ -80,6 +80,9 @@ All the usages are replaced by BaseHttpSolrClient.RemoteSolrException and BaseHt* SOLR-15409: Zookeeper client libraries upgraded to 3.7.0, which may not be compatible with your existing server installations+* SOLR-15809: Get rid of blacklist/whitelist terminology. JWTAuthPlugin parameter `algWhitelist` is now `algAllowlist`. The old parameter will still+ work in 9.x. Environment variables `SOLR_IP_WHITELIST` and `SOLR_IP_BLACKLIST` are no longer supported, but replaced with `SOLR_IP_ALLOWLIST` and `SOLR_IP_DENYLIST`.+== New Features & Enhancementsdiff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adocindex 31fd7d5db3a..39162eb8d64 100644--- a/solr/solr-ref-guide/src/securing-solr.adoc+++ b/solr/solr-ref-guide/src/securing-solr.adoc@@ -87,14 +87,14 @@ You can enable request logging by setting `SOLR_REQUESTLOG_ENABLED=true` via env== IP Access Control-Restrict network access to specific hosts, by setting `SOLR_IP_WHITELIST`/`SOLR_IP_BLACKLIST` via environment variables or in `solr.in.sh`/`solr.in.cmd`.+Restrict network access to specific hosts, by setting `SOLR_IP_ALLOWLIST`/`SOLR_IP_DENYLIST` via environment variables or in `solr.in.sh`/`solr.in.cmd`.[source,bash]----# Allow IPv4/IPv6 localhost, the 192.168.0.x IPv4 network, and 2000:123:4:5:: IPv6 network.-SOLR_IP_WHITELIST="127.0.0.1, [::1], 192.168.0.0/24, [2000:123:4:5::]/64"+SOLR_IP_ALLOWLIST="127.0.0.1, [::1], 192.168.0.0/24, [2000:123:4:5::]/64"# Explicitly deny access to two problematic hosts.-SOLR_IP_BLACKLIST="192.168.0.3, 192.168.0.4"+SOLR_IP_DENYLIST="192.168.0.3, 192.168.0.4"----== Securing ZooKeeper Trafficdiff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.javaindex 647d7aef5b6..84e7e789712 100644--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java@@ -521,7 +521,7 @@ public class SolrTestCaseHS extends SolrTestCaseJ4 {// silly stuff included from solrconfig.snippet.randomindexconfig.xmlSystem.setProperty("solr.tests.maxBufferedDocs", String.valueOf(100000));- // If we want to run with whitelist list, this must be explicitly set to true for the test+ // If we want to run with allowlist, this must be explicitly set to true for the test// otherwise we disable the checkif (System.getProperty(AllowListUrlChecker.DISABLE_URL_ALLOW_LIST) == null) {systemSetPropertySolrDisableUrlAllowList("true");@@ -593,20 +593,6 @@ public class SolrTestCaseHS extends SolrTestCaseJ4 {return getShardsParam(slist);}- public String getWhitelistString() {- StringBuilder sb = new StringBuilder();- boolean first = true;- for (SolrInstance instance : slist) {- if (first) {- first = false;- } else {- sb.append(',');- }- sb.append( instance.getBaseURL().replace("/solr", ""));- }- return sb.toString();- }-public List<SolrClient> getSolrJs() {List<SolrClient> solrjs = new ArrayList<>(slist.size());for (SolrInstance instance : slist) {diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.javaindex 2ea25fabeef..9530b95ca68 100644--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java@@ -2892,7 +2892,7 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {if (null != allowedAlg) {// the user has explicitly requested to bypass our assertions and allow a particular alg- // the only thing we should do is assert that the algorithm they have whitelisted is actaully used+ // the only thing we should do is assert that the algorithm they have allowed is actually usedfinal String actualAlg = (new SecureRandom()).getAlgorithm();@@ -2916,7 +2916,7 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {"Please report the details of this failure (and your JVM vendor/version) to [email protected]. " +"You can try to run your tests with -D"+EGD+"="+URANDOM+" or bypass this check using " +"-Dtest.solr.allowed.securerandom="+ algorithm +" as a JVM option when running tests.",- // be permissive in our checks and blacklist only algorithms+ // be permissive in our checks and deny only algorithms// that are known to be blocking under some circumstancesalgorithm.equals("NativePRNG") || algorithm.equals("NativePRNGBlocking"));}