Apache Solr open-source search software
SOLR-15914 Make it super simple to add a (contrib) module to shared classpath (#557)
Jan Høydahl086b4dc56bdb
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 4952c7de4ad..bc4253f6f5b 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -99,6 +99,10 @@ New Features* SOLR-15197: Support temporal graph queries with DAY and WEEKDAY windows (Joel Bernstein)+* SOLR-15914: Official Solr modules (contribs) can now easily be added to shared class path by environment variable+ 'SOLR_MODULES' or system property 'solr.modules'. E.g: SOLR_MODULES=extracting,langid+ (janhoy, David Smiley, Houston Putman)+* SOLR-15880: Introduce support for k nearest neighbors search (Alessandro Benedetti, Elia Porciani)Improvementsdiff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmdindex c554be260a7..72c535be07e 100755--- a/solr/bin/solr.in.cmd+++ b/solr/bin/solr.in.cmd@@ -222,3 +222,6 @@ REM set SOLR_SOLRXML_REQUIRED=falseREM Some previous versions of Solr use an outdated log4j dependency. If you are unable to use at least log4j version 2.15.0REM then enable the following setting to address CVE-2021-44228REM set SOLR_OPTS=%SOLR_OPTS% -Dlog4j2.formatMsgNoLookups=true++REM The bundled plugins in the "modules" folder can easily be enabled as a comma-separated list in SOLR_MODULES variable+REM set SOLR_MODULES=extraction,ltr\ No newline at end of filediff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.shindex aaea9c84515..5ec8e9e5bd3 100644--- a/solr/bin/solr.in.sh+++ b/solr/bin/solr.in.sh@@ -266,3 +266,6 @@# Some previous versions of Solr use an outdated log4j dependency. If you are unable to use at least log4j version 2.15.0# then enable the following setting to address CVE-2021-44228# SOLR_OPTS="$SOLR_OPTS -Dlog4j2.formatMsgNoLookups=true"++# The bundled plugins in the "modules" folder can easily be enabled as a comma-separated list in SOLR_MODULES variable+# SOLR_MODULES=extraction,ltr\ No newline at end of filediff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex 65ad8addf30..747dc2ef5a1 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -116,7 +116,6 @@ import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-import java.io.Closeable;import java.io.IOException;import java.lang.invoke.MethodHandles;import java.nio.file.Files;@@ -127,7 +126,6 @@ import java.text.SimpleDateFormat;import java.util.ArrayList;import java.util.Arrays;import java.util.Date;-import java.util.LinkedHashSet;import java.util.List;import java.util.Locale;import java.util.Map;@@ -669,32 +667,6 @@ public class CoreContainer {log.debug("Loading cores into CoreContainer [instanceDir={}]", getSolrHome());}- // Always add $SOLR_HOME/lib to the shared resource loader- Set<String> libDirs = new LinkedHashSet<>();- libDirs.add("lib");-- if (!StringUtils.isBlank(cfg.getSharedLibDirectory())) {- List<String> sharedLibs = Arrays.asList(cfg.getSharedLibDirectory().split("\\s*,\\s*"));- libDirs.addAll(sharedLibs);- }-- boolean modified = false;- // add the sharedLib to the shared resource loader before initializing cfg based plugins- for (String libDir : libDirs) {- Path libPath = Paths.get(getSolrHome()).resolve(libDir);- if (Files.exists(libPath)) {- try {- loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));- modified = true;- } catch (IOException e) {- throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't load libs: " + e, e);- }- }- }- if (modified) {- loader.reloadLuceneSPI();- }-ClusterEventProducerFactory clusterEventProducerFactory = new ClusterEventProducerFactory(this);clusterEventProducer = clusterEventProducerFactory;@@ -1714,7 +1686,6 @@ public class CoreContainer {// CoreDescriptor and we need to reload it from the disk filesCoreDescriptor cd = reloadCoreDescriptor(core.getCoreDescriptor());solrCores.addCoreDescriptor(cd);- Closeable oldCore = null;boolean success = false;try {solrCores.waitAddPendingCoreOps(cd.getName());diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java b/solr/core/src/java/org/apache/solr/core/NodeConfig.javaindex 03195c73392..9b87d2a803f 100644--- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java+++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java@@ -22,17 +22,23 @@ import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.logging.LogWatcherConfig;-+import org.apache.solr.servlet.SolrDispatchFilter;import org.apache.solr.update.UpdateShardHandlerConfig;+import org.apache.solr.util.ModuleUtils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import java.io.ByteArrayInputStream;+import java.io.IOException;import java.lang.invoke.MethodHandles;+import java.net.URL;+import java.nio.file.Files;import java.nio.file.Path;+import java.nio.file.Paths;import java.util.Arrays;import java.util.Collections;import java.util.HashSet;+import java.util.LinkedHashSet;import java.util.List;import java.util.Properties;import java.util.Set;@@ -59,6 +65,8 @@ public class NodeConfig {private final String sharedLibDirectory;+ private final String modules;+private final PluginInfo shardHandlerFactoryConfig;private final UpdateShardHandlerConfig updateShardHandlerConfig;@@ -116,7 +124,7 @@ public class NodeConfig {Properties solrProperties, PluginInfo[] backupRepositoryPlugins,MetricsConfig metricsConfig, PluginInfo transientCacheConfig, PluginInfo tracerConfig,boolean fromZookeeper, String defaultZkHost, Set<Path> allowPaths, List<String> allowUrls,- String configSetServiceClass) {+ String configSetServiceClass, String modules) {// all Path params here are absolute and normalized.this.nodeName = nodeName;this.coreRootDirectory = coreRootDirectory;@@ -150,6 +158,7 @@ public class NodeConfig {this.allowPaths = allowPaths;this.allowUrls = allowUrls;this.configSetServiceClass = configSetServiceClass;+ this.modules = modules;if (this.cloudConfig != null && this.getCoreLoadThreadCount(false) < 2) {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,@@ -157,6 +166,9 @@ public class NodeConfig {}if (null == this.solrHome) throw new NullPointerException("solrHome");if (null == this.loader) throw new NullPointerException("loader");++ setupSharedLib();+ initModules();}/**@@ -206,7 +218,20 @@ public class NodeConfig {return solrDataHome;}- /**+ /**+ * Obtain the path of solr's binary installation directory, e.g. <code>/opt/solr</code>+ * @return path to install dir+ * @throws SolrException if property 'solr.install.dir' has not been initialized+ */+ public Path getSolrInstallDir() {+ String prop = System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE);+ if (prop == null || prop.isBlank()) {+ throw new SolrException(ErrorCode.SERVER_ERROR, "solr.install.dir property not initialized");+ }+ return Paths.get(prop);+ }++ /*** If null, the lucene default will not be overridden** @see IndexSearcher#setMaxClauseCount@@ -360,6 +385,81 @@ public class NodeConfig {return allowUrls;}+ // Configures SOLR_HOME/lib to the shared class loader+ private void setupSharedLib() {+ // Always add $SOLR_HOME/lib to the shared resource loader+ Set<String> libDirs = new LinkedHashSet<>();+ libDirs.add("lib");++ if (!StringUtils.isBlank(getSharedLibDirectory())) {+ List<String> sharedLibs = Arrays.asList(getSharedLibDirectory().split("\\s*,\\s*"));+ libDirs.addAll(sharedLibs);+ }++ addFoldersToSharedLib(libDirs);+ }++ /**+ * Returns the modules as configured in solr.xml. Comma separated list. May be null if not defined+ */+ public String getModules() {+ return modules;+ }++ // Finds every jar in each folder and adds it to shardLib, then reloads Lucene SPI+ private void addFoldersToSharedLib(Set<String> libDirs) {+ boolean modified = false;+ // add the sharedLib to the shared resource loader before initializing cfg based plugins+ for (String libDir : libDirs) {+ Path libPath = getSolrHome().resolve(libDir);+ if (Files.exists(libPath)) {+ try {+ loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));+ modified = true;+ } catch (IOException e) {+ throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't load libs: " + e, e);+ }+ }+ }+ if (modified) {+ loader.reloadLuceneSPI();+ }+ }++ // Adds modules to shared classpath+ private void initModules() {+ var moduleNames = ModuleUtils.resolveModulesFromStringOrSyspropOrEnv(getModules());+ boolean modified = false;+ for (String m : moduleNames) {+ if (!ModuleUtils.moduleExists(getSolrInstallDir(), m)) {+ log.error("No module with name {}, available modules are {}", m, ModuleUtils.listAvailableModules(getSolrInstallDir()));+ // Fail-fast if user requests a non-existing module+ throw new SolrException(ErrorCode.SERVER_ERROR, "No module with name " + m);+ }+ Path moduleLibPath = ModuleUtils.getModuleLibPath(getSolrInstallDir(), m);+ if (Files.exists(moduleLibPath)) {+ try {+ List<URL> urls = SolrResourceLoader.getURLs(moduleLibPath);+ loader.addToClassLoader(urls);+ if (log.isInfoEnabled()) {+ log.info("Added module {}. libPath={} with {} libs", m, moduleLibPath, urls.size());+ }+ if (log.isDebugEnabled()) {+ log.debug("Libs loaded from {}: {}", moduleLibPath, urls);+ }+ modified = true;+ } catch (IOException e) {+ throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't load libs for module " + m + ": " + e, e);+ }+ } else {+ throw new SolrException(ErrorCode.SERVER_ERROR, "Module lib folder " + moduleLibPath + " not found.");+ }+ }+ if (modified) {+ loader.reloadLuceneSPI();+ }+ }+public static class NodeConfigBuilder {// all Path fields here are absolute and normalized.private SolrResourceLoader loader;@@ -368,6 +468,7 @@ public class NodeConfig {private Integer booleanQueryMaxClauseCount;private Path configSetBaseDirectory;private String sharedLibDirectory;+ private String modules;private PluginInfo shardHandlerFactoryConfig;private UpdateShardHandlerConfig updateShardHandlerConfig = UpdateShardHandlerConfig.DEFAULT;private String configSetServiceClass;@@ -581,6 +682,15 @@ public class NodeConfig {return this;}+ /**+ * Set list of modules to add to class path+ * @param moduleNames comma separated list of module names to add to class loader, e.g. "extracting,ltr,langid"+ */+ public NodeConfigBuilder setModules(String moduleNames) {+ this.modules = moduleNames;+ return this;+ }+public NodeConfig build() {// if some things weren't set then set them now. Simple primitives are set on the field declarationif (loader == null) {@@ -595,7 +705,8 @@ public class NodeConfig {transientCacheSize, useSchemaCache, managementPath,solrHome, loader, solrProperties,backupRepositoryPlugins, metricsConfig, transientCacheConfig, tracerConfig,- fromZookeeper, defaultZkHost, allowPaths, allowUrls, configSetServiceClass);+ fromZookeeper, defaultZkHost, allowPaths, allowUrls, configSetServiceClass,+ modules);}public NodeConfigBuilder setSolrResourceLoader(SolrResourceLoader resourceLoader) {diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.javaindex 48f23be0e76..abc065c2917 100644--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java@@ -335,6 +335,9 @@ public class SolrXmlConfig {case "sharedLib":builder.setSharedLibDirectory(value);break;+ case "modules":+ builder.setModules(value);+ break;case "allowPaths":builder.setAllowPaths(separatePaths(value));break;diff --git a/solr/core/src/java/org/apache/solr/util/ModuleUtils.java b/solr/core/src/java/org/apache/solr/util/ModuleUtils.javanew file mode 100644index 00000000000..434f96bfa58--- /dev/null+++ b/solr/core/src/java/org/apache/solr/util/ModuleUtils.java@@ -0,0 +1,125 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.util;++import org.apache.solr.common.StringUtils;+import org.apache.solr.common.util.StrUtils;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.Collection;+import java.util.Collections;+import java.util.HashSet;+import java.util.Set;+import java.util.regex.Pattern;+import java.util.stream.Collectors;++/**+ * Parses the list of modules the user has requested in solr.xml, property solr.modules or environment SOLR_MODULES.+ * Then resolves the lib folder for each, so they can be added to class path.+ */+public class ModuleUtils {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ private static final String MODULES_FOLDER_NAME = "modules";+ private static final Pattern validModNamesPattern = Pattern.compile("[\\w\\d-_]+");++ /**+ * Returns a path to a module's lib folder+ * @param moduleName name of module+ * @return the path to the module's lib folder+ */+ public static Path getModuleLibPath(Path solrInstallDirPath, String moduleName) {+ return getModulesPath(solrInstallDirPath).resolve(moduleName).resolve("lib");+ }++ /**+ * Finds list of module names requested by system property or environment variable+ * @return set of raw volume names from sysprop and/or env.var+ */+ static Set<String> resolveFromSyspropOrEnv() {+ // Fall back to sysprop and env.var if nothing configured through solr.xml+ Set<String> mods = new HashSet<>();+ String modulesFromProps = System.getProperty("solr.modules");+ if (!StringUtils.isEmpty(modulesFromProps)) {+ mods.addAll(StrUtils.splitSmart(modulesFromProps, ',', true));+ }+ String modulesFromEnv = System.getenv("SOLR_MODULES");+ if (!StringUtils.isEmpty(modulesFromEnv)) {+ mods.addAll(StrUtils.splitSmart(modulesFromEnv, ',', true));+ }+ return mods.stream().map(String::trim).collect(Collectors.toSet());+ }++ /**+ * Returns true if a module name is valid and exists in the system+ */+ public static boolean moduleExists(Path solrInstallDirPath, String moduleName) {+ if (!isValidName(moduleName)) return false;+ Path modPath = getModulesPath(solrInstallDirPath).resolve(moduleName);+ return Files.isDirectory(modPath);+ }++ /**+ * Returns nam of all existing modules+ */+ public static Set<String> listAvailableModules(Path solrInstallDirPath) {+ try (var moduleFilesStream = Files.list(getModulesPath(solrInstallDirPath))) {+ return moduleFilesStream.filter(Files::isDirectory)+ .map(p -> p.getFileName().toString()).collect(Collectors.toSet());+ } catch (IOException e) {+ log.warn("Found no modules in {}", getModulesPath(solrInstallDirPath), e);+ return Collections.emptySet();+ }+ }++ /**+ * Parses comma separated string of module names, in practice found in solr.xml.+ * If input string is empty (nothing configured) or null (e.g. tag not present in solr.xml),+ * we continue to resolve from system property <code>-Dsolr.modules</code> and if still empty,+ * fall back to environment variable <code>SOLR_MODULES</code>.+ * @param modulesFromString raw string of comma-separated module names+ * @return a set of module+ */+ public static Collection<String> resolveModulesFromStringOrSyspropOrEnv(String modulesFromString) {+ Collection<String> moduleNames;+ if (modulesFromString != null && !modulesFromString.isBlank()) {+ moduleNames = StrUtils.splitSmart(modulesFromString, ',', true);+ } else {+ // If nothing configured in solr.xml, check sysprop and environment+ moduleNames = resolveFromSyspropOrEnv();+ }+ return moduleNames.stream().map(String::trim).collect(Collectors.toSet());+ }++ /**+ * Returns true if module name is valid+ */+ public static boolean isValidName(String moduleName) {+ return validModNamesPattern.matcher(moduleName).matches();+ }++ /**+ * Returns path for modules directory, given the solr install dir path+ */+ public static Path getModulesPath(Path solrInstallDirPath) {+ return solrInstallDirPath.resolve(MODULES_FOLDER_NAME);+ }+}diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.javaindex 7c5e47cc793..6a8c0f01647 100644--- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java+++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java@@ -19,6 +19,7 @@ package org.apache.solr.core;import java.io.File;import java.io.FileOutputStream;import java.io.IOException;+import java.nio.file.Files;import java.nio.file.Path;import java.nio.file.Paths;@@ -42,7 +43,10 @@ import org.apache.solr.handler.admin.CollectionsHandler;import org.apache.solr.handler.admin.ConfigSetsHandler;import org.apache.solr.handler.admin.CoreAdminHandler;import org.apache.solr.handler.admin.InfoHandler;+import org.apache.solr.servlet.SolrDispatchFilter;+import org.apache.solr.util.ModuleUtils;import org.junit.AfterClass;+import org.junit.Assert;import org.junit.Assume;import org.junit.BeforeClass;import org.junit.Test;@@ -88,7 +92,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 {public void testSolrHomeAndResourceLoader() throws Exception {// regardless of what sys prop may be set, the CoreContainer's init arg should be the definitive- // solr home -- and nothing i nthe call stack should be "setting" the sys prop to make that work...+ // solr home -- and nothing in the call stack should be "setting" the sys prop to make that work...final Path fakeSolrHome = createTempDir().toAbsolutePath();System.setProperty(SOLR_HOME_PROP, fakeSolrHome.toString());final Path realSolrHome = createTempDir().toAbsolutePath();@@ -420,6 +424,40 @@ public class TestCoreContainer extends SolrTestCaseJ4 {}}+ @Test+ public void testModuleLibs() throws Exception {+ Path tmpRoot = createTempDir("testModLib");++ File lib = Files.createDirectories(ModuleUtils.getModuleLibPath(tmpRoot, "mod1")).toFile();;++ try (JarOutputStream jar1 = new JarOutputStream(new FileOutputStream(new File(lib, "jar1.jar")))) {+ jar1.putNextEntry(new JarEntry("moduleLibFile"));+ jar1.closeEntry();+ }++ System.setProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE, tmpRoot.toAbsolutePath().toString());+ final CoreContainer cc1 = init(tmpRoot, "<solr></solr>");+ try {+ Assert.assertThrows(SolrResourceNotFoundException.class, () -> cc1.loader.openResource("moduleLibFile").close());+ } finally {+ cc1.shutdown();+ }++ // Explicitly declaring 'lib' makes no change compared to the default+ final CoreContainer cc2 = init(tmpRoot, "<solr><str name=\"modules\">mod1</str></solr>");+ try {+ cc2.loader.openResource("moduleLibFile").close();+ } finally {+ cc2.shutdown();+ }++ SolrException ex = Assert.assertThrows(SolrException.class, () ->+ init(tmpRoot, "<solr><str name=\"modules\">nope</str></solr>"));+ assertEquals("No module with name nope", ex.getMessage());++ System.clearProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE);+ }+private static final String CONFIGSETS_SOLR_XML ="<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" +"<solr>\n" +"<str name=\"configSetBaseDir\">${configsets:configsets}</str>\n" +diff --git a/solr/core/src/test/org/apache/solr/util/ModuleUtilsTest.java b/solr/core/src/test/org/apache/solr/util/ModuleUtilsTest.javanew file mode 100644index 00000000000..d2c2db2f436--- /dev/null+++ b/solr/core/src/test/org/apache/solr/util/ModuleUtilsTest.java@@ -0,0 +1,85 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.util;++import junit.framework.TestCase;++import java.io.IOException;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.Collections;+import java.util.Set;++public class ModuleUtilsTest extends TestCase {+ private Path mockRootDir;+ private final Set<String> expectedMods = Set.of("mod1", "mod2");++ @Override+ public void setUp() throws Exception {+ mockRootDir = setupMockInstallDir(expectedMods);+ }++ public void testModuleExists() {+ assertTrue(ModuleUtils.moduleExists(mockRootDir, "mod1"));+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "mod3"));+ }++ public void testIsValidName() {+ assertTrue(ModuleUtils.isValidName("mod1-foo_bar-123"));+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "not valid"));+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "not/valid"));+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "not>valid"));+ }++ public void testGetModuleLibPath() {+ assertEquals(mockRootDir.resolve("modules")+ .resolve("mod1").resolve("lib"), ModuleUtils.getModuleLibPath(mockRootDir, "mod1"));+ }++ public void testResolveFromSyspropOrEnv() {+ assertEquals(Collections.emptySet(), ModuleUtils.resolveFromSyspropOrEnv());+ System.setProperty("solr.modules", "foo ,bar, baz,mod1");+ assertEquals(Set.of("foo", "bar", "baz", "mod1"), ModuleUtils.resolveFromSyspropOrEnv());+ System.clearProperty("solr.modules");+ }++ public void testListAvailableModules() {+ assertEquals(expectedMods, ModuleUtils.listAvailableModules(mockRootDir));+ }++ public void testResolveModules() {+ assertEquals(Set.of("foo", "bar", "baz", "mod1"), ModuleUtils.resolveModulesFromStringOrSyspropOrEnv("foo ,bar, baz,mod1"));+ assertEquals(Collections.emptySet(), ModuleUtils.resolveModulesFromStringOrSyspropOrEnv(""));+ System.setProperty("solr.modules", "foo ,bar, baz,mod1");+ assertEquals(Set.of("foo", "bar", "baz", "mod1"), ModuleUtils.resolveModulesFromStringOrSyspropOrEnv(null));+ System.clearProperty("solr.modules");+ }++ private Path setupMockInstallDir(Set<String> modules) throws IOException {+ Path root = Files.createTempDirectory("moduleUtilsTest");+ Path modPath = root.resolve("modules");+ Files.createDirectories(modPath);+ for (var m : modules) {+ Path libPath = modPath.resolve(m).resolve("lib");+ Files.createDirectories(libPath);+ Files.createFile(libPath.resolve("jar1.jar"));+ Files.createFile(libPath.resolve("jar2.jar"));+ Files.createFile(libPath.resolve("README.md"));+ }+ return root;+ }+}\ No newline at end of filediff --git a/solr/server/solr/solr.xml b/solr/server/solr/solr.xmlindex 8e919c616f3..60031538f7f 100644--- a/solr/server/solr/solr.xml+++ b/solr/server/solr/solr.xml@@ -30,6 +30,7 @@<int name="maxBooleanClauses">${solr.max.booleanClauses:1024}</int><str name="sharedLib">${solr.sharedLib:}</str>+ <str name="modules">${solr.modules:}</str><str name="allowPaths">${solr.allowPaths:}</str><str name="allowUrls">${solr.allowUrls:}</str>diff --git a/solr/solr-ref-guide/src/configuring-solr-xml.adoc b/solr/solr-ref-guide/src/configuring-solr-xml.adocindex 84246e2492f..64b8d2a5781 100644--- a/solr/solr-ref-guide/src/configuring-solr-xml.adoc+++ b/solr/solr-ref-guide/src/configuring-solr-xml.adoc@@ -33,6 +33,7 @@ The default `solr.xml` file is found in `$SOLR_TIP/server/solr/solr.xml` and loo<int name="maxBooleanClauses">${solr.max.booleanClauses:1024}</int><str name="sharedLib">${solr.sharedLib:}</str>+ <str name="modules">${solr.modules:}</str><str name="allowPaths">${solr.allowPaths:}</str><str name="allowUrls">${solr.allowUrls:}</str>@@ -175,6 +176,19 @@ If the specified path is not absolute, it will be relative to `$SOLR_HOME`.Custom handlers may be placed in this directory.Note that specifying `sharedLib` will not remove `$SOLR_HOME/lib` from Solr's class path.+`modules`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Takes a list of bundled <<solr-modules.adoc#,solr modules>> to add to Solr's class path+on startup. This way of adding modules will add them to the shared class loader, making them+available for every collection in Solr, unlike `<lib>` tag in `solrconfig.xml` which is only+for that one collection. Example value: `extracting,ltr`. See chapter+<<solr-modules.adoc#,Solr Modules>> for more details.+`allowPaths`::+[%autowidth,frame=none]diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adocindex 1ca6633aed7..17cd3a1de96 100644--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc@@ -245,6 +245,8 @@ Instead, these libraries will be included with all other module dependencies in* SOLR-15954: The prometheus-exporter is no longer packaged as a Solr module. It can be found under `solr/prometheus-exporter/`.+* SOLR-15914: Solr modules (formerly known as contribs) can now easily be enabled by an environment variable (e.g. in `solr.in.sh` or `solr.in.cmd`) or as a system property (e.g. in `SOLR_OPTS`). Example: `SOLR_MODULES=extraction,ltr`.+== Deprecations & Removed FeaturesThe following list of features have been permanently removed from Solr:diff --git a/solr/solr-ref-guide/src/solr-modules.adoc b/solr/solr-ref-guide/src/solr-modules.adocnew file mode 100644index 00000000000..b9683b542cc--- /dev/null+++ b/solr/solr-ref-guide/src/solr-modules.adoc@@ -0,0 +1,41 @@+= Solr Modules++// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++Solr modules are addon Solr plugins that are not part of solr-core, but officially maintained+by the Solr project. They provide well-defined features such as the "extracting" module which lets+users index rich text documents with Apache Tika. A single module can contain multiple Plugins.+Modules were earlier known as "contribs".++Each module produces a separate `.jar` file in the build, and additional dependencies required by+each module are also packaged with the module. This helps keep the main core of Solr small and lean.++== Installing a module++The easiest way to enable a module is to list the modules you intend to use either in the+system property `solr.modules` or in the environment variable `SOLR_MODULES` (e.g. in`solr.in.sh`+or `solr.in.cmd`). You can also add a `<str name="modules">` tag to+<<configuring-solr-xml.adoc#,solr.xml>>. The expected value is a comma separated list+of module names, e.g. `SOLR_MODULES=extracting,ltr`. This way of adding modules will add+them to the shared class loader, making them available for every collection in Solr.++If you only wish to enable a module for a single collection, you may add `<lib>` tags to `solrconfig.xml`+as explained in <<libs.adoc#,Lib Directories>>.++Some modules may have been made available as packages for the <<package-manager.adoc#,Package Manager>>,+check by listing available pacakges.\ No newline at end of filediff --git a/solr/solr-ref-guide/src/solr-plugins.adoc b/solr/solr-ref-guide/src/solr-plugins.adocindex 210106e70ed..dd2ba2c41ae 100644--- a/solr/solr-ref-guide/src/solr-plugins.adoc+++ b/solr/solr-ref-guide/src/solr-plugins.adoc@@ -2,7 +2,8 @@:page-children: libs, \package-manager, \cluster-plugins, \- replica-placement-plugins+ replica-placement-plugins, \+ solr-modules// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -45,8 +46,8 @@ Examples of these are <<authentication-and-authorization-plugins.adoc#,authentic== Installing Plugins ==-Most plugins are built-in to Solr and there is nothing to install.-The subject here is how to make other plugins available to Solr, including those in modules.+Most plugins are built-in to Solr core and there is nothing to install.+The subject here is how to make the bundled modules and other plugins available to Solr.Plugins are packaged into a Java jar file and may have other dependent jar files to function.The next sections describe some options:@@ -57,6 +58,7 @@ The next sections describe some options:[cols="1,1",frame=none,grid=none,stripes=none]|===| <<libs.adoc#,Lib Directories>>: Plugins as libraries on the filesystem.+| <<solr-modules.adoc#,Solr Modules>>: Loading bundled modules.| <<package-manager.adoc#,Package Management>>: Package-based plugins.| <<cluster-plugins.adoc#,Cluster Plugins>>: Cluster-level plugins.| <<replica-placement-plugins.adoc#,Replica Placement Plugins>>: Plugins specifically for replica placement.