Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15914 Make it super simple to add a (contrib) module to shared classpath (#557)

Jan Høydahl086b4dc56bdb
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4952c7de4ad..bc4253f6f5b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -99,6 +99,10 @@ New Features
* SOLR-15197: Support temporal graph queries with DAY and WEEKDAY windows (Joel Bernstein)
+* SOLR-15914: Official Solr modules (contribs) can now easily be added to shared class path by environment variable
+ 'SOLR_MODULES' or system property 'solr.modules'. E.g: SOLR_MODULES=extracting,langid
+ (janhoy, David Smiley, Houston Putman)
+
* SOLR-15880: Introduce support for k nearest neighbors search (Alessandro Benedetti, Elia Porciani)
Improvements
diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd
index c554be260a7..72c535be07e 100755
--- a/solr/bin/solr.in.cmd
+++ b/solr/bin/solr.in.cmd
@@ -222,3 +222,6 @@ REM set SOLR_SOLRXML_REQUIRED=false
REM Some previous versions of Solr use an outdated log4j dependency. If you are unable to use at least log4j version 2.15.0
REM then enable the following setting to address CVE-2021-44228
REM set SOLR_OPTS=%SOLR_OPTS% -Dlog4j2.formatMsgNoLookups=true
+
+REM The bundled plugins in the "modules" folder can easily be enabled as a comma-separated list in SOLR_MODULES variable
+REM set SOLR_MODULES=extraction,ltr
\ No newline at end of file
diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh
index aaea9c84515..5ec8e9e5bd3 100644
--- a/solr/bin/solr.in.sh
+++ b/solr/bin/solr.in.sh
@@ -266,3 +266,6 @@
# Some previous versions of Solr use an outdated log4j dependency. If you are unable to use at least log4j version 2.15.0
# then enable the following setting to address CVE-2021-44228
# SOLR_OPTS="$SOLR_OPTS -Dlog4j2.formatMsgNoLookups=true"
+
+# The bundled plugins in the "modules" folder can easily be enabled as a comma-separated list in SOLR_MODULES variable
+# SOLR_MODULES=extraction,ltr
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 65ad8addf30..747dc2ef5a1 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -116,7 +116,6 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Files;
@@ -127,7 +126,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -669,32 +667,6 @@ public class CoreContainer {
log.debug("Loading cores into CoreContainer [instanceDir={}]", getSolrHome());
}
- // Always add $SOLR_HOME/lib to the shared resource loader
- Set<String> libDirs = new LinkedHashSet<>();
- libDirs.add("lib");
-
- if (!StringUtils.isBlank(cfg.getSharedLibDirectory())) {
- List<String> sharedLibs = Arrays.asList(cfg.getSharedLibDirectory().split("\\s*,\\s*"));
- libDirs.addAll(sharedLibs);
- }
-
- boolean modified = false;
- // add the sharedLib to the shared resource loader before initializing cfg based plugins
- for (String libDir : libDirs) {
- Path libPath = Paths.get(getSolrHome()).resolve(libDir);
- if (Files.exists(libPath)) {
- try {
- loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));
- modified = true;
- } catch (IOException e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't load libs: " + e, e);
- }
- }
- }
- if (modified) {
- loader.reloadLuceneSPI();
- }
-
ClusterEventProducerFactory clusterEventProducerFactory = new ClusterEventProducerFactory(this);
clusterEventProducer = clusterEventProducerFactory;
@@ -1714,7 +1686,6 @@ public class CoreContainer {
// CoreDescriptor and we need to reload it from the disk files
CoreDescriptor cd = reloadCoreDescriptor(core.getCoreDescriptor());
solrCores.addCoreDescriptor(cd);
- Closeable oldCore = null;
boolean success = false;
try {
solrCores.waitAddPendingCoreOps(cd.getName());
diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java b/solr/core/src/java/org/apache/solr/core/NodeConfig.java
index 03195c73392..9b87d2a803f 100644
--- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java
@@ -22,17 +22,23 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.logging.LogWatcherConfig;
-
+import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.solr.update.UpdateShardHandlerConfig;
+import org.apache.solr.util.ModuleUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.net.URL;
+import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
@@ -59,6 +65,8 @@ public class NodeConfig {
private final String sharedLibDirectory;
+ private final String modules;
+
private final PluginInfo shardHandlerFactoryConfig;
private final UpdateShardHandlerConfig updateShardHandlerConfig;
@@ -116,7 +124,7 @@ public class NodeConfig {
Properties solrProperties, PluginInfo[] backupRepositoryPlugins,
MetricsConfig metricsConfig, PluginInfo transientCacheConfig, PluginInfo tracerConfig,
boolean fromZookeeper, String defaultZkHost, Set<Path> allowPaths, List<String> allowUrls,
- String configSetServiceClass) {
+ String configSetServiceClass, String modules) {
// all Path params here are absolute and normalized.
this.nodeName = nodeName;
this.coreRootDirectory = coreRootDirectory;
@@ -150,6 +158,7 @@ public class NodeConfig {
this.allowPaths = allowPaths;
this.allowUrls = allowUrls;
this.configSetServiceClass = configSetServiceClass;
+ this.modules = modules;
if (this.cloudConfig != null && this.getCoreLoadThreadCount(false) < 2) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@@ -157,6 +166,9 @@ public class NodeConfig {
}
if (null == this.solrHome) throw new NullPointerException("solrHome");
if (null == this.loader) throw new NullPointerException("loader");
+
+ setupSharedLib();
+ initModules();
}
/**
@@ -206,7 +218,20 @@ public class NodeConfig {
return solrDataHome;
}
- /**
+ /**
+ * Obtain the path of solr's binary installation directory, e.g. <code>/opt/solr</code>
+ * @return path to install dir
+ * @throws SolrException if property 'solr.install.dir' has not been initialized
+ */
+ public Path getSolrInstallDir() {
+ String prop = System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE);
+ if (prop == null || prop.isBlank()) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "solr.install.dir property not initialized");
+ }
+ return Paths.get(prop);
+ }
+
+ /**
* If null, the lucene default will not be overridden
*
* @see IndexSearcher#setMaxClauseCount
@@ -360,6 +385,81 @@ public class NodeConfig {
return allowUrls;
}
+ // Configures SOLR_HOME/lib to the shared class loader
+ private void setupSharedLib() {
+ // Always add $SOLR_HOME/lib to the shared resource loader
+ Set<String> libDirs = new LinkedHashSet<>();
+ libDirs.add("lib");
+
+ if (!StringUtils.isBlank(getSharedLibDirectory())) {
+ List<String> sharedLibs = Arrays.asList(getSharedLibDirectory().split("\\s*,\\s*"));
+ libDirs.addAll(sharedLibs);
+ }
+
+ addFoldersToSharedLib(libDirs);
+ }
+
+ /**
+ * Returns the modules as configured in solr.xml. Comma separated list. May be null if not defined
+ */
+ public String getModules() {
+ return modules;
+ }
+
+ // Finds every jar in each folder and adds it to shardLib, then reloads Lucene SPI
+ private void addFoldersToSharedLib(Set<String> libDirs) {
+ boolean modified = false;
+ // add the sharedLib to the shared resource loader before initializing cfg based plugins
+ for (String libDir : libDirs) {
+ Path libPath = getSolrHome().resolve(libDir);
+ if (Files.exists(libPath)) {
+ try {
+ loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));
+ modified = true;
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't load libs: " + e, e);
+ }
+ }
+ }
+ if (modified) {
+ loader.reloadLuceneSPI();
+ }
+ }
+
+ // Adds modules to shared classpath
+ private void initModules() {
+ var moduleNames = ModuleUtils.resolveModulesFromStringOrSyspropOrEnv(getModules());
+ boolean modified = false;
+ for (String m : moduleNames) {
+ if (!ModuleUtils.moduleExists(getSolrInstallDir(), m)) {
+ log.error("No module with name {}, available modules are {}", m, ModuleUtils.listAvailableModules(getSolrInstallDir()));
+ // Fail-fast if user requests a non-existing module
+ throw new SolrException(ErrorCode.SERVER_ERROR, "No module with name " + m);
+ }
+ Path moduleLibPath = ModuleUtils.getModuleLibPath(getSolrInstallDir(), m);
+ if (Files.exists(moduleLibPath)) {
+ try {
+ List<URL> urls = SolrResourceLoader.getURLs(moduleLibPath);
+ loader.addToClassLoader(urls);
+ if (log.isInfoEnabled()) {
+ log.info("Added module {}. libPath={} with {} libs", m, moduleLibPath, urls.size());
+ }
+ if (log.isDebugEnabled()) {
+ log.debug("Libs loaded from {}: {}", moduleLibPath, urls);
+ }
+ modified = true;
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't load libs for module " + m + ": " + e, e);
+ }
+ } else {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Module lib folder " + moduleLibPath + " not found.");
+ }
+ }
+ if (modified) {
+ loader.reloadLuceneSPI();
+ }
+ }
+
public static class NodeConfigBuilder {
// all Path fields here are absolute and normalized.
private SolrResourceLoader loader;
@@ -368,6 +468,7 @@ public class NodeConfig {
private Integer booleanQueryMaxClauseCount;
private Path configSetBaseDirectory;
private String sharedLibDirectory;
+ private String modules;
private PluginInfo shardHandlerFactoryConfig;
private UpdateShardHandlerConfig updateShardHandlerConfig = UpdateShardHandlerConfig.DEFAULT;
private String configSetServiceClass;
@@ -581,6 +682,15 @@ public class NodeConfig {
return this;
}
+ /**
+ * Set list of modules to add to class path
+ * @param moduleNames comma separated list of module names to add to class loader, e.g. "extracting,ltr,langid"
+ */
+ public NodeConfigBuilder setModules(String moduleNames) {
+ this.modules = moduleNames;
+ return this;
+ }
+
public NodeConfig build() {
// if some things weren't set then set them now. Simple primitives are set on the field declaration
if (loader == null) {
@@ -595,7 +705,8 @@ public class NodeConfig {
transientCacheSize, useSchemaCache, managementPath,
solrHome, loader, solrProperties,
backupRepositoryPlugins, metricsConfig, transientCacheConfig, tracerConfig,
- fromZookeeper, defaultZkHost, allowPaths, allowUrls, configSetServiceClass);
+ fromZookeeper, defaultZkHost, allowPaths, allowUrls, configSetServiceClass,
+ modules);
}
public NodeConfigBuilder setSolrResourceLoader(SolrResourceLoader resourceLoader) {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
index 48f23be0e76..abc065c2917 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
@@ -335,6 +335,9 @@ public class SolrXmlConfig {
case "sharedLib":
builder.setSharedLibDirectory(value);
break;
+ case "modules":
+ builder.setModules(value);
+ break;
case "allowPaths":
builder.setAllowPaths(separatePaths(value));
break;
diff --git a/solr/core/src/java/org/apache/solr/util/ModuleUtils.java b/solr/core/src/java/org/apache/solr/util/ModuleUtils.java
new file mode 100644
index 00000000000..434f96bfa58
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/ModuleUtils.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import org.apache.solr.common.StringUtils;
+import org.apache.solr.common.util.StrUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * Parses the list of modules the user has requested in solr.xml, property solr.modules or environment SOLR_MODULES.
+ * Then resolves the lib folder for each, so they can be added to class path.
+ */
+public class ModuleUtils {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final String MODULES_FOLDER_NAME = "modules";
+ private static final Pattern validModNamesPattern = Pattern.compile("[\\w\\d-_]+");
+
+ /**
+ * Returns a path to a module's lib folder
+ * @param moduleName name of module
+ * @return the path to the module's lib folder
+ */
+ public static Path getModuleLibPath(Path solrInstallDirPath, String moduleName) {
+ return getModulesPath(solrInstallDirPath).resolve(moduleName).resolve("lib");
+ }
+
+ /**
+ * Finds list of module names requested by system property or environment variable
+ * @return set of raw volume names from sysprop and/or env.var
+ */
+ static Set<String> resolveFromSyspropOrEnv() {
+ // Fall back to sysprop and env.var if nothing configured through solr.xml
+ Set<String> mods = new HashSet<>();
+ String modulesFromProps = System.getProperty("solr.modules");
+ if (!StringUtils.isEmpty(modulesFromProps)) {
+ mods.addAll(StrUtils.splitSmart(modulesFromProps, ',', true));
+ }
+ String modulesFromEnv = System.getenv("SOLR_MODULES");
+ if (!StringUtils.isEmpty(modulesFromEnv)) {
+ mods.addAll(StrUtils.splitSmart(modulesFromEnv, ',', true));
+ }
+ return mods.stream().map(String::trim).collect(Collectors.toSet());
+ }
+
+ /**
+ * Returns true if a module name is valid and exists in the system
+ */
+ public static boolean moduleExists(Path solrInstallDirPath, String moduleName) {
+ if (!isValidName(moduleName)) return false;
+ Path modPath = getModulesPath(solrInstallDirPath).resolve(moduleName);
+ return Files.isDirectory(modPath);
+ }
+
+ /**
+ * Returns nam of all existing modules
+ */
+ public static Set<String> listAvailableModules(Path solrInstallDirPath) {
+ try (var moduleFilesStream = Files.list(getModulesPath(solrInstallDirPath))) {
+ return moduleFilesStream.filter(Files::isDirectory)
+ .map(p -> p.getFileName().toString()).collect(Collectors.toSet());
+ } catch (IOException e) {
+ log.warn("Found no modules in {}", getModulesPath(solrInstallDirPath), e);
+ return Collections.emptySet();
+ }
+ }
+
+ /**
+ * Parses comma separated string of module names, in practice found in solr.xml.
+ * If input string is empty (nothing configured) or null (e.g. tag not present in solr.xml),
+ * we continue to resolve from system property <code>-Dsolr.modules</code> and if still empty,
+ * fall back to environment variable <code>SOLR_MODULES</code>.
+ * @param modulesFromString raw string of comma-separated module names
+ * @return a set of module
+ */
+ public static Collection<String> resolveModulesFromStringOrSyspropOrEnv(String modulesFromString) {
+ Collection<String> moduleNames;
+ if (modulesFromString != null && !modulesFromString.isBlank()) {
+ moduleNames = StrUtils.splitSmart(modulesFromString, ',', true);
+ } else {
+ // If nothing configured in solr.xml, check sysprop and environment
+ moduleNames = resolveFromSyspropOrEnv();
+ }
+ return moduleNames.stream().map(String::trim).collect(Collectors.toSet());
+ }
+
+ /**
+ * Returns true if module name is valid
+ */
+ public static boolean isValidName(String moduleName) {
+ return validModNamesPattern.matcher(moduleName).matches();
+ }
+
+ /**
+ * Returns path for modules directory, given the solr install dir path
+ */
+ public static Path getModulesPath(Path solrInstallDirPath) {
+ return solrInstallDirPath.resolve(MODULES_FOLDER_NAME);
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
index 7c5e47cc793..6a8c0f01647 100644
--- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
+++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
@@ -19,6 +19,7 @@ package org.apache.solr.core;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -42,7 +43,10 @@ import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.admin.ConfigSetsHandler;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.admin.InfoHandler;
+import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.util.ModuleUtils;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -88,7 +92,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 {
public void testSolrHomeAndResourceLoader() throws Exception {
// regardless of what sys prop may be set, the CoreContainer's init arg should be the definitive
- // solr home -- and nothing i nthe call stack should be "setting" the sys prop to make that work...
+ // solr home -- and nothing in the call stack should be "setting" the sys prop to make that work...
final Path fakeSolrHome = createTempDir().toAbsolutePath();
System.setProperty(SOLR_HOME_PROP, fakeSolrHome.toString());
final Path realSolrHome = createTempDir().toAbsolutePath();
@@ -420,6 +424,40 @@ public class TestCoreContainer extends SolrTestCaseJ4 {
}
}
+ @Test
+ public void testModuleLibs() throws Exception {
+ Path tmpRoot = createTempDir("testModLib");
+
+ File lib = Files.createDirectories(ModuleUtils.getModuleLibPath(tmpRoot, "mod1")).toFile();;
+
+ try (JarOutputStream jar1 = new JarOutputStream(new FileOutputStream(new File(lib, "jar1.jar")))) {
+ jar1.putNextEntry(new JarEntry("moduleLibFile"));
+ jar1.closeEntry();
+ }
+
+ System.setProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE, tmpRoot.toAbsolutePath().toString());
+ final CoreContainer cc1 = init(tmpRoot, "<solr></solr>");
+ try {
+ Assert.assertThrows(SolrResourceNotFoundException.class, () -> cc1.loader.openResource("moduleLibFile").close());
+ } finally {
+ cc1.shutdown();
+ }
+
+ // Explicitly declaring 'lib' makes no change compared to the default
+ final CoreContainer cc2 = init(tmpRoot, "<solr><str name=\"modules\">mod1</str></solr>");
+ try {
+ cc2.loader.openResource("moduleLibFile").close();
+ } finally {
+ cc2.shutdown();
+ }
+
+ SolrException ex = Assert.assertThrows(SolrException.class, () ->
+ init(tmpRoot, "<solr><str name=\"modules\">nope</str></solr>"));
+ assertEquals("No module with name nope", ex.getMessage());
+
+ System.clearProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE);
+ }
+
private static final String CONFIGSETS_SOLR_XML ="<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" +
"<solr>\n" +
"<str name=\"configSetBaseDir\">${configsets:configsets}</str>\n" +
diff --git a/solr/core/src/test/org/apache/solr/util/ModuleUtilsTest.java b/solr/core/src/test/org/apache/solr/util/ModuleUtilsTest.java
new file mode 100644
index 00000000000..d2c2db2f436
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/ModuleUtilsTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Set;
+
+public class ModuleUtilsTest extends TestCase {
+ private Path mockRootDir;
+ private final Set<String> expectedMods = Set.of("mod1", "mod2");
+
+ @Override
+ public void setUp() throws Exception {
+ mockRootDir = setupMockInstallDir(expectedMods);
+ }
+
+ public void testModuleExists() {
+ assertTrue(ModuleUtils.moduleExists(mockRootDir, "mod1"));
+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "mod3"));
+ }
+
+ public void testIsValidName() {
+ assertTrue(ModuleUtils.isValidName("mod1-foo_bar-123"));
+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "not valid"));
+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "not/valid"));
+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "not>valid"));
+ }
+
+ public void testGetModuleLibPath() {
+ assertEquals(mockRootDir.resolve("modules")
+ .resolve("mod1").resolve("lib"), ModuleUtils.getModuleLibPath(mockRootDir, "mod1"));
+ }
+
+ public void testResolveFromSyspropOrEnv() {
+ assertEquals(Collections.emptySet(), ModuleUtils.resolveFromSyspropOrEnv());
+ System.setProperty("solr.modules", "foo ,bar, baz,mod1");
+ assertEquals(Set.of("foo", "bar", "baz", "mod1"), ModuleUtils.resolveFromSyspropOrEnv());
+ System.clearProperty("solr.modules");
+ }
+
+ public void testListAvailableModules() {
+ assertEquals(expectedMods, ModuleUtils.listAvailableModules(mockRootDir));
+ }
+
+ public void testResolveModules() {
+ assertEquals(Set.of("foo", "bar", "baz", "mod1"), ModuleUtils.resolveModulesFromStringOrSyspropOrEnv("foo ,bar, baz,mod1"));
+ assertEquals(Collections.emptySet(), ModuleUtils.resolveModulesFromStringOrSyspropOrEnv(""));
+ System.setProperty("solr.modules", "foo ,bar, baz,mod1");
+ assertEquals(Set.of("foo", "bar", "baz", "mod1"), ModuleUtils.resolveModulesFromStringOrSyspropOrEnv(null));
+ System.clearProperty("solr.modules");
+ }
+
+ private Path setupMockInstallDir(Set<String> modules) throws IOException {
+ Path root = Files.createTempDirectory("moduleUtilsTest");
+ Path modPath = root.resolve("modules");
+ Files.createDirectories(modPath);
+ for (var m : modules) {
+ Path libPath = modPath.resolve(m).resolve("lib");
+ Files.createDirectories(libPath);
+ Files.createFile(libPath.resolve("jar1.jar"));
+ Files.createFile(libPath.resolve("jar2.jar"));
+ Files.createFile(libPath.resolve("README.md"));
+ }
+ return root;
+ }
+}
\ No newline at end of file
diff --git a/solr/server/solr/solr.xml b/solr/server/solr/solr.xml
index 8e919c616f3..60031538f7f 100644
--- a/solr/server/solr/solr.xml
+++ b/solr/server/solr/solr.xml
@@ -30,6 +30,7 @@
<int name="maxBooleanClauses">${solr.max.booleanClauses:1024}</int>
<str name="sharedLib">${solr.sharedLib:}</str>
+ <str name="modules">${solr.modules:}</str>
<str name="allowPaths">${solr.allowPaths:}</str>
<str name="allowUrls">${solr.allowUrls:}</str>
diff --git a/solr/solr-ref-guide/src/configuring-solr-xml.adoc b/solr/solr-ref-guide/src/configuring-solr-xml.adoc
index 84246e2492f..64b8d2a5781 100644
--- a/solr/solr-ref-guide/src/configuring-solr-xml.adoc
+++ b/solr/solr-ref-guide/src/configuring-solr-xml.adoc
@@ -33,6 +33,7 @@ The default `solr.xml` file is found in `$SOLR_TIP/server/solr/solr.xml` and loo
<int name="maxBooleanClauses">${solr.max.booleanClauses:1024}</int>
<str name="sharedLib">${solr.sharedLib:}</str>
+ <str name="modules">${solr.modules:}</str>
<str name="allowPaths">${solr.allowPaths:}</str>
<str name="allowUrls">${solr.allowUrls:}</str>
@@ -175,6 +176,19 @@ If the specified path is not absolute, it will be relative to `$SOLR_HOME`.
Custom handlers may be placed in this directory.
Note that specifying `sharedLib` will not remove `$SOLR_HOME/lib` from Solr's class path.
+`modules`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: none
+|===
++
+Takes a list of bundled <<solr-modules.adoc#,solr modules>> to add to Solr's class path
+on startup. This way of adding modules will add them to the shared class loader, making them
+available for every collection in Solr, unlike `<lib>` tag in `solrconfig.xml` which is only
+for that one collection. Example value: `extracting,ltr`. See chapter
+<<solr-modules.adoc#,Solr Modules>> for more details.
+
`allowPaths`::
+
[%autowidth,frame=none]
diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
index 1ca6633aed7..17cd3a1de96 100644
--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
@@ -245,6 +245,8 @@ Instead, these libraries will be included with all other module dependencies in
* SOLR-15954: The prometheus-exporter is no longer packaged as a Solr module. It can be found under `solr/prometheus-exporter/`.
+* SOLR-15914: Solr modules (formerly known as contribs) can now easily be enabled by an environment variable (e.g. in `solr.in.sh` or `solr.in.cmd`) or as a system property (e.g. in `SOLR_OPTS`). Example: `SOLR_MODULES=extraction,ltr`.
+
== Deprecations & Removed Features
The following list of features have been permanently removed from Solr:
diff --git a/solr/solr-ref-guide/src/solr-modules.adoc b/solr/solr-ref-guide/src/solr-modules.adoc
new file mode 100644
index 00000000000..b9683b542cc
--- /dev/null
+++ b/solr/solr-ref-guide/src/solr-modules.adoc
@@ -0,0 +1,41 @@
+= Solr Modules
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+Solr modules are addon Solr plugins that are not part of solr-core, but officially maintained
+by the Solr project. They provide well-defined features such as the "extracting" module which lets
+users index rich text documents with Apache Tika. A single module can contain multiple Plugins.
+Modules were earlier known as "contribs".
+
+Each module produces a separate `.jar` file in the build, and additional dependencies required by
+each module are also packaged with the module. This helps keep the main core of Solr small and lean.
+
+== Installing a module
+
+The easiest way to enable a module is to list the modules you intend to use either in the
+system property `solr.modules` or in the environment variable `SOLR_MODULES` (e.g. in`solr.in.sh`
+or `solr.in.cmd`). You can also add a `<str name="modules">` tag to
+<<configuring-solr-xml.adoc#,solr.xml>>. The expected value is a comma separated list
+of module names, e.g. `SOLR_MODULES=extracting,ltr`. This way of adding modules will add
+them to the shared class loader, making them available for every collection in Solr.
+
+If you only wish to enable a module for a single collection, you may add `<lib>` tags to `solrconfig.xml`
+as explained in <<libs.adoc#,Lib Directories>>.
+
+Some modules may have been made available as packages for the <<package-manager.adoc#,Package Manager>>,
+check by listing available pacakges.
\ No newline at end of file
diff --git a/solr/solr-ref-guide/src/solr-plugins.adoc b/solr/solr-ref-guide/src/solr-plugins.adoc
index 210106e70ed..dd2ba2c41ae 100644
--- a/solr/solr-ref-guide/src/solr-plugins.adoc
+++ b/solr/solr-ref-guide/src/solr-plugins.adoc
@@ -2,7 +2,8 @@
:page-children: libs, \
package-manager, \
cluster-plugins, \
- replica-placement-plugins
+ replica-placement-plugins, \
+ solr-modules
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -45,8 +46,8 @@ Examples of these are <<authentication-and-authorization-plugins.adoc#,authentic
== Installing Plugins ==
-Most plugins are built-in to Solr and there is nothing to install.
-The subject here is how to make other plugins available to Solr, including those in modules.
+Most plugins are built-in to Solr core and there is nothing to install.
+The subject here is how to make the bundled modules and other plugins available to Solr.
Plugins are packaged into a Java jar file and may have other dependent jar files to function.
The next sections describe some options:
@@ -57,6 +58,7 @@ The next sections describe some options:
[cols="1,1",frame=none,grid=none,stripes=none]
|===
| <<libs.adoc#,Lib Directories>>: Plugins as libraries on the filesystem.
+| <<solr-modules.adoc#,Solr Modules>>: Loading bundled modules.
| <<package-manager.adoc#,Package Management>>: Package-based plugins.
| <<cluster-plugins.adoc#,Cluster Plugins>>: Cluster-level plugins.
| <<replica-placement-plugins.adoc#,Replica Placement Plugins>>: Plugins specifically for replica placement.