Apache Solr open-source search software
Revert "Revert "SOLR-15950 Create SOLR_HOME/filestore lazily on first use, do not create userfiles folder (#593)""
Jan Høydahl067944ca587b
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 87f727b09b7..780575c166f 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -581,6 +581,8 @@ and each individual module's jar will be included in its directory's lib/ folder* SOLR-15936: Reduce unnecessary startup logging, such as SSL warnings when SSL not in use (janhoy)+* SOLR-15950: Create SOLR_HOME/filestore lazily on first use. Do not automatically create SOLR_HOME/userfiles (janhoy)+* SOLR-15777: Forbid useDocValuesAsStored for ICUCollationField (warn for luceneMatchVersion < 9.0.0). (Michael Gibney)* SOLR-12901: Highlighting: hl.method=unified is the new default. (David Smiley)diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex 99abee8c70b..152ce1f65f9 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -118,7 +118,6 @@ import org.slf4j.LoggerFactory;import java.io.IOException;import java.lang.invoke.MethodHandles;-import java.nio.file.Files;import java.nio.file.Path;import java.nio.file.Paths;import java.security.spec.InvalidKeySpecException;@@ -829,9 +828,6 @@ public class CoreContainer {metricManager.loadClusterReporters(metricReporters, this);}- // Do this before cores get created- createUserFilesDirectory();-// setup executor to load cores in parallelExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService(ExecutorUtil.newMDCAwareFixedThreadPool(@@ -948,17 +944,6 @@ public class CoreContainer {status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;}- private void createUserFilesDirectory() {- if (isZooKeeperAware()) {- Path userFilesPath = getUserFilesPath(); // TODO make configurable on cfg?- try {- Files.createDirectories(userFilesPath); // does nothing if already exists- } catch (Exception e) {- log.warn("Unable to create [{}]. Features requiring this directory may fail.", userFilesPath, e);- }- }- }-public void securityNodeChanged() {log.info("Security node changed, reloading security.json");reloadSecurityProperties();@@ -2084,8 +2069,7 @@ public class CoreContainer {/*** A path where Solr users can retrieve arbitrary files from. Absolute.* <p>- * This directory is generally created by each node on startup. Files located in this directory can then be- * manipulated using select Solr features (e.g. streaming expressions).+ * Files located in this directory can be manipulated using select Solr features (e.g. streaming expressions).*/public Path getUserFilesPath() {return solrHome.resolve("userfiles");diff --git a/solr/core/src/java/org/apache/solr/core/SolrPaths.java b/solr/core/src/java/org/apache/solr/core/SolrPaths.javaindex 37fe247a209..00ec58b26e3 100644--- a/solr/core/src/java/org/apache/solr/core/SolrPaths.java+++ b/solr/core/src/java/org/apache/solr/core/SolrPaths.java@@ -71,16 +71,10 @@ public final class SolrPaths {public static void assertPathAllowed(Path pathToAssert, Set<Path> allowPaths) throws SolrException {if (ALL_PATHS.equals(allowPaths)) return; // Catch-all allows all paths (*/_ALL_)if (pathToAssert == null) return;- if (OS.isFamilyWindows() && pathToAssert.toString().startsWith("\\\\")) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,- "Path " + pathToAssert + " disallowed. UNC paths not supported. Please use drive letter instead.");- }+ assertNotUnc(pathToAssert);// Conversion Path -> String -> Path is to be able to compare against org.apache.lucene.mockfile.FilterPath instancesfinal Path path = Path.of(pathToAssert.toString()).normalize();- if (path.startsWith("..")) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,- "Path " + pathToAssert + " disallowed due to path traversal..");- }+ assertNoPathTraversal(path);if (!path.isAbsolute()) return; // All relative paths are acceptedif (allowPaths.stream().noneMatch(p -> path.startsWith(Path.of(p.toString())))) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,@@ -88,6 +82,26 @@ public final class SolrPaths {}}+ /**+ * Asserts that a path does not contain directory traversal+ */+ public static void assertNoPathTraversal(Path pathToAssert) {+ if (pathToAssert.toString().contains("..")) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,+ "Path " + pathToAssert + " disallowed due to path traversal..");+ }+ }++ /**+ * Asserts that a path is not a Windows UNC path+ */+ public static void assertNotUnc(Path pathToAssert) {+ if (OS.isFamilyWindows() && pathToAssert.toString().startsWith("\\\\")) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,+ "Path " + pathToAssert + " disallowed. UNC paths not supported.");+ }+ }+/*** Builds a set of allowed {@link Path}.* Detects special paths "*" and "_ALL_" that mean all paths are allowed.diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.javaindex 0df05381479..775c0096973 100644--- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java+++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java@@ -51,6 +51,7 @@ import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.util.Utils;import org.apache.solr.common.annotation.SolrThreadUnsafe;import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.SolrPaths;import org.apache.solr.filestore.PackageStoreAPI.MetaData;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;@@ -91,11 +92,16 @@ public class DistribPackageStore implements PackageStore {if (File.separatorChar == '\\') {path = path.replace('/', File.separatorChar);}- if (!path.isEmpty() && path.charAt(0) != File.separatorChar) {- path = File.separator + path;+ SolrPaths.assertNotUnc(Path.of(path));+ while (path.startsWith(File.separator)) { // Trim all leading slashes+ path = path.substring(1);}- // Use concat because path might start with a slash and be incorrectly interpreted as absolute- return solrHome.resolve(PackageStoreAPI.PACKAGESTORE_DIRECTORY + path);+ var finalPath = getPackageStoreDirPath(solrHome).resolve(path);+ // Guard against path traversal by asserting final path is sub path of filestore+ if (!finalPath.normalize().startsWith(getPackageStoreDirPath(solrHome).normalize())) {+ throw new SolrException(BAD_REQUEST, "Illegal path " + path);+ }+ return finalPath;}class FileInfo {@@ -571,8 +577,17 @@ public class DistribPackageStore implements PackageStore {}}- public static Path getPackageStoreDirPath(Path solrHome) {- return solrHome.resolve(PackageStoreAPI.PACKAGESTORE_DIRECTORY);+ public static synchronized Path getPackageStoreDirPath(Path solrHome) {+ var path = solrHome.resolve(PackageStoreAPI.PACKAGESTORE_DIRECTORY);+ if (!Files.exists(path)) {+ try {+ Files.createDirectories(path);+ log.info("Created filestore folder {}", path);+ } catch (IOException e) {+ throw new SolrException(SERVER_ERROR, "Faild creating 'filestore' folder in SOLR_HOME", e);+ }+ }+ return path;}private static String _getMetapath(String path) {diff --git a/solr/solr-ref-guide/modules/query-guide/pages/loading.adoc b/solr/solr-ref-guide/modules/query-guide/pages/loading.adocindex 571bff93098..150f981ff2c 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/loading.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/loading.adoc@@ -22,7 +22,7 @@ These functions are designed to cut down the time spent on data preparation and== Reading FilesThe `cat` function can be used to read files under the *userfiles* directory in-`$SOLR_HOME`.+`$SOLR_HOME`. This directory must be created by the user.The `cat` function takes two parameters.The first parameter is a comma-delimited list of paths.diff --git a/solr/solr-ref-guide/modules/query-guide/pages/stream-source-reference.adoc b/solr/solr-ref-guide/modules/query-guide/pages/stream-source-reference.adocindex 60f421f380c..abe3b97e5d8 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/stream-source-reference.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/stream-source-reference.adoc@@ -325,7 +325,7 @@ Each emitted tuple contains two fields: `file` and `line`.* `filePaths`: (Mandatory) a comma separated list of filepaths to read lines from.If the specified path is a directory, it will be crawled recursively and all contained files will be read.-To prevent malicious users from reading arbitrary files from Solr nodes, `filePaths` must be a relative path measured from a chroot of `$SOLR_HOME/userfiles` on the node running the streaming expression.+To prevent malicious users from reading arbitrary files from Solr nodes, `filePaths` must be a relative path measured from a chroot of `$SOLR_HOME/userfiles` on the node running the streaming expression. This directory must be created by the user.* `maxLines`: (defaults to -1) The maximum number of lines to read (and tuples to emit).If a negative value is specified, all lines in the specified files will be emitted as tuples.Files are read in the order that they appear in the comma-separated `filePaths` argument.diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adocindex b03493b4461..2bc9c22a164 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc@@ -128,6 +128,8 @@ changes, however the module needs to be installed - see the section xref:deploym* SOLR-13989: Hadoop authentication support has been moved to the hadoop-auth module. Existing Solr configurations do not need any Hadoop authentication relatedchanges, however the module needs to be installed - see the section xref:deployment-guide:hadoop-authentication-plugin.adoc[].+* SOLR-15950: The folder $SOLR_HOME/userfiles, used by the "cat" streaming expression, is no longer created automatically on startup. The user must create this folder.+* SOLR-15097: JWTAuthPlugin has been moved to a module. Users need to add the module to classpath. The plugin has alsochanged package name to `org.apache.solr.security.jwt`, but can still be loaded as shortform `class="solr.JWTAuthPlugin"`.diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.javaindex 40437087a22..38c09023ef4 100644--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java@@ -3917,7 +3917,6 @@ public class StreamExpressionTest extends SolrCloudTestCase {private static Path findUserFilesDataDir() {for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {- final String baseDir = cluster.getBaseDir().toAbsolutePath().toString();for (CoreDescriptor coreDescriptor : jetty.getCoreContainer().getCoreDescriptors()) {if (coreDescriptor.getCollectionName().equals(FILESTREAM_COLLECTION)) {return jetty.getCoreContainer().getUserFilesPath();