Apache Solr open-source search software
SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
Jan Høydahl15534754f492
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 40da8c435ac..2fd94304a24 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -169,6 +169,8 @@ Other Changes* SOLR-17091: dev tools script cloud.sh became broken after changes in 9.3 added a new -slim.tgz file it was not expectingcloud.sh has been updated to ignore the -slim.tgz version of the tarball.+* SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman)+================== 9.4.0 ==================New Features---------------------diff --git a/solr/core/build.gradle b/solr/core/build.gradleindex 61ecd1713af..ed2c8a370ae 100644--- a/solr/core/build.gradle+++ b/solr/core/build.gradle@@ -159,6 +159,8 @@ dependencies {compileOnly 'com.github.stephenc.jcip:jcip-annotations'+ implementation 'com.j256.simplemagic:simplemagic'+// -- Test DependenciestestRuntimeOnly 'org.slf4j:jcl-over-slf4j'diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.javaindex 5fd4a538bd7..6576742a195 100644--- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java+++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java@@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.impl.SolrZkClientTimeout;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.cloud.ZkMaintenanceUtils;import org.apache.solr.core.ConfigSetService;+import org.apache.solr.util.FileTypeMagicUtil;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -101,6 +102,7 @@ public class ConfigSetUploadTool extends ToolBase {+ cli.getOptionValue("confname")+ " to ZooKeeper at "+ zkHost);+ FileTypeMagicUtil.assertConfigSetFolderLegal(confPath);ZkMaintenanceUtils.uploadToZK(zkClient,confPath,diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.javaindex f02404d636d..9abde098e1c 100644--- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java+++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java@@ -22,6 +22,7 @@ import java.nio.file.Path;import java.util.ArrayList;import java.util.Collections;import java.util.List;+import java.util.Locale;import java.util.Map;import java.util.Objects;import org.apache.solr.client.solrj.cloud.SolrCloudManager;@@ -39,6 +40,7 @@ import org.apache.solr.core.CoreContainer;import org.apache.solr.core.CoreDescriptor;import org.apache.solr.core.SolrConfig;import org.apache.solr.core.SolrResourceLoader;+import org.apache.solr.util.FileTypeMagicUtil;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;import org.apache.zookeeper.data.Stat;@@ -199,6 +201,15 @@ public class ZkConfigSetService extends ConfigSetService {try {if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);+ } else if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ String.format(+ Locale.ROOT,+ "Not uploading file %s to config, as it matched the MAGIC signature of a forbidden mime type %s",+ fileName,+ mimeType));} else {// if overwriteOnExists is true then zkClient#makePath failOnExists is set to falsezkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true);@@ -340,7 +351,15 @@ public class ZkConfigSetService extends ConfigSetService {} else {log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);byte[] data = zkClient.getData(fromZkFilePath, null, null, true);- zkClient.makePath(toZkFilePath, data, true);+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {+ zkClient.makePath(toZkFilePath, data, true);+ } else {+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);+ log.warn(+ "Skipping copy of file {} in ZK, as it matched the MAGIC signature of a forbidden mime type {}",+ fromZkFilePath,+ mimeType);+ }}}diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.javaindex 1f1a42b15e1..4b041252211 100644--- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java+++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java@@ -37,6 +37,7 @@ import java.util.stream.Stream;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.ZkMaintenanceUtils;import org.apache.solr.common.util.Utils;+import org.apache.solr.util.FileTypeMagicUtil;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -150,9 +151,17 @@ public class FileSystemConfigSetService extends ConfigSetService {if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);} else {- Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));- if (!Files.exists(filePath) || overwriteOnExists) {- Files.write(filePath, data);+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {+ Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));+ if (!Files.exists(filePath) || overwriteOnExists) {+ Files.write(filePath, data);+ }+ } else {+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);+ log.warn(+ "Not including uploading file {}, as it matched the MAGIC signature of a forbidden mime type {}",+ fileName,+ mimeType);}}}@@ -205,8 +214,17 @@ public class FileSystemConfigSetService extends ConfigSetService {"Not including uploading file to config, as it is a forbidden type: {}",file.getFileName());} else {- Files.copy(- file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(Files.newInputStream(file))) {+ Files.copy(+ file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);+ } else {+ String mimeType =+ FileTypeMagicUtil.INSTANCE.guessMimeType(Files.newInputStream(file));+ log.warn(+ "Not copying file {}, as it matched the MAGIC signature of a forbidden mime type {}",+ file.getFileName(),+ mimeType);+ }}return FileVisitResult.CONTINUE;}diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.javaindex be6a1a83c2f..8ff78b27e08 100644--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java@@ -40,6 +40,7 @@ import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.Utils;import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.backup.repository.BackupRepository;+import org.apache.solr.util.FileTypeMagicUtil;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;@@ -349,8 +350,16 @@ public class BackupManager {if (data == null) {data = new byte[0];}- try (OutputStream os = repository.createOutput(uri)) {- os.write(data);+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {+ try (OutputStream os = repository.createOutput(uri)) {+ os.write(data);+ }+ } else {+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);+ log.warn(+ "Not including zookeeper file {} in backup, as it matched the MAGIC signature of a forbidden mime type {}",+ filePath,+ mimeType);}}} else {@@ -379,7 +388,15 @@ public class BackupManager {// probably ok since the config file should be small.byte[] arr = new byte[(int) is.length()];is.readBytes(arr, 0, (int) is.length());- configSetService.uploadFileToConfig(configName, filePath, arr, false);+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(arr)) {+ configSetService.uploadFileToConfig(configName, filePath, arr, false);+ } else {+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(arr);+ log.warn(+ "Not including zookeeper file {} in restore, as it matched the MAGIC signature of a forbidden mime type {}",+ filePath,+ mimeType);+ }}}break;diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.javaindex 98889aff563..2380a79a92b 100644--- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java+++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java@@ -27,6 +27,7 @@ import org.apache.solr.common.params.ConfigSetParams;import org.apache.solr.core.CoreContainer;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;+import org.apache.solr.util.FileTypeMagicUtil;/*** V2 API for adding or updating a single file within a configset.@@ -67,11 +68,13 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase {if (fixedSingleFilePath.charAt(0) == '/') {fixedSingleFilePath = fixedSingleFilePath.substring(1);}+ byte[] data = inputStream.readAllBytes();if (fixedSingleFilePath.isEmpty()) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"The file path provided for upload, '" + singleFilePath + "', is not valid.");- } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)) {+ } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)+ || FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"The file type provided for upload, '"@@ -87,8 +90,7 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase {// For creating the baseNode, the cleanup parameter is only allowed to be true when// singleFilePath is not passed.createBaseNode(configSetService, overwritesExisting, requestIsTrusted, configSetName);- configSetService.uploadFileToConfig(- configSetName, fixedSingleFilePath, inputStream.readAllBytes(), allowOverwrite);+ configSetService.uploadFileToConfig(configSetName, fixedSingleFilePath, data, allowOverwrite);}}}diff --git a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.javanew file mode 100644index 00000000000..cfb6c9fa0af--- /dev/null+++ b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java@@ -0,0 +1,166 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.util;++import com.j256.simplemagic.ContentInfo;+import com.j256.simplemagic.ContentInfoUtil;+import com.j256.simplemagic.ContentType;+import java.io.ByteArrayInputStream;+import java.io.IOException;+import java.io.InputStream;+import java.nio.file.FileVisitResult;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.SimpleFileVisitor;+import java.nio.file.attribute.BasicFileAttributes;+import java.util.Arrays;+import java.util.HashSet;+import java.util.Locale;+import java.util.Set;+import org.apache.solr.common.SolrException;++/** Utility class to guess the mime type of file based on its magic number. */+public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack {+ private final ContentInfoUtil util;+ private static final Set<String> SKIP_FOLDERS = new HashSet<>(Arrays.asList(".", ".."));++ public static FileTypeMagicUtil INSTANCE = new FileTypeMagicUtil();++ FileTypeMagicUtil() {+ try {+ util = new ContentInfoUtil("/magic/executables", this);+ } catch (IOException e) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error parsing magic file", e);+ }+ }++ /**+ * Asserts that an entire configset folder is legal to upload.+ *+ * @param confPath the path to the folder+ * @throws SolrException if an illegal file is found in the folder structure+ */+ public static void assertConfigSetFolderLegal(Path confPath) throws IOException {+ Files.walkFileTree(+ confPath,+ new SimpleFileVisitor<Path>() {+ @Override+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)+ throws IOException {+ // Read first 100 bytes of the file to determine the mime type+ try (InputStream fileStream = Files.newInputStream(file)) {+ byte[] bytes = new byte[100];+ fileStream.read(bytes);+ if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ String.format(+ Locale.ROOT,+ "Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s",+ file,+ FileTypeMagicUtil.INSTANCE.guessMimeType(bytes)));+ }+ return FileVisitResult.CONTINUE;+ }+ }++ @Override+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)+ throws IOException {+ if (SKIP_FOLDERS.contains(dir.getFileName().toString()))+ return FileVisitResult.SKIP_SUBTREE;++ return FileVisitResult.CONTINUE;+ }+ });+ }++ /**+ * Guess the mime type of file based on its magic number.+ *+ * @param stream input stream of the file+ * @return string with content-type or "application/octet-stream" if unknown+ */+ public String guessMimeType(InputStream stream) {+ try {+ ContentInfo info = util.findMatch(stream);+ if (info == null) {+ return ContentType.OTHER.getMimeType();+ }+ return info.getContentType().getMimeType();+ } catch (IOException e) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ }+ }++ /**+ * Guess the mime type of file bytes based on its magic number.+ *+ * @param bytes the first bytes at start of the file+ * @return string with content-type or "application/octet-stream" if unknown+ */+ public String guessMimeType(byte[] bytes) {+ return guessMimeType(new ByteArrayInputStream(bytes));+ }++ @Override+ public void error(String line, String details, Exception e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR,+ String.format(Locale.ROOT, "%s: %s", line, details),+ e);+ }++ /**+ * Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types+ * are:+ *+ * <ul>+ * <li><code>application/x-java-applet</code>: java class file+ * <li><code>application/zip</code>: jar or zip archives+ * <li><code>application/x-tar</code>: tar archives+ * <li><code>text/x-shellscript</code>: shell or bash script+ * </ul>+ *+ * @param fileStream stream from the file content+ * @return true if file is among the forbidden mime-types+ */+ public static boolean isFileForbiddenInConfigset(InputStream fileStream) {+ return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream));+ }++ /**+ * Determine forbidden file type based on magic bytes matching of the first bytes of the file.+ *+ * @param bytes byte array of the file content+ * @return true if file is among the forbidden mime-types+ */+ public static boolean isFileForbiddenInConfigset(byte[] bytes) {+ if (bytes == null || bytes.length == 0)+ return false; // A ZK znode may be a folder with no content+ return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes));+ }++ private static final Set<String> forbiddenTypes =+ new HashSet<>(+ Arrays.asList(+ System.getProperty(+ "solr.configset.upload.mimetypes.forbidden",+ "application/x-java-applet,application/zip,application/x-tar,text/x-shellscript")+ .split(",")));+}diff --git a/solr/core/src/resources/magic/executables b/solr/core/src/resources/magic/executablesnew file mode 100644index 00000000000..04094eaf797--- /dev/null+++ b/solr/core/src/resources/magic/executables@@ -0,0 +1,74 @@+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# POSIX tar archives+# URL: https://en.wikipedia.org/wiki/Tar_(computing)+# Reference: https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current+# header mainly padded with nul bytes+500 quad 0+!:strength /2+# filename or extended attribute printable strings in range space null til umlaut ue+>0 ubeshort >0x1F00+>>0 ubeshort <0xFCFD+# last 4 header bytes often null but tar\0 in gtarfail2.tar gtarfail.tar-bad+# at https://sourceforge.net/projects/s-tar/files/testscripts/+>>>508 ubelong&0x8B9E8DFF 0+# nul, space or ascii digit 0-7 at start of mode+>>>>100 ubyte&0xC8 =0+>>>>>101 ubyte&0xC8 =0+# nul, space at end of check sum+>>>>>>155 ubyte&0xDF =0+# space or ascii digit 0 at start of check sum+>>>>>>>148 ubyte&0xEF =0x20+# check for specific 1st member name that indicates other mime type and file name suffix+>>>>>>>>0 string TpmEmuTpms/permall+!:mime application/x-tar+!:ext tar+# other stuff in padding+# some implementations add new fields to the blank area at the end of the header record+# created for example by DOS TAR 3.20g 1994 Tim V.Shapore with -j option+>>257 ulong !0 tar archive (old)+!:mime application/x-tar+!:ext tar+# magic in newer, GNU, posix variants+>257 string =ustar+# 2 last char of magic and UStar version because string expression does not work+# 2 space characters followed by a null for GNU variant+>>261 ubelong =0x72202000 POSIX tar archive (GNU)+!:mime application/x-gtar+!:ext tar/gtar+++# Zip archives (Greg Roelofs, c/o [email protected])+0 string PK\005\006 Zip archive data (empty)+0 string PK\003\004 Zip archive data+!:strength +1+!:mime application/zip+!:ext zip/cbz+++# JAVA+0 belong 0xcafebabe+>4 ubelong >30 compiled Java class data,+!:mime application/x-java-applet+#!:mime application/java-byte-code+!:ext class+++# SHELL scripts+#0 string/w : shell archive or script for antique kernel text+0 regex \^#!\\s?(/bin/|/usr/) POSIX shell script text executable+!:mime text/x-shellscript+!:ext sh/bash\ No newline at end of filediff --git a/solr/core/src/test-files/magic/HelloWorld.java.txt b/solr/core/src/test-files/magic/HelloWorld.java.txtnew file mode 100644index 00000000000..ca9518d2afa--- /dev/null+++ b/solr/core/src/test-files/magic/HelloWorld.java.txt@@ -0,0 +1,5 @@+class HelloWorld {+ public static void main(String[] args) {+ System.out.println("Hellow world");+ }+}\ No newline at end of filediff --git a/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.binnew file mode 100644index 00000000000..e15d0a6c5b9Binary files /dev/null and b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin differdiff --git a/solr/core/src/test-files/magic/README.md b/solr/core/src/test-files/magic/README.mdnew file mode 100644index 00000000000..6e499a2f711--- /dev/null+++ b/solr/core/src/test-files/magic/README.md@@ -0,0 +1,29 @@+<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+ -->++The two binary files were created by the following commands:++```bash+echo "Hello" > hello.txt && \+ tar -cvf hello.tar.bin hello.txt && \+ rm hello.txt++cp HelloWorld.java.txt HelloWorld.java && \+ javac HelloWorld.java && \+ mv HelloWorld.class HelloWorldJavaClass.class.bin && \+ rm HelloWorld.java+```\ No newline at end of filediff --git a/solr/core/src/test-files/magic/hello.tar.bin b/solr/core/src/test-files/magic/hello.tar.binnew file mode 100644index 00000000000..68ca23c362aBinary files /dev/null and b/solr/core/src/test-files/magic/hello.tar.bin differdiff --git a/solr/core/src/test-files/magic/plain.txt b/solr/core/src/test-files/magic/plain.txtnew file mode 100644index 00000000000..70c379b63ff--- /dev/null+++ b/solr/core/src/test-files/magic/plain.txt@@ -0,0 +1 @@+Hello world\ No newline at end of filediff --git a/solr/core/src/test-files/magic/shell.sh.txt b/solr/core/src/test-files/magic/shell.sh.txtnew file mode 100644index 00000000000..9ea411e111d--- /dev/null+++ b/solr/core/src/test-files/magic/shell.sh.txt@@ -0,0 +1,2 @@+#! /usr/bin/env bash+echo Hello\ No newline at end of filediff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.javaindex 1da60d85ff9..f7c9431c296 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java@@ -45,6 +45,7 @@ import java.util.LinkedHashMap;import java.util.List;import java.util.Locale;import java.util.Map;+import java.util.Objects;import java.util.Properties;import java.util.Set;import java.util.concurrent.TimeUnit;@@ -592,14 +593,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {assertEquals("Can't overwrite an existing configset unless the overwrite parameter is set",400,- uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false));+ uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false, false));unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper");assertEquals("Expecting version to remain equal",solrconfigZkVersion,getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));assertEquals(- 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));assertTrue("Expecting version bump",solrconfigZkVersion@@ -638,13 +639,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {zkClient.makePath(f, true);}assertEquals(- 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));for (String f : extraFiles) {assertTrue("Expecting file " + f + " to exist in ConfigSet but it's gone",zkClient.exists(f, true));}- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false));+ assertEquals(+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false));for (String f : extraFiles) {assertFalse("Expecting file " + f + " to be deleted from ConfigSet but it wasn't",@@ -675,7 +677,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {.withConnTimeOut(45000, TimeUnit.MILLISECONDS).build()) {String configPath = "/configs/" + configsetName + configsetSuffix;- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true));+ assertEquals(+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true, false));for (String fileEnding : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {String f = configPath + "/test." + fileEnding;assertFalse(@@ -710,7 +713,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");// Was untrusted, overwrite with untrustedassertEquals(- 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));assertTrue("Expecting version bump",solrconfigZkVersion@@ -721,7 +724,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {// Was untrusted, overwrite with trusted but no cleanupassertEquals(- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false));+ 0,+ uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false));assertTrue("Expecting version bump",solrconfigZkVersion@@ -747,7 +751,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {// Was untrusted, overwrite with trusted with cleanupassertEquals(- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false));+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false));assertTrue("Expecting version bump",solrconfigZkVersion@@ -761,7 +765,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {assertEquals("Can't upload a trusted configset with an untrusted request",400,- uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));+ uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));assertEquals("Expecting version to remain equal",solrconfigZkVersion,@@ -773,7 +777,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {assertEquals("Can't upload a trusted configset with an untrusted request",400,- uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false));+ uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false));assertEquals("Expecting version to remain equal",solrconfigZkVersion,@@ -783,7 +787,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {// Was trusted, overwrite with trusted no cleanupassertEquals(- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false));+ 0,+ uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false));assertTrue("Expecting version bump",solrconfigZkVersion@@ -794,7 +799,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {// Was trusted, overwrite with trusted with cleanupassertEquals(- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false));+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false));assertTrue("Expecting version bump",solrconfigZkVersion@@ -1457,6 +1462,13 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {.get("id"));}+ @Test+ public void testUploadWithForbiddenContent() throws Exception {+ // Uploads a config set containing a script, a class file and jar file, will return 400 error+ long res = uploadConfigSet("forbidden", "suffix", "foo", true, false, true, false, true);+ assertEquals(400, res);+ }+private static String getSecurityJson() {return "{\n"+ " 'authentication':{\n"@@ -1511,7 +1523,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {String configSetName, String suffix, String username, SolrZkClient zkClient, boolean v2)throws IOException {assertFalse(getConfigSetService().checkConfigExists(configSetName + suffix));- return uploadConfigSet(configSetName, suffix, username, false, false, v2, false);+ return uploadConfigSet(configSetName, suffix, username, false, false, v2, false, false);}private long uploadConfigSet(@@ -1521,21 +1533,25 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {boolean overwrite,boolean cleanup,boolean v2,- boolean forbiddenTypes)+ boolean forbiddenTypes,+ boolean forbiddenContent)throws IOException {+ File zipFile;+ if (forbiddenTypes) {+ log.info("Uploading configset with forbidden file endings");+ zipFile =+ createTempZipFileWithForbiddenTypes(+ "solr/configsets/upload/" + configSetName + "/solrconfig.xml");+ } else if (forbiddenContent) {+ log.info("Uploading configset with forbidden file content");+ zipFile = createTempZipFileWithForbiddenContent("magic");+ } else {+ zipFile = createTempZipFile("solr/configsets/upload/" + configSetName);+ }+// Read zipped sample config- return uploadGivenConfigSet(- forbiddenTypes- ? createTempZipFileWithForbiddenTypes(- "solr/configsets/upload/" + configSetName + "/solrconfig.xml")- : createTempZipFile("solr/configsets/upload/" + configSetName),- configSetName,- suffix,- username,- overwrite,- cleanup,- v2);+ return uploadGivenConfigSet(zipFile, configSetName, suffix, username, overwrite, cleanup, v2);}private long uploadBadConfigSet(String configSetName, String suffix, String username, boolean v2)@@ -1702,31 +1718,68 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {}}- private static void zipWithForbiddenEndings(File file, File zipfile) throws IOException {- OutputStream out = new FileOutputStream(zipfile);- ZipOutputStream zout = new ZipOutputStream(out);+ /** Create a zip file (in the temp directory) containing files with forbidden content */+ private File createTempZipFileWithForbiddenContent(String resourcePath) {try {- for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {- zout.putNextEntry(new ZipEntry("test." + fileType));+ final File zipFile = createTempFile("configset", "zip").toFile();+ final File directory = SolrTestCaseJ4.getFile(resourcePath);+ if (log.isInfoEnabled()) {+ log.info("Directory: {}", directory.getAbsolutePath());+ }+ zipWithForbiddenContent(directory, zipFile);+ if (log.isInfoEnabled()) {+ log.info("Zipfile: {}", zipFile.getAbsolutePath());+ }+ return zipFile;+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }- InputStream in = new FileInputStream(file);- try {- byte[] buffer = new byte[1024];- while (true) {- int readCount = in.read(buffer);- if (readCount < 0) {- break;+ private static void zipWithForbiddenContent(File directory, File zipfile) throws IOException {+ OutputStream out = Files.newOutputStream(zipfile.toPath());+ assertTrue(directory.isDirectory());+ try (ZipOutputStream zout = new ZipOutputStream(out)) {+ // Copy in all files from the directory+ for (File file : Objects.requireNonNull(directory.listFiles())) {+ zout.putNextEntry(new ZipEntry(file.getName()));+ zout.write(Files.readAllBytes(file.toPath()));+ zout.closeEntry();+ }+ }+ }++ private static void zipWithForbiddenEndings(File fileOrDirectory, File zipfile)+ throws IOException {+ OutputStream out = new FileOutputStream(zipfile);+ try (ZipOutputStream zout = new ZipOutputStream(out)) {+ if (fileOrDirectory.isFile()) {+ // Create entries with given file, one for each forbidden endding+ for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {+ zout.putNextEntry(new ZipEntry("test." + fileType));++ try (InputStream in = new FileInputStream(fileOrDirectory)) {+ byte[] buffer = new byte[1024];+ while (true) {+ int readCount = in.read(buffer);+ if (readCount < 0) {+ break;+ }+ zout.write(buffer, 0, readCount);}- zout.write(buffer, 0, readCount);}- } finally {- in.close();- }- zout.closeEntry();+ zout.closeEntry();+ }+ }+ if (fileOrDirectory.isDirectory()) {+ // Copy in all files from the directory+ for (File file : Objects.requireNonNull(fileOrDirectory.listFiles())) {+ zout.putNextEntry(new ZipEntry(file.getName()));+ zout.write(Files.readAllBytes(file.toPath()));+ zout.closeEntry();+ }}- } finally {- zout.close();}}diff --git a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.javanew file mode 100644index 00000000000..b8e9a35a3d9--- /dev/null+++ b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java@@ -0,0 +1,54 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.util;++import org.apache.solr.SolrTestCaseJ4;++public class FileTypeMagicUtilTest extends SolrTestCaseJ4 {+ public void testGuessMimeType() {+ assertEquals(+ "application/x-java-applet",+ FileTypeMagicUtil.INSTANCE.guessMimeType(+ FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));+ assertEquals(+ "application/zip",+ FileTypeMagicUtil.INSTANCE.guessMimeType(+ FileTypeMagicUtil.class.getResourceAsStream(+ "/runtimecode/containerplugin.v.1.jar.bin")));+ assertEquals(+ "application/x-tar",+ FileTypeMagicUtil.INSTANCE.guessMimeType(+ FileTypeMagicUtil.class.getResourceAsStream("/magic/hello.tar.bin")));+ assertEquals(+ "text/x-shellscript",+ FileTypeMagicUtil.INSTANCE.guessMimeType(+ FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));+ }++ public void testIsFileForbiddenInConfigset() {+ assertTrue(+ FileTypeMagicUtil.isFileForbiddenInConfigset(+ FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));+ assertTrue(+ FileTypeMagicUtil.isFileForbiddenInConfigset(+ FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));+ assertFalse(+ FileTypeMagicUtil.isFileForbiddenInConfigset(+ FileTypeMagicUtil.class.getResourceAsStream("/magic/plain.txt")));+ }+}diff --git a/solr/licenses/simplemagic-1.17.jar.sha1 b/solr/licenses/simplemagic-1.17.jar.sha1new file mode 100644index 00000000000..cf101094cc8--- /dev/null+++ b/solr/licenses/simplemagic-1.17.jar.sha1@@ -0,0 +1 @@+b6e2d1e47d7172e57fa858a2e3940c09a590e61ediff --git a/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txtnew file mode 100644index 00000000000..9228230f933--- /dev/null+++ b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt@@ -0,0 +1,15 @@+ISC License (https://opensource.org/licenses/ISC)++Copyright 2021, Gray Watson++Permission to use, copy, modify, and/or distribute this software for any+purpose with or without fee is hereby granted, provided that the above+copyright notice and this permission notice appear in all copies.++THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\ No newline at end of filediff --git a/solr/licenses/simplemagic-NOTICE.txt b/solr/licenses/simplemagic-NOTICE.txtnew file mode 100644index 00000000000..e69de29bb2ddiff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.javaindex d571339880c..e40294a6683 100644--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java@@ -348,6 +348,7 @@ public class ZkMaintenanceUtils {USE_FORBIDDEN_FILE_TYPES);return FileVisitResult.CONTINUE;}+ // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in coreString zkNode = createZkNodeName(zkPath, rootPath, file);try {// if the path exists (and presumably we're uploading data to it) just set its data@@ -437,6 +438,7 @@ public class ZkMaintenanceUtils {if (isFileForbiddenInConfigSets(zkPath)) {log.warn("Skipping download of file from ZK, as it is a forbidden type: {}", zkPath);} else {+ // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in coreif (copyDataDown(zkClient, zkPath, file) == 0) {Files.createFile(file);}diff --git a/versions.lock b/versions.lockindex 9c0f9ef53d4..6b6ea467e0d 100644--- a/versions.lock+++ b/versions.lock@@ -62,6 +62,7 @@ com.googlecode.plist:dd-plist:1.24 (1 constraints: 300c84f5)com.healthmarketscience.jackcess:jackcess:4.0.2 (1 constraints: 5d0cf201)com.healthmarketscience.jackcess:jackcess-encrypt:4.0.1 (1 constraints: 5c0cf101)com.ibm.icu:icu4j:70.1 (1 constraints: a90f1784)+com.j256.simplemagic:simplemagic:1.17 (1 constraints: dd04f830)com.jayway.jsonpath:json-path:2.8.0 (2 constraints: 6c12952c)com.lmax:disruptor:3.4.4 (1 constraints: 0d050a36)com.mchange:c3p0:0.9.5.5 (1 constraints: c80c571b)diff --git a/versions.props b/versions.propsindex 10b583c43e2..cfa127e1094 100644--- a/versions.props+++ b/versions.props@@ -13,6 +13,7 @@ com.google.cloud:google-cloud-bom=0.204.0com.google.errorprone:*=2.23.0com.google.guava:guava=32.1.3-jrecom.google.re2j:re2j=1.7+com.j256.simplemagic:simplemagic=1.17com.jayway.jsonpath:json-path=2.8.0com.lmax:disruptor=3.4.4com.tdunning:t-digest=3.1