Apache Solr open-source search software
Update to Hadoop 3.3.3 (#907)
Mike Drob21598a9efa0e
diff --git a/solr/licenses/hadoop-annotations-3.3.2.jar.sha1 b/solr/licenses/hadoop-annotations-3.3.2.jar.sha1deleted file mode 100644index d5a4186fb51..00000000000--- a/solr/licenses/hadoop-annotations-3.3.2.jar.sha1+++ /dev/null@@ -1 +0,0 @@-2b47cd39c02c873b6bb29193962735d1d56f6572diff --git a/solr/licenses/hadoop-annotations-3.3.3.jar.sha1 b/solr/licenses/hadoop-annotations-3.3.3.jar.sha1new file mode 100644index 00000000000..b5e4d34342b--- /dev/null+++ b/solr/licenses/hadoop-annotations-3.3.3.jar.sha1@@ -0,0 +1 @@+f093f02eb980be6afcda2c86aff71bcf8d9c5d9cdiff --git a/solr/licenses/hadoop-auth-3.3.2.jar.sha1 b/solr/licenses/hadoop-auth-3.3.2.jar.sha1deleted file mode 100644index 889352f9ca9..00000000000--- a/solr/licenses/hadoop-auth-3.3.2.jar.sha1+++ /dev/null@@ -1 +0,0 @@-32b81a77ea6ffcbf524d1cda8ab20ea3522e6fd5diff --git a/solr/licenses/hadoop-auth-3.3.3.jar.sha1 b/solr/licenses/hadoop-auth-3.3.3.jar.sha1new file mode 100644index 00000000000..4c2d0542654--- /dev/null+++ b/solr/licenses/hadoop-auth-3.3.3.jar.sha1@@ -0,0 +1 @@+6788fe94013801dca0712e3fd19b3c140f03b6e0diff --git a/solr/licenses/hadoop-client-api-3.3.2.jar.sha1 b/solr/licenses/hadoop-client-api-3.3.2.jar.sha1deleted file mode 100644index 9c2cc214789..00000000000--- a/solr/licenses/hadoop-client-api-3.3.2.jar.sha1+++ /dev/null@@ -1 +0,0 @@-48f1af0a3a0270095dc59dc9f7d698969de4b4bfdiff --git a/solr/licenses/hadoop-client-api-3.3.3.jar.sha1 b/solr/licenses/hadoop-client-api-3.3.3.jar.sha1new file mode 100644index 00000000000..cd914d8cd2d--- /dev/null+++ b/solr/licenses/hadoop-client-api-3.3.3.jar.sha1@@ -0,0 +1 @@+d0593aed2d4df9bcee507550913d29d589ebd84adiff --git a/solr/licenses/hadoop-client-minicluster-3.3.2.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.3.2.jar.sha1deleted file mode 100644index 379ff4a3bb3..00000000000--- a/solr/licenses/hadoop-client-minicluster-3.3.2.jar.sha1+++ /dev/null@@ -1 +0,0 @@-c08ddd065de27d21c2c2b398084092377f16a06bdiff --git a/solr/licenses/hadoop-client-minicluster-3.3.3.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.3.3.jar.sha1new file mode 100644index 00000000000..d0b9ae91b1b--- /dev/null+++ b/solr/licenses/hadoop-client-minicluster-3.3.3.jar.sha1@@ -0,0 +1 @@+5820fe54634c163e08c8040747fb089f31acb90ddiff --git a/solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1deleted file mode 100644index bd6ffdb79bc..00000000000--- a/solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1+++ /dev/null@@ -1 +0,0 @@-0112f2b7420fa77c62148799175c073594197e6cdiff --git a/solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1new file mode 100644index 00000000000..1821815ab2a--- /dev/null+++ b/solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1@@ -0,0 +1 @@+52619ecfb0225d7ae67b15264521064824ac57cadiff --git a/solr/licenses/hadoop-common-3.3.2.jar.sha1 b/solr/licenses/hadoop-common-3.3.2.jar.sha1deleted file mode 100644index a97b8aa2432..00000000000--- a/solr/licenses/hadoop-common-3.3.2.jar.sha1+++ /dev/null@@ -1 +0,0 @@-edec4cdc7f1b1208f7d135f9f228ba44b83cf58fdiff --git a/solr/licenses/hadoop-common-3.3.3.jar.sha1 b/solr/licenses/hadoop-common-3.3.3.jar.sha1new file mode 100644index 00000000000..46804c863ce--- /dev/null+++ b/solr/licenses/hadoop-common-3.3.3.jar.sha1@@ -0,0 +1 @@+2e1293fddedc9d5c42ce07703d22c33eb3b39aa3diff --git a/solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1deleted file mode 100644index a3abe8edce5..00000000000--- a/solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1+++ /dev/null@@ -1 +0,0 @@-20b47a6fe7780ae0c250081d2b9f30cd07691b6cdiff --git a/solr/licenses/hadoop-hdfs-3.3.2.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.2.jar.sha1deleted file mode 100644index 7f48dfd8182..00000000000--- a/solr/licenses/hadoop-hdfs-3.3.2.jar.sha1+++ /dev/null@@ -1 +0,0 @@-aed57238fd4e669043bcc29d6bea3f0e6420950ddiff --git a/solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1new file mode 100644index 00000000000..f8fe3f45396--- /dev/null+++ b/solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1@@ -0,0 +1 @@+1bbe7acb60ffd432cc4bce646673279d0febdbc2diff --git a/solr/licenses/hadoop-hdfs-3.3.3.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.3.jar.sha1new file mode 100644index 00000000000..5bc3a315c32--- /dev/null+++ b/solr/licenses/hadoop-hdfs-3.3.3.jar.sha1@@ -0,0 +1 @@+d4d199760c11d47f90e12fe3882e2b24c77e4eb5diff --git a/solr/licenses/hadoop-minikdc-3.3.2.jar.sha1 b/solr/licenses/hadoop-minikdc-3.3.2.jar.sha1deleted file mode 100644index 7e62097c3a6..00000000000--- a/solr/licenses/hadoop-minikdc-3.3.2.jar.sha1+++ /dev/null@@ -1 +0,0 @@-749d44e9fe84566b4daa8898c62d8e88fd8e016fdiff --git a/solr/licenses/hadoop-minikdc-3.3.3.jar.sha1 b/solr/licenses/hadoop-minikdc-3.3.3.jar.sha1new file mode 100644index 00000000000..ff534b2c382--- /dev/null+++ b/solr/licenses/hadoop-minikdc-3.3.3.jar.sha1@@ -0,0 +1 @@+1871c97108af3689f82c619cfd92f01b38e4f47cdiff --git a/solr/modules/hadoop-auth/build.gradle b/solr/modules/hadoop-auth/build.gradleindex 5dd2233ecb7..a0672bf772f 100644--- a/solr/modules/hadoop-auth/build.gradle+++ b/solr/modules/hadoop-auth/build.gradle@@ -96,6 +96,7 @@ dependencies {testImplementation ('org.apache.hadoop:hadoop-minikdc', {exclude group:'org.apache.kerby', module:'kerby-xdr'exclude group:'org.apache.kerby', module:'token-provider'+ exclude group:'org.slf4j', module:'slf4j-reload4j'})// Zookeeper dependency - some tests like HdfsCloudBackupRestore need thisdiff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java b/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.javaindex dbbb2b9f29e..a960b8ed3d1 100644--- a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java+++ b/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java@@ -54,6 +54,7 @@ import java.nio.charset.StandardCharsets;import java.nio.file.AccessDeniedException;import java.nio.file.FileSystems;import java.nio.file.Files;+import java.nio.file.Paths;import java.util.ArrayList;import java.util.Enumeration;import java.util.List;@@ -888,10 +889,13 @@ public class FileUtil {private static void unTarUsingTar(File inFile, File untarDir,boolean gzipped) throws IOException {StringBuffer untarCommand = new StringBuffer();+ // not using canonical path here; this postpones relative path+ // resolution until bash is executed.+ final String source = "'" + FileUtil.makeSecureShellPath(inFile) + "'";if (gzipped) {- untarCommand.append(" gzip -dc '")- .append(FileUtil.makeSecureShellPath(inFile))- .append("' | (");+ untarCommand.append(" gzip -dc ")+ .append(source)+ .append(" | (");}untarCommand.append("cd '").append(FileUtil.makeSecureShellPath(untarDir))@@ -901,15 +905,17 @@ public class FileUtil {if (gzipped) {untarCommand.append(" -)");} else {- untarCommand.append(FileUtil.makeSecureShellPath(inFile));+ untarCommand.append(source);}+ LOG.debug("executing [{}]", untarCommand);String[] shellCmd = { "bash", "-c", untarCommand.toString() };ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);shexec.execute();int exitcode = shexec.getExitCode();if (exitcode != 0) {throw new IOException("Error untarring file " + inFile +- ". Tar process exited with exit code " + exitcode);+ ". Tar process exited with exit code " + exitcode+ + " from command " + untarCommand);}}@@ -966,6 +972,14 @@ public class FileUtil {+ " would create entry outside of " + outputDir);}+ if (entry.isSymbolicLink() || entry.isLink()) {+ String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);+ if (!canonicalTargetPath.startsWith(targetDirPath)) {+ throw new IOException(+ "expanding " + entry.getName() + " would create entry outside of " + outputDir);+ }+ }+if (entry.isDirectory()) {File subDir = new File(outputDir, entry.getName());if (!subDir.mkdirs() && !subDir.isDirectory()) {@@ -981,10 +995,12 @@ public class FileUtil {}if (entry.isSymbolicLink()) {- // Create symbolic link relative to tar parent dir- Files.createSymbolicLink(FileSystems.getDefault()- .getPath(outputDir.getPath(), entry.getName()),- FileSystems.getDefault().getPath(entry.getLinkName()));+ // Create symlink with canonical target path to ensure that we don't extract+ // outside targetDirPath+ String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);+ Files.createSymbolicLink(+ FileSystems.getDefault().getPath(outputDir.getPath(), entry.getName()),+ FileSystems.getDefault().getPath(canonicalTargetPath));return;}@@ -996,7 +1012,8 @@ public class FileUtil {}if (entry.isLink()) {- File src = new File(outputDir, entry.getLinkName());+ String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);+ File src = new File(canonicalTargetPath);HardLink.createHardLink(src, outputFile);return;}@@ -1004,6 +1021,20 @@ public class FileUtil {org.apache.commons.io.FileUtils.copyToFile(tis, outputFile);}+ /**+ * Gets the canonical path for the given path.+ *+ * @param path The path for which the canonical path needs to be computed.+ * @param parentDir The parent directory to use if the path is a relative path.+ * @return The canonical path of the given path.+ */+ private static String getCanonicalPath(String path, File parentDir) throws IOException {+ java.nio.file.Path targetPath = Paths.get(path);+ return (targetPath.isAbsolute() ?+ new File(path) :+ new File(parentDir, path)).getCanonicalPath();+ }+/*** Class for creating hardlinks.* Supports Unix, WindXP.diff --git a/versions.lock b/versions.lockindex ec3866498d7..ac0c1ddcbab 100644--- a/versions.lock+++ b/versions.lock@@ -129,11 +129,11 @@ org.apache.commons:commons-text:1.8 (2 constraints: fe1528d0)org.apache.curator:curator-client:4.3.0 (2 constraints: e214cba2)org.apache.curator:curator-framework:4.3.0 (2 constraints: ff13b474)org.apache.curator:curator-recipes:4.3.0 (1 constraints: 09050836)-org.apache.hadoop:hadoop-annotations:3.3.2 (1 constraints: 0a050536)-org.apache.hadoop:hadoop-auth:3.3.2 (1 constraints: 0a050536)-org.apache.hadoop:hadoop-client-api:3.3.2 (3 constraints: 1928ac5e)-org.apache.hadoop:hadoop-client-runtime:3.3.2 (2 constraints: 67170443)-org.apache.hadoop:hadoop-common:3.3.2 (1 constraints: 0a050536)+org.apache.hadoop:hadoop-annotations:3.3.3 (1 constraints: 0b050636)+org.apache.hadoop:hadoop-auth:3.3.3 (1 constraints: 0b050636)+org.apache.hadoop:hadoop-client-api:3.3.3 (3 constraints: 1c28435f)+org.apache.hadoop:hadoop-client-runtime:3.3.3 (2 constraints: 69173a43)+org.apache.hadoop:hadoop-common:3.3.3 (1 constraints: 0b050636)org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 (1 constraints: 0505f435)org.apache.httpcomponents:httpclient:4.5.13 (10 constraints: d1907cfc)org.apache.httpcomponents:httpcore:4.4.15 (9 constraints: b87d9f84)@@ -265,7 +265,7 @@ org.quicktheories:quicktheories:0.26 (1 constraints: dc04f530)org.reactivestreams:reactive-streams:1.0.3 (3 constraints: 3c2b02fd)org.slf4j:jcl-over-slf4j:1.7.36 (3 constraints: 05188eb8)org.slf4j:jul-to-slf4j:1.7.36 (3 constraints: 5928c263)-org.slf4j:slf4j-api:1.7.36 (38 constraints: 9dd389ac)+org.slf4j:slf4j-api:1.7.36 (38 constraints: a3d39db9)org.tallison:isoparser:1.9.41.7 (1 constraints: fb0c5528)org.tallison:jmatio:1.5 (1 constraints: ff0b57e9)org.tallison:metadata-extractor:2.15.0.1 (1 constraints: ee0c2f28)@@ -330,9 +330,9 @@ net.bytebuddy:byte-buddy:1.10.20 (2 constraints: 7c10a9d0)net.minidev:accessors-smart:2.4.7 (1 constraints: 4e0a90b8)net.minidev:json-smart:2.4.7 (1 constraints: 160e936e)no.nav.security:mock-oauth2-server:0.4.3 (1 constraints: 0905fa35)-org.apache.hadoop:hadoop-client-minicluster:3.3.2 (1 constraints: 0a050536)-org.apache.hadoop:hadoop-hdfs:3.3.2 (1 constraints: 0a050536)-org.apache.hadoop:hadoop-minikdc:3.3.2 (1 constraints: 0a050536)+org.apache.hadoop:hadoop-client-minicluster:3.3.3 (1 constraints: 0b050636)+org.apache.hadoop:hadoop-hdfs:3.3.3 (1 constraints: 0b050636)+org.apache.hadoop:hadoop-minikdc:3.3.3 (1 constraints: 0b050636)org.apache.kerby:kerb-admin:1.0.1 (1 constraints: 840d892f)org.apache.kerby:kerb-client:1.0.1 (1 constraints: 840d892f)org.apache.kerby:kerb-common:1.0.1 (2 constraints: a51841ca)diff --git a/versions.props b/versions.propsindex ecf354b9477..3a4a1741d04 100644--- a/versions.props+++ b/versions.props@@ -42,7 +42,7 @@ org.apache.commons:commons-math3=3.6.1org.apache.commons:commons-text=1.8org.apache.curator:*=4.3.0org.apache.hadoop.thirdparty:*=1.1.1-org.apache.hadoop:*=3.3.2+org.apache.hadoop:*=3.3.3org.apache.httpcomponents:httpclient=4.5.13org.apache.httpcomponents:httpcore=4.4.15org.apache.httpcomponents:httpmime=4.5.13