Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-17609: Remove HDFS module (#2923)

Eric Pugh1e77bdf062db
diff --git a/.github/labeler.yml b/.github/labeler.yml
index 11c43e84f84..beb19670e48 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -125,11 +125,6 @@ module:gcs-repository:
- any-glob-to-any-file:
- solr/modules/gcs-repository/**
-module:hdfs:
- - changed-files:
- - any-glob-to-any-file:
- - solr/modules/hdfs/**
-
module:jwt-auth:
- changed-files:
- any-glob-to-any-file:
@@ -197,9 +192,7 @@ cat:index:
- changed-files:
- any-glob-to-any-file:
- '**/org/apache/solr/index/**'
- - '**/org/apache/solr/hdfs/index/**'
- '**/org/apache/solr/update/**'
- - '**/org/apache/solr/hdfs/update/**'
- '**/org/apache/solr/scripting/update/**'
- '**/org/apache/solr/handler/loader/**'
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 70ce5cc71c2..209cb50fcd1 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -32,8 +32,6 @@ apache-commons-math3 = "3.6.1"
# @keep for version alignment
apache-commons-text = "1.12.0"
apache-curator = "5.7.1"
-apache-hadoop = "3.4.0"
-apache-hadoop-thirdparty = "1.2.0"
apache-httpcomponents-httpclient = "4.5.14"
apache-httpcomponents-httpcore = "4.4.16"
apache-httpcomponents-httpmime = "4.5.14"
@@ -243,11 +241,6 @@ apache-curator-client = { module = "org.apache.curator:curator-client", version.
apache-curator-framework = { module = "org.apache.curator:curator-framework", version.ref = "apache-curator" }
apache-curator-recipes = { module = "org.apache.curator:curator-recipes", version.ref = "apache-curator" }
apache-curator-test = { module = "org.apache.curator:curator-test", version.ref = "apache-curator" }
-apache-hadoop-client-api = { module = "org.apache.hadoop:hadoop-client-api", version.ref = "apache-hadoop" }
-apache-hadoop-client-minicluster = { module = "org.apache.hadoop:hadoop-client-minicluster", version.ref = "apache-hadoop" }
-apache-hadoop-client-runtime = { module = "org.apache.hadoop:hadoop-client-runtime", version.ref = "apache-hadoop" }
-apache-hadoop-hdfs = { module = "org.apache.hadoop:hadoop-hdfs", version.ref = "apache-hadoop" }
-apache-hadoop-thirdparty-shadedguava = { module = "org.apache.hadoop.thirdparty:hadoop-shaded-guava", version.ref = "apache-hadoop-thirdparty" }
apache-httpcomponents-httpclient = { module = "org.apache.httpcomponents:httpclient", version.ref = "apache-httpcomponents-httpclient" }
apache-httpcomponents-httpcore = { module = "org.apache.httpcomponents:httpcore", version.ref = "apache-httpcomponents-httpcore" }
apache-httpcomponents-httpmime = { module = "org.apache.httpcomponents:httpmime", version.ref = "apache-httpcomponents-httpmime" }
diff --git a/gradle/solr/solr-forbidden-apis.gradle b/gradle/solr/solr-forbidden-apis.gradle
index bb4dc601ee8..f7bdb5dd4da 100644
--- a/gradle/solr/solr-forbidden-apis.gradle
+++ b/gradle/solr/solr-forbidden-apis.gradle
@@ -21,6 +21,5 @@
configure(project(":solr:core")) {
tasks.matching { it.name == "forbiddenApisMain" || it.name == "forbiddenApisTest" }.all {
exclude "org/apache/solr/internal/**"
- exclude "org/apache/hadoop/**"
}
-}
\ No newline at end of file
+}
diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy
index 4d61f7985c0..56f97971d20 100644
--- a/gradle/testing/randomization/policies/solr-tests.policy
+++ b/gradle/testing/randomization/policies/solr-tests.policy
@@ -103,16 +103,8 @@ grant {
permission java.lang.RuntimePermission "closeClassLoader";
// needed by HttpSolrClient
permission java.lang.RuntimePermission "getFileSystemAttributes";
- // needed by hadoop hdfs (TODO: there is a cleaner way to handle this)
- permission java.lang.RuntimePermission "loadLibrary.jaas";
- permission java.lang.RuntimePermission "loadLibrary.jaas_unix";
- permission java.lang.RuntimePermission "loadLibrary.jaas_nt";
- // needed by hadoop common RawLocalFileSystem for java nio getOwner
- permission java.lang.RuntimePermission "accessUserInformation";
- // needed by hadoop hdfs
- permission java.lang.RuntimePermission "readFileDescriptor";
- permission java.lang.RuntimePermission "writeFileDescriptor";
- // needed by hadoop http
+
+ // needed by mockito http
permission java.lang.RuntimePermission "getProtectionDomain";
// SolrProcessMgr to list processes
permission java.lang.RuntimePermission "manageProcess";
@@ -139,20 +131,11 @@ grant {
permission javax.management.MBeanServerPermission "releaseMBeanServer";
permission javax.management.MBeanTrustPermission "register";
- // needed by hadoop hdfs
- permission javax.security.auth.AuthPermission "getSubject";
- permission javax.security.auth.AuthPermission "modifyPrincipals";
- permission javax.security.auth.AuthPermission "doAs";
- permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
- permission javax.security.auth.AuthPermission "modifyPublicCredentials";
- permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
-
// needed by crossdc
permission javax.security.auth.AuthPermission "getLoginConfiguration";
permission javax.security.auth.AuthPermission "setLoginConfiguration";
- // needed by hadoop security
- permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
+ // needed by benchmark
permission java.security.SecurityPermission "insertProvider";
// Needed by JWT integration tests & S3 tests
@@ -219,10 +202,7 @@ grant {
// additional permissions based on system properties set by /bin/solr
// NOTE: if the property is not set, the permission entry is ignored.
-grant {
- permission java.io.FilePermission "${hadoop.security.credential.provider.path}", "read,write,delete,readlink";
- permission java.io.FilePermission "${hadoop.security.credential.provider.path}${/}-", "read,write,delete,readlink";
-
+grant {
permission java.io.FilePermission "${solr.jetty.keystore}", "read,write,delete,readlink";
permission java.io.FilePermission "${solr.jetty.keystore}${/}-", "read,write,delete,readlink";
diff --git a/gradle/validation/forbidden-apis.gradle b/gradle/validation/forbidden-apis.gradle
index 2a09ff49dda..90f385755c8 100644
--- a/gradle/validation/forbidden-apis.gradle
+++ b/gradle/validation/forbidden-apis.gradle
@@ -130,11 +130,6 @@ allprojects { prj ->
forbiddenApisMain.exclude("**/jmh_generated/**")
}
- // Disable checks on code copied from Hadoop
- if (prj.path == ":solr:modules:hdfs") {
- forbiddenApisTest.exclude("**/org/apache/hadoop/**")
- }
-
// We rely on resolved configurations to compute the relevant set of rule
// files for forbiddenApis. Since we don't want to resolve these configurations until
// the task is executed, we can't really use them as task inputs properly. This is a
diff --git a/gradle/validation/owasp-dependency-check/exclusions.xml b/gradle/validation/owasp-dependency-check/exclusions.xml
index b56304ec69e..9e7d6e91226 100644
--- a/gradle/validation/owasp-dependency-check/exclusions.xml
+++ b/gradle/validation/owasp-dependency-check/exclusions.xml
@@ -76,14 +76,4 @@
<cpe>cpe:/a:eclipse:jetty</cpe>
<cpe>cpe:/a:jetty:jetty</cpe>
</suppress>
- <suppress>
- <notes><![CDATA[hadoop-shaded-guava-1.1.1.jar is not Apache Hadoop 1.1.1]]></notes>
- <packageUrl regex="true">^pkg:maven/org\.apache\.hadoop\.thirdparty/hadoop\-shaded\-guava@.*$</packageUrl>
- <cpe>cpe:/a:apache:hadoop:1.1.1</cpe>
- </suppress>
- <suppress>
- <notes><![CDATA[hadoop-client-runtime-3.*.jar is not Apache Hadoop 1.1.1]]></notes>
- <packageUrl regex="true">^pkg:maven/org\.apache\.hadoop/hadoop\-client\-runtime@3.*$</packageUrl>
- <cpe>cpe:/a:apache:hadoop:1.1.1</cpe>
- </suppress>
</suppressions>
diff --git a/gradle/validation/rat-sources.gradle b/gradle/validation/rat-sources.gradle
index 5b600f956c0..37dde589a52 100644
--- a/gradle/validation/rat-sources.gradle
+++ b/gradle/validation/rat-sources.gradle
@@ -106,12 +106,6 @@ allprojects {
exclude "src/test-files/META-INF/services/*"
break
- case ":solr:modules:hdfs":
- exclude "src/test-files/**/*.aff"
- exclude "src/test-files/**/*.dic"
- exclude "src/test-files/**/*.incl"
- break
-
case ":solr:modules:langid":
exclude "**/langdetect-profiles/*"
break
diff --git a/gradle/validation/spotless.gradle b/gradle/validation/spotless.gradle
index 32234c93a4a..52b95c410cc 100644
--- a/gradle/validation/spotless.gradle
+++ b/gradle/validation/spotless.gradle
@@ -65,10 +65,6 @@ configure(allprojects) { prj ->
"src/java/org/apache/solr/parser/Token.java",
"src/java/org/apache/solr/parser/TokenMgrError.java"
break
- case ":solr:modules:hdfs":
- // Exclude Hadoop copied files to make upgrades easier
- targetExclude "src/**/org/apache/hadoop/**"
- break
case ":solr:solr-ref-guide":
// Apply to all example Java sources in the ref-guide
target "modules/**/examples/*.java"
diff --git a/settings.gradle b/settings.gradle
index cf4acf46a84..14b916902c6 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -49,7 +49,6 @@ include "solr:modules:cross-dc"
include "solr:modules:opentelemetry"
include "solr:modules:extraction"
include "solr:modules:gcs-repository"
-include "solr:modules:hdfs"
include "solr:modules:jwt-auth"
include "solr:modules:langid"
include "solr:modules:llm"
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 277691d7ad5..3cdc85bc35e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -105,6 +105,8 @@ Deprecation Removals
* SOLR-17540: Removed the Hadoop Auth module, and thus Kerberos authentication and other exotic options. (Eric Pugh)
+* SOLR-17609: Removd the HDFS module and therefore support for storing indexes in HDFS. Existing "blockcache" code preserved by migrating it to core o.a.s.blackcache package.
+
Dependency Upgrades
---------------------
(No changes)
diff --git a/solr/benchmark/log4j2-bench.xml b/solr/benchmark/log4j2-bench.xml
index eb961d4dc0e..628bfbcac64 100644
--- a/solr/benchmark/log4j2-bench.xml
+++ b/solr/benchmark/log4j2-bench.xml
@@ -83,7 +83,6 @@
</Appenders>
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
- <AsyncLogger name="org.apache.hadoop" level="warn"/>
<AsyncLogger name="org.apache.solr.update.LoggingInfoStream" level="off"/>
<AsyncLogger name="org.apache.zookeeper" level="warn"/>
<!-- HttpSolrCall adds markers denoting the handler class to allow fine grained control, metrics are
@@ -104,4 +103,3 @@
</AsyncRoot>
</Loggers>
</Configuration>
-
diff --git a/solr/benchmark/src/test-files/log4j2.xml b/solr/benchmark/src/test-files/log4j2.xml
index e54e11b474b..f65aca5e456 100644
--- a/solr/benchmark/src/test-files/log4j2.xml
+++ b/solr/benchmark/src/test-files/log4j2.xml
@@ -77,7 +77,6 @@
</Appenders>
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
- <AsyncLogger name="org.apache.hadoop" level="warn"/>
<AsyncLogger name="org.apache.solr.update.LoggingInfoStream" level="off"/>
<AsyncLogger name="org.apache.zookeeper" level="warn"/>
<!-- HttpSolrCall adds markers denoting the handler class to allow fine grained control, metrics are
@@ -99,4 +98,3 @@
</AsyncRoot>
</Loggers>
</Configuration>
-
diff --git a/solr/bin/solr b/solr/bin/solr
index e8d3b005248..09784892f90 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -202,10 +202,6 @@ SOLR_URL_SCHEME=http
SOLR_JETTY_CONFIG=()
SOLR_SSL_OPTS=""
-if [ -n "${SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH:-}" ]; then
- SOLR_SSL_OPTS+=" -Dhadoop.security.credential.provider.path=$SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH"
-fi
-
if [ -z "${SOLR_SSL_ENABLED:-}" ]; then
if [ -n "${SOLR_SSL_KEY_STORE:-}" ]; then
SOLR_SSL_ENABLED="true" # implicitly from earlier behaviour
diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd
index 435a49e8125..3b971b68dfc 100755
--- a/solr/bin/solr.cmd
+++ b/solr/bin/solr.cmd
@@ -78,10 +78,6 @@ set SOLR_URL_SCHEME=http
set "SOLR_JETTY_CONFIG=--module=http"
set "SOLR_SSL_OPTS= "
-IF DEFINED SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH (
- set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dhadoop.security.credential.provider.path=%SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH%"
-)
-
IF NOT DEFINED SOLR_SSL_ENABLED (
IF DEFINED SOLR_SSL_KEY_STORE (
set "SOLR_SSL_ENABLED=true"
diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd
index 8fa3db7aebf..60d8aaeac0a 100755
--- a/solr/bin/solr.in.cmd
+++ b/solr/bin/solr.in.cmd
@@ -171,18 +171,6 @@ REM set SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
REM set SOLR_SSL_CLIENT_KEY_STORE_TYPE=
REM set SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
-REM Sets path of Hadoop credential provider (hadoop.security.credential.provider.path property) and
-REM enables usage of credential store.
-REM Credential provider should store the following keys:
-REM * solr.jetty.keystore.password
-REM * solr.jetty.truststore.password
-REM Set the two below if you want to set specific store passwords for HTTP client
-REM * javax.net.ssl.keyStorePassword
-REM * javax.net.ssl.trustStorePassword
-REM More info: https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html
-REM set SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
-REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.ssl.credential.provider.chain=hadoop
-
REM Settings for authentication
REM Please configure only one of SOLR_AUTHENTICATION_CLIENT_BUILDER or SOLR_AUTH_TYPE parameters
REM set SOLR_AUTHENTICATION_CLIENT_BUILDER=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory
@@ -230,7 +218,7 @@ REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.sharedLib=/path/to/lib
REM Runs solr in a java security manager sandbox. This can protect against some attacks.
REM Runtime properties are passed to the security policy file (server\etc\security.policy)
REM You can also tweak via standard JDK files such as ~\.java.policy, see https://s.apache.org/java8policy
-REM This is experimental! It may not work at all with Hadoop/HDFS features.
+REM This is experimental!
REM set SOLR_SECURITY_MANAGER_ENABLED=true
REM This variable provides you with the option to disable the Admin UI. if you uncomment the variable below and
diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh
index 46f541f056a..4692343d85f 100644
--- a/solr/bin/solr.in.sh
+++ b/solr/bin/solr.in.sh
@@ -187,18 +187,6 @@
#SOLR_SSL_CLIENT_KEY_STORE_TYPE=
#SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
-# Sets path of Hadoop credential provider (hadoop.security.credential.provider.path property) and
-# enables usage of credential store.
-# Credential provider should store the following keys:
-# * solr.jetty.keystore.password
-# * solr.jetty.truststore.password
-# Set the two below if you want to set specific store passwords for HTTP client
-# * javax.net.ssl.keyStorePassword
-# * javax.net.ssl.trustStorePassword
-# More info: https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html
-#SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
-#SOLR_OPTS=" -Dsolr.ssl.credential.provider.chain=hadoop"
-
# Settings for authentication
# Please configure only one of SOLR_AUTHENTICATION_CLIENT_BUILDER or SOLR_AUTH_TYPE parameters
#SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"
@@ -256,7 +244,7 @@
# Runs solr in java security manager sandbox. This can protect against some attacks.
# Runtime properties are passed to the security policy file (server/etc/security.policy)
# You can also tweak via standard JDK files such as ~/.java.policy, see https://s.apache.org/java8policy
-# This is experimental! It may not work at all with Hadoop/HDFS features.
+# This is experimental!
#SOLR_SECURITY_MANAGER_ENABLED=true
# This variable provides you with the option to disable the Admin UI. If you uncomment the variable below and
@@ -299,6 +287,6 @@
# See https://solr.apache.org/guide/solr/latest/configuration-guide/replica-placement-plugins.html for details
#SOLR_PLACEMENTPLUGIN_DEFAULT=simple
-# Solr internally doesn't use cookies other than for modules such as Kerberos/Hadoop Auth. If you don't need any of those
-# And you don't need them for an external system (such as a load balancer), you can disable the use of a CookieStore with:
+# Solr internally doesn't use cookies. If you don't need any of those, and you don't
+# need them for an external system (such as a load balancer), you can disable the use of a CookieStore with:
# SOLR_OPTS="$SOLR_OPTS -Dsolr.http.disableCookies=true"
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCache.java b/solr/core/src/java/org/apache/solr/blockcache/BlockCache.java
similarity index 99%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCache.java
rename to solr/core/src/java/org/apache/solr/blockcache/BlockCache.java
index 1cf033e2acc..e5a8e7c3449 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCache.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/BlockCache.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCacheKey.java b/solr/core/src/java/org/apache/solr/blockcache/BlockCacheKey.java
similarity index 97%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCacheKey.java
rename to solr/core/src/java/org/apache/solr/blockcache/BlockCacheKey.java
index be6e2daa4ee..d96f23c2df2 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCacheKey.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/BlockCacheKey.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import java.util.Objects;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCacheLocation.java b/solr/core/src/java/org/apache/solr/blockcache/BlockCacheLocation.java
similarity index 97%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCacheLocation.java
rename to solr/core/src/java/org/apache/solr/blockcache/BlockCacheLocation.java
index a7f8ad5fa3d..df73cd14e8d 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCacheLocation.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/BlockCacheLocation.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.solr.common.util.SuppressForbidden;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockDirectory.java b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java
similarity index 98%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockDirectory.java
rename to solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java
index 75859bbdd97..b9cff0bbcc7 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockDirectory.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import java.io.File;
import java.io.FileNotFoundException;
@@ -29,7 +29,6 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.solr.core.ShutdownAwareDirectory;
-import org.apache.solr.hdfs.store.HdfsDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -296,8 +295,6 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
throw new FileNotFoundException("File [" + name + "] not found");
}
return file.lastModified();
- } else if (in instanceof HdfsDirectory) {
- return ((HdfsDirectory) in).fileModified(name);
} else {
throw new UnsupportedOperationException();
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockDirectoryCache.java b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectoryCache.java
similarity index 97%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockDirectoryCache.java
rename to solr/core/src/java/org/apache/solr/blockcache/BlockDirectoryCache.java
index 85a1cf48d6c..7deebcf11e0 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockDirectoryCache.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectoryCache.java
@@ -14,14 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.solr.hdfs.store.blockcache.BlockCache.OnRelease;
+import org.apache.solr.blockcache.BlockCache.OnRelease;
/**
* @lucene.experimental
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockLocks.java b/solr/core/src/java/org/apache/solr/blockcache/BlockLocks.java
similarity index 98%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockLocks.java
rename to solr/core/src/java/org/apache/solr/blockcache/BlockLocks.java
index ced5460d3b0..bf119b2aa6c 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockLocks.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/BlockLocks.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import java.util.concurrent.atomic.AtomicLongArray;
import org.apache.lucene.util.LongBitSet;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BufferStore.java b/solr/core/src/java/org/apache/solr/blockcache/BufferStore.java
similarity index 98%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BufferStore.java
rename to solr/core/src/java/org/apache/solr/blockcache/BufferStore.java
index 37a384c9f19..5f661fadaea 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BufferStore.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/BufferStore.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/Cache.java b/solr/core/src/java/org/apache/solr/blockcache/Cache.java
similarity index 97%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/Cache.java
rename to solr/core/src/java/org/apache/solr/blockcache/Cache.java
index c04f6ee1f11..bc814ce9601 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/Cache.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/Cache.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
/**
* @lucene.experimental
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/CachedIndexOutput.java b/solr/core/src/java/org/apache/solr/blockcache/CachedIndexOutput.java
similarity index 98%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/CachedIndexOutput.java
rename to solr/core/src/java/org/apache/solr/blockcache/CachedIndexOutput.java
index a78f6c5d455..a06b26e49da 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/CachedIndexOutput.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/CachedIndexOutput.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import java.io.IOException;
import org.apache.lucene.store.IndexOutput;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/CustomBufferedIndexInput.java b/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java
similarity index 99%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/CustomBufferedIndexInput.java
rename to solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java
index b758dad0033..79d44d0fc6b 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/CustomBufferedIndexInput.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import java.io.EOFException;
import java.io.IOException;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/Metrics.java b/solr/core/src/java/org/apache/solr/blockcache/Metrics.java
similarity index 99%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/Metrics.java
rename to solr/core/src/java/org/apache/solr/blockcache/Metrics.java
index 296ea06dbc7..d7e13b9132f 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/Metrics.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/Metrics.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/ReusedBufferedIndexOutput.java b/solr/core/src/java/org/apache/solr/blockcache/ReusedBufferedIndexOutput.java
similarity index 99%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/ReusedBufferedIndexOutput.java
rename to solr/core/src/java/org/apache/solr/blockcache/ReusedBufferedIndexOutput.java
index 5169b0eb246..7b22b8b80ec 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/ReusedBufferedIndexOutput.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/ReusedBufferedIndexOutput.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import java.io.IOException;
import org.apache.lucene.store.IndexOutput;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/Store.java b/solr/core/src/java/org/apache/solr/blockcache/Store.java
similarity index 95%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/Store.java
rename to solr/core/src/java/org/apache/solr/blockcache/Store.java
index ceb8441ea80..e527a7dd06a 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/Store.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/Store.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
/**
* @lucene.experimental
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/package-info.java b/solr/core/src/java/org/apache/solr/blockcache/package-info.java
similarity index 86%
rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/package-info.java
rename to solr/core/src/java/org/apache/solr/blockcache/package-info.java
index 96f3d82984a..fdd7f428c1c 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/package-info.java
+++ b/solr/core/src/java/org/apache/solr/blockcache/package-info.java
@@ -15,5 +15,5 @@
* limitations under the License.
*/
-/** HDFS implementation. */
-package org.apache.solr.hdfs;
+/** A generic Directory layer/wrapper that caches data, on or off heap as desired. */
+package org.apache.solr.blockcache;
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 6d1065a0e54..ffd9bc7c8d2 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -107,10 +107,7 @@ public class SolrResourceLoader
"security.",
"handler.admin.",
"security.jwt.",
- "security.hadoop.",
"handler.sql.",
- "hdfs.",
- "hdfs.update.",
"crossdc.handler.",
"crossdc.update.processor."
};
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
index 9b56a39a52f..aa008353763 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
@@ -20,14 +20,7 @@
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}">
- <!-- used to keep RAM reqs down for HdfsDirectoryFactory -->
- <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
- <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:1024}</int>
- <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
- <str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
- <str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:false}</str>
- </directoryFactory>
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>
<dataDir>${solr.data.dir:}</dataDir>
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml
index 5e366166ea1..c9df6426693 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml
@@ -21,13 +21,7 @@
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}">
- <!-- used to keep RAM reqs down for HdfsDirectoryFactory -->
- <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
- <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:1024}</int>
- <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
- <str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
- </directoryFactory>
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>
<schemaFactory class="ClassicIndexSchemaFactory"/>
<dataDir>${solr.data.dir:}</dataDir>
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml
index 0d32632fc94..bdeedb6bcc0 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml
@@ -43,12 +43,6 @@
<double name="maxWriteMBPerSecFlush">2000000</double>
<double name="maxWriteMBPerSecMerge">3000000</double>
<double name="maxWriteMBPerSecRead">4000000</double>
- <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
- <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
- <bool name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</bool>
- <bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
- <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:10}</int>
- <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
</directoryFactory>
<schemaFactory class="ClassicIndexSchemaFactory"/>
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
index aff143c616f..8c29b8411ca 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
@@ -21,14 +21,7 @@
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}">
- <!-- used to keep RAM reqs down for HdfsDirectoryFactory -->
- <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
- <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:1024}</int>
- <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
- <str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
- <str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</str>
- </directoryFactory>
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>
<schemaFactory class="ClassicIndexSchemaFactory"/>
<requestDispatcher>
<requestParsers />
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
index 856079f329e..0e191aa3d41 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
@@ -42,13 +42,7 @@
<double name="maxWriteMBPerSecDefault">1000000</double>
<double name="maxWriteMBPerSecFlush">2000000</double>
<double name="maxWriteMBPerSecMerge">3000000</double>
- <double name="maxWriteMBPerSecRead">4000000</double>
- <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
- <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
- <bool name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</bool>
- <bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
- <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:10}</int>
- <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
+ <double name="maxWriteMBPerSecRead">4000000</double>
</directoryFactory>
<schemaFactory class="ClassicIndexSchemaFactory"/>
diff --git a/solr/core/src/test-files/solr/configsets/cloud-hdfs/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-hdfs/conf/schema.xml
deleted file mode 100644
index 53ca02dbde2..00000000000
--- a/solr/core/src/test-files/solr/configsets/cloud-hdfs/conf/schema.xml
+++ /dev/null
@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<schema name="minimal" version="1.7">
- <fieldType name="string" class="solr.StrField"/>
- <fieldType name="int" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
- <dynamicField name="*" type="string" indexed="true" stored="true"/>
- <!-- for versioning -->
- <field name="_version_" type="long" indexed="true" stored="true"/>
- <field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
- <field name="id" type="string" indexed="true" stored="true"/>
- <uniqueKey>id</uniqueKey>
-</schema>
diff --git a/solr/core/src/test-files/solr/configsets/cloud-hdfs/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-hdfs/conf/solrconfig.xml
deleted file mode 100644
index 39edc032c8b..00000000000
--- a/solr/core/src/test-files/solr/configsets/cloud-hdfs/conf/solrconfig.xml
+++ /dev/null
@@ -1,51 +0,0 @@
-<?xml version="1.0" ?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
-
-<config>
-
- <directoryFactory name="DirectoryFactory"
- class="solr.HdfsDirectoryFactory"/>
- <indexConfig>
- <lockType>hdfs</lockType>
- </indexConfig>
-
- <schemaFactory class="ClassicIndexSchemaFactory"/>
-
- <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
-
- <updateHandler class="solr.DirectUpdateHandler2">
- <commitWithin>
- <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
- </commitWithin>
- <updateLog>
- <str name="dir">${solr.ulog.dir:}</str>
- </updateLog>
- </updateHandler>
-
- <requestHandler name="/select" class="solr.SearchHandler">
- <lst name="defaults">
- <str name="echoParams">explicit</str>
- <str name="indent">true</str>
- <str name="df">text</str>
- </lst>
-
- </requestHandler>
-</config>
diff --git a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml
index 7e1f3434e96..79e7578d186 100644
--- a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml
@@ -22,14 +22,7 @@
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}">
- <!-- used to keep RAM reqs down for HdfsDirectoryFactory -->
- <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
- <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:1024}</int>
- <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
- <str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
- <str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</str>
- </directoryFactory>
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>
<schemaFactory class="ClassicIndexSchemaFactory"/>
<indexReaderFactory name="IndexReaderFactory"
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/blockcache/BlockCacheTest.java b/solr/core/src/test/org/apache/solr/blockcache/BlockCacheTest.java
similarity index 99%
rename from solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/blockcache/BlockCacheTest.java
rename to solr/core/src/test/org/apache/solr/blockcache/BlockCacheTest.java
index 6cd2cf35d63..b28c7ed0410 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/blockcache/BlockCacheTest.java
+++ b/solr/core/src/test/org/apache/solr/blockcache/BlockCacheTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/blockcache/BlockDirectoryTest.java b/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java
similarity index 99%
rename from solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/blockcache/BlockDirectoryTest.java
rename to solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java
index f9e0dba9ac0..e114d1175a7 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/blockcache/BlockDirectoryTest.java
+++ b/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.io.File;
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/blockcache/BufferStoreTest.java b/solr/core/src/test/org/apache/solr/blockcache/BufferStoreTest.java
similarity index 98%
rename from solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/blockcache/BufferStoreTest.java
rename to solr/core/src/test/org/apache/solr/blockcache/BufferStoreTest.java
index 02331904cf4..77d92378f1a 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/blockcache/BufferStoreTest.java
+++ b/solr/core/src/test/org/apache/solr/blockcache/BufferStoreTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.hdfs.store.blockcache;
+package org.apache.solr.blockcache;
import java.math.BigDecimal;
import java.util.Map;
diff --git a/solr/cross-dc-manager/conf/log4j2.xml b/solr/cross-dc-manager/conf/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/cross-dc-manager/conf/log4j2.xml
+++ b/solr/cross-dc-manager/conf/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/cross-dc-manager/src/test-files/log4j2.xml b/solr/cross-dc-manager/src/test-files/log4j2.xml
index 72ea357c3db..0f12a41ea10 100644
--- a/solr/cross-dc-manager/src/test-files/log4j2.xml
+++ b/solr/cross-dc-manager/src/test-files/log4j2.xml
@@ -37,9 +37,7 @@
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/docker/tests/cases/test_log4j/log4j2.xml b/solr/docker/tests/cases/test_log4j/log4j2.xml
index 5e319cc12be..f1958fc4eeb 100644
--- a/solr/docker/tests/cases/test_log4j/log4j2.xml
+++ b/solr/docker/tests/cases/test_log4j/log4j2.xml
@@ -73,7 +73,6 @@
</Appenders>
<Loggers>
- <Logger name="org.apache.hadoop" level="warn"/>
<Logger name="org.apache.solr.update.LoggingInfoStream" level="off"/>
<Logger name="org.apache.zookeeper" level="warn"/>
<Logger name="org.apache.solr.core.SolrCore.SlowRequest" level="info" additivity="false">
diff --git a/solr/licenses/hadoop-client-api-3.4.0.jar.sha1 b/solr/licenses/hadoop-client-api-3.4.0.jar.sha1
deleted file mode 100644
index 6f3d8ba0f89..00000000000
--- a/solr/licenses/hadoop-client-api-3.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-14b26fbacedff53bc10eebc1fddface4f1f655af
diff --git a/solr/licenses/hadoop-client-api-LICENSE-ASL.txt b/solr/licenses/hadoop-client-api-LICENSE-ASL.txt
deleted file mode 100644
index 9a8e847ee84..00000000000
--- a/solr/licenses/hadoop-client-api-LICENSE-ASL.txt
+++ /dev/null
@@ -1,244 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
-APACHE HADOOP SUBCOMPONENTS:
-
-The Apache Hadoop project contains subcomponents with separate copyright
-notices and license terms. Your use of the source code for the these
-subcomponents is subject to the terms and conditions of the following
-licenses.
-
-For the org.apache.hadoop.util.bloom.* classes:
-
-/**
- *
- * Copyright (c) 2005, European Commission project OneLab under contract
- * 034819 (http://www.one-lab.org)
- * All rights reserved.
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the distribution.
- * - Neither the name of the University Catholique de Louvain - UCL
- * nor the names of its contributors may be used to endorse or
- * promote products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
\ No newline at end of file
diff --git a/solr/licenses/hadoop-client-api-NOTICE.txt b/solr/licenses/hadoop-client-api-NOTICE.txt
deleted file mode 100644
index c56a5e4eac1..00000000000
--- a/solr/licenses/hadoop-client-api-NOTICE.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/solr/licenses/hadoop-client-minicluster-3.4.0.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.4.0.jar.sha1
deleted file mode 100644
index 22656693fa1..00000000000
--- a/solr/licenses/hadoop-client-minicluster-3.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a1af909b1cb220c0f81224b2241c4a0aa2860ee6
diff --git a/solr/licenses/hadoop-client-minicluster-LICENSE-ASL.txt b/solr/licenses/hadoop-client-minicluster-LICENSE-ASL.txt
deleted file mode 100644
index 9a8e847ee84..00000000000
--- a/solr/licenses/hadoop-client-minicluster-LICENSE-ASL.txt
+++ /dev/null
@@ -1,244 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
-APACHE HADOOP SUBCOMPONENTS:
-
-The Apache Hadoop project contains subcomponents with separate copyright
-notices and license terms. Your use of the source code for the these
-subcomponents is subject to the terms and conditions of the following
-licenses.
-
-For the org.apache.hadoop.util.bloom.* classes:
-
-/**
- *
- * Copyright (c) 2005, European Commission project OneLab under contract
- * 034819 (http://www.one-lab.org)
- * All rights reserved.
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the distribution.
- * - Neither the name of the University Catholique de Louvain - UCL
- * nor the names of its contributors may be used to endorse or
- * promote products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
\ No newline at end of file
diff --git a/solr/licenses/hadoop-client-minicluster-NOTICE.txt b/solr/licenses/hadoop-client-minicluster-NOTICE.txt
deleted file mode 100644
index c56a5e4eac1..00000000000
--- a/solr/licenses/hadoop-client-minicluster-NOTICE.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/solr/licenses/hadoop-client-runtime-3.4.0.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.4.0.jar.sha1
deleted file mode 100644
index 5b15d693af4..00000000000
--- a/solr/licenses/hadoop-client-runtime-3.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-de1ed42503ec64b84a91b330fdd2ed508b6f9df8
diff --git a/solr/licenses/hadoop-client-runtime-LICENSE-ASL.txt b/solr/licenses/hadoop-client-runtime-LICENSE-ASL.txt
deleted file mode 100644
index 9a8e847ee84..00000000000
--- a/solr/licenses/hadoop-client-runtime-LICENSE-ASL.txt
+++ /dev/null
@@ -1,244 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
-APACHE HADOOP SUBCOMPONENTS:
-
-The Apache Hadoop project contains subcomponents with separate copyright
-notices and license terms. Your use of the source code for the these
-subcomponents is subject to the terms and conditions of the following
-licenses.
-
-For the org.apache.hadoop.util.bloom.* classes:
-
-/**
- *
- * Copyright (c) 2005, European Commission project OneLab under contract
- * 034819 (http://www.one-lab.org)
- * All rights reserved.
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the distribution.
- * - Neither the name of the University Catholique de Louvain - UCL
- * nor the names of its contributors may be used to endorse or
- * promote products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
\ No newline at end of file
diff --git a/solr/licenses/hadoop-client-runtime-NOTICE.txt b/solr/licenses/hadoop-client-runtime-NOTICE.txt
deleted file mode 100644
index c56a5e4eac1..00000000000
--- a/solr/licenses/hadoop-client-runtime-NOTICE.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/solr/licenses/hadoop-hdfs-3.4.0-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.4.0-tests.jar.sha1
deleted file mode 100644
index ef9038e9433..00000000000
--- a/solr/licenses/hadoop-hdfs-3.4.0-tests.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-18f9797a908899efbe4e27f6d5b76420f446a695
diff --git a/solr/licenses/hadoop-hdfs-3.4.0.jar.sha1 b/solr/licenses/hadoop-hdfs-3.4.0.jar.sha1
deleted file mode 100644
index cf6f21b9017..00000000000
--- a/solr/licenses/hadoop-hdfs-3.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d46983ccf965bffaeab8b67713aa9442c5b7d5bf
diff --git a/solr/licenses/hadoop-hdfs-LICENSE-ASL.txt b/solr/licenses/hadoop-hdfs-LICENSE-ASL.txt
deleted file mode 100644
index 9a8e847ee84..00000000000
--- a/solr/licenses/hadoop-hdfs-LICENSE-ASL.txt
+++ /dev/null
@@ -1,244 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
-APACHE HADOOP SUBCOMPONENTS:
-
-The Apache Hadoop project contains subcomponents with separate copyright
-notices and license terms. Your use of the source code for the these
-subcomponents is subject to the terms and conditions of the following
-licenses.
-
-For the org.apache.hadoop.util.bloom.* classes:
-
-/**
- *
- * Copyright (c) 2005, European Commission project OneLab under contract
- * 034819 (http://www.one-lab.org)
- * All rights reserved.
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the distribution.
- * - Neither the name of the University Catholique de Louvain - UCL
- * nor the names of its contributors may be used to endorse or
- * promote products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
\ No newline at end of file
diff --git a/solr/licenses/hadoop-hdfs-NOTICE.txt b/solr/licenses/hadoop-hdfs-NOTICE.txt
deleted file mode 100644
index c56a5e4eac1..00000000000
--- a/solr/licenses/hadoop-hdfs-NOTICE.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/solr/licenses/hadoop-shaded-guava-1.2.0.jar.sha1 b/solr/licenses/hadoop-shaded-guava-1.2.0.jar.sha1
deleted file mode 100644
index 2e3ae98fd2c..00000000000
--- a/solr/licenses/hadoop-shaded-guava-1.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a0a6e7aa87c838ea418fc36ed290987dd90f5f6c
diff --git a/solr/licenses/hadoop-shaded-guava-LICENSE-ASL.txt b/solr/licenses/hadoop-shaded-guava-LICENSE-ASL.txt
deleted file mode 100644
index 9a8e847ee84..00000000000
--- a/solr/licenses/hadoop-shaded-guava-LICENSE-ASL.txt
+++ /dev/null
@@ -1,244 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
-APACHE HADOOP SUBCOMPONENTS:
-
-The Apache Hadoop project contains subcomponents with separate copyright
-notices and license terms. Your use of the source code for the these
-subcomponents is subject to the terms and conditions of the following
-licenses.
-
-For the org.apache.hadoop.util.bloom.* classes:
-
-/**
- *
- * Copyright (c) 2005, European Commission project OneLab under contract
- * 034819 (http://www.one-lab.org)
- * All rights reserved.
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the distribution.
- * - Neither the name of the University Catholique de Louvain - UCL
- * nor the names of its contributors may be used to endorse or
- * promote products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
\ No newline at end of file
diff --git a/solr/licenses/hadoop-shaded-guava-NOTICE.txt b/solr/licenses/hadoop-shaded-guava-NOTICE.txt
deleted file mode 100644
index c56a5e4eac1..00000000000
--- a/solr/licenses/hadoop-shaded-guava-NOTICE.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/solr/modules/analysis-extras/src/test-files/log4j2.xml b/solr/modules/analysis-extras/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/modules/analysis-extras/src/test-files/log4j2.xml
+++ b/solr/modules/analysis-extras/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/modules/clustering/src/test-files/log4j2.xml b/solr/modules/clustering/src/test-files/log4j2.xml
index 5e696935965..f668c06a1ed 100644
--- a/solr/modules/clustering/src/test-files/log4j2.xml
+++ b/solr/modules/clustering/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
@@ -40,4 +38,3 @@
</Root>
</Loggers>
</Configuration>
-
diff --git a/solr/modules/cross-dc/src/test-files/log4j2.xml b/solr/modules/cross-dc/src/test-files/log4j2.xml
index 7b40b882f19..6ba5e7c96aa 100644
--- a/solr/modules/cross-dc/src/test-files/log4j2.xml
+++ b/solr/modules/cross-dc/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Logger name="org.apache.solr.crossdc.update.processor.KafkaRequestMirroringHandler" level="TRACE"/>
diff --git a/solr/modules/extraction/src/test-files/log4j2.xml b/solr/modules/extraction/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/modules/extraction/src/test-files/log4j2.xml
+++ b/solr/modules/extraction/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/modules/hdfs/README.md b/solr/modules/hdfs/README.md
deleted file mode 100644
index ba0cac71cad..00000000000
--- a/solr/modules/hdfs/README.md
+++ /dev/null
@@ -1,46 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-Apache Solr HDFS Module
-===============================
-
-Introduction
-------------
-This module implements the support for Hadoop Distributed File System in Apache Solr.
-
-Building
---------
-The HDFS module uses the same Gradle build as the core Solr components.
-
-To build the module, you can use
-
-```
-./gradlew :solr:modules:hdfs:assemble
-```
-
-The resulting module will be placed to the libs directory, for example:
-`solr/modules/hdfs/build/libs/solr-hdfs-9.0.0-SNAPSHOT.jar`
-
-To execute the module tests:
-
-```
-./gradlew :solr:modules:hdfs:test
-```
-
-Usage
------
-Please refer to the 'Running Solr on HDFS' section of the reference guide: https://solr.apache.org/guide/solr/latest/deployment-guide/solr-on-hdfs.html
diff --git a/solr/modules/hdfs/bin/prepare-snapshot-export.sh b/solr/modules/hdfs/bin/prepare-snapshot-export.sh
deleted file mode 100755
index c74aaf7c627..00000000000
--- a/solr/modules/hdfs/bin/prepare-snapshot-export.sh
+++ /dev/null
@@ -1,186 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-usage() {
- echo "This script is to support the specific use case of using Hadoop specific libraries for managing Solr snapshot."
- echo "--prepare-snapshot-export <arg> This command will prepare"
- echo " copylistings for the specified"
- echo " snapshot. This command should only"
- echo " be used only if Solr is deployed"
- echo " with Hadoop and collection index"
- echo " files are stored on a shared"
- echo " file-system e.g. HDFS"
-}
-
-distcp_warning() {
- echo "SOLR_USE_DISTCP environment variable is not set. \
- Do you want to use hadoop distcp tool for exporting Solr collection snapshot ?"
-}
-
-use_bin_solr_warning() {
- echo "Please use the bin/solr script to execute this command."
-}
-
-parse_options() {
- OPTIND=3
- while getopts ":c:d:s:" o ; do
- case "${o}" in
- d)
- destPath=${OPTARG}
- ;;
- s)
- sourcePath=${OPTARG}
- ;;
- c)
- collectionName=${OPTARG}
- ;;
- *)
- echo "Unknown option ${OPTARG}"
- usage 1>&2
- exit 1
- ;;
- esac
- done
-}
-
-run_solr_snapshot_tool() {
- JVM="java"
- scriptDir=$(dirname "$0")
- if [ -n "$LOG4J_PROPS" ]; then
- log4j_config="file:${LOG4J_PROPS}"
- else
- log4j_config="file:${scriptDir}/../../../server/resources/log4j2-console.xml"
- fi
- PATH=${JAVA_HOME}/bin:${PATH} ${JVM} ${ZKCLI_JVM_FLAGS} -Dlog4j.configurationFile=${log4j_config} \
- -classpath "${solrLibPath}" org.apache.solr.hdfs.snapshots.SolrOnHdfsSnapshotsTool "$@" 2> /dev/null
-}
-
-prepare_snapshot_export() {
- #Make sure to cleanup the temporary files.
- scratch=$(mktemp -d -t solrsnaps.XXXXXXXXXX)
- function finish {
- rm -rf "${scratch}"
- }
- trap finish EXIT
-
- if hdfs dfs -test -d "${destPath}" ; then
- run_solr_snapshot_tool --prepare-snapshot-export "$@" -t "${scratch}"
-
- hdfs dfs -mkdir -p "${copyListingDirPath}" > /dev/null
- find "${scratch}" -type f -printf "%f\n" | while read shardId; do
- echo "Copying the copy-listing for $shardId"
- hdfs dfs -copyFromLocal "${scratch}/${shardId}" "${copyListingDirPath}" > /dev/null
- done
- else
- echo "Directory ${destPath} does not exist."
- exit 1
- fi
-}
-
-copy_snapshot_files() {
- copylisting_dir_path="$1"
-
- if hdfs dfs -test -d "${copylisting_dir_path}" ; then
- for shardId in $(hdfs dfs -stat "%n" "${copylisting_dir_path}/*"); do
- oPath="${destPath}/${snapshotName}/snapshot.${shardId}"
- echo "Copying the index files for ${shardId} to ${oPath}"
- ${distCpCmd} -f "${copylisting_dir_path}/${shardId}" "${oPath}" > /dev/null
- done
- else
- echo "Directory ${copylisting_dir_path} does not exist."
- exit 1
- fi
-}
-
-collectionName=""
-destPath=""
-sourcePath=""
-cmd="$1"
-snapshotName="$2"
-copyListingDirPath=""
-distCpCmd="${SOLR_DISTCP_CMD:-hadoop distcp}"
-scriptDir=$(dirname "$0")
-solrLibPath="${SOLR_LIB_PATH:-${scriptDir}/../../../server/solr-webapp/webapp/WEB-INF/lib/*:${scriptDir}/../../../server/lib/ext/*:${scriptDir}/../lib/*}"
-case "${cmd}" in
- --create)
- use_bin_solr_warning
- ;;
- --delete)
- use_bin_solr_warning
- ;;
- --list)
- use_bin_solr_warning
- ;;
- --describe)
- use_bin_solr_warning
- ;;
- --prepare-snapshot-export)
- : "${SOLR_USE_DISTCP:? $(distcp_warning)}"
-
- parse_options "$@"
-
- : "${destPath:? Please specify destination directory using -d option}"
-
- copyListingDirPath="${destPath}/copylistings"
- prepare_snapshot_export "${@:2}"
- echo "Done. GoodBye!"
- ;;
- --export)
- if [ -z "${SOLR_USE_DISTCP}" ]; then
- echo "Not using the distcp command"
- use_bin_solr_warning
- echo "Done. GoodBye!"
- exit 0
- fi
-
- parse_options "$@"
-
- : "${snapshotName:? Please specify the name of the snapshot}"
- : "${destPath:? Please specify destination directory using -d option}"
-
- if [ -n "${collectionName}" ] && [ -n "${sourcePath}" ]; then
- echo "The -c and -s options can not be specified together"
- exit 1
- fi
-
- if [ -z "${collectionName}" ] && [ -z "${sourcePath}" ]; then
- echo "At least one of options (-c or -s) must be specified"
- exit 1
- fi
-
- if [ -n "${collectionName}" ]; then
- copyListingDirPath="${destPath}/${snapshotName}/copylistings"
- prepare_snapshot_export "${@:2}"
- copy_snapshot_files "${destPath}/${snapshotName}/copylistings"
- hdfs dfs -rm -r -f -skipTrash "${destPath}/${snapshotName}/copylistings" > /dev/null
- else
- copy_snapshot_files "${sourcePath}/copylistings"
- echo "Copying the collection meta-data to ${destPath}/${snapshotName}"
- ${distCpCmd} "${sourcePath}/${snapshotName}/*" "${destPath}/${snapshotName}/" > /dev/null
- fi
-
- echo "Done. GoodBye!"
- ;;
- --help|-h)
- usage 1>&2
- ;;
- *)
- echo "Unknown command ${cmd}"
- usage 1>&2
- exit 1
-esac
diff --git a/solr/modules/hdfs/build.gradle b/solr/modules/hdfs/build.gradle
deleted file mode 100644
index 691c585a755..00000000000
--- a/solr/modules/hdfs/build.gradle
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-apply plugin: 'java-library'
-
-description = 'HDFS Module'
-
-dependencies {
- implementation platform(project(':platform'))
- implementation project(':solr:core')
- implementation project(':solr:solrj')
-
- implementation libs.apache.lucene.core
-
- implementation libs.slf4j.api
-
- api libs.apache.hadoop.client.api
- runtimeOnly libs.apache.hadoop.client.runtime
-
- // Guava implements Preconditions, caches, and VisibleForTesting annotations
- implementation libs.google.guava
- implementation libs.dropwizard.metrics.core
-
- // Caffeine cache to implement HDFS block caching
- implementation libs.benmanes.caffeine
-
- implementation libs.commonscli.commonscli
-
- testImplementation project(':solr:test-framework')
- testImplementation project(':solr:solrj-zookeeper')
- testImplementation libs.apache.lucene.testframework
- testImplementation libs.carrotsearch.randomizedtesting.runner
- testImplementation libs.junit.junit
-
- // hadoop dependencies for tests
- testImplementation(libs.apache.hadoop.hdfs) { transitive = false }
- testImplementation(variantOf(libs.apache.hadoop.hdfs) { classifier 'tests' }) { transitive = false }
- testImplementation libs.apache.hadoop.thirdparty.shadedguava
- testRuntimeOnly libs.apache.hadoop.client.minicluster
-
- testRuntimeOnly libs.apache.log4j1.api
-
- // classes like solr.ICUCollationField, used by NNFailoverTest for example.
- testRuntimeOnly project(':solr:modules:analysis-extras')
-
- // used by the hadoop-specific test framework classes
- testImplementation libs.commonsio.commonsio
- testImplementation libs.apache.commons.compress
- testImplementation libs.apache.commons.collections4
- testImplementation libs.apache.commons.lang3
- testImplementation libs.dropwizard.metrics.core
-
- // Zookeeper dependency - some tests like HdfsCloudBackupRestore need this
- permitTestUnusedDeclared libs.apache.zookeeper.zookeeper
- testImplementation(libs.apache.zookeeper.zookeeper, {
- exclude group: "org.apache.yetus", module: "audience-annotations"
- })
- // required for instantiating a Zookeeper server in tests or embedded
- testRuntimeOnly libs.xerial.snappy.java
-}
-
-
-// Copy all the test resource files from core to the build/resources/test directory
-// of the HDFS module so we can avoid duplication of the test resource files like
-// schemas and SolrConfigs. This can be improved later by making the test classes
-// load the resources from core directories directly.
-tasks.register('copySolrCoreTestResources', Copy) {
- from(project(':solr:core').sourceSets.test.resources.srcDirs) {
- exclude '**/*.java'
- }
- into sourceSets.test.output.resourcesDir
-}
-
-tasks.named('processTestResources').configure {
- dependsOn copySolrCoreTestResources
-}
-
-// Hadoop mini cluster doesn't handle reflection properly in JDK 16+
-// https://docs.gradle.org/current/userguide/upgrading_version_7.html#removes_implicit_add_opens_for_test_workers
-tasks.withType(Test).configureEach {
- jvmArgs(["--add-opens=java.base/java.lang=ALL-UNNAMED",
- "--add-opens=java.base/java.util=ALL-UNNAMED"])
-}
-
-assemblePackaging {
- // Add bin folder to default packaging.
- from(projectDir, {
- include "bin/**"
- })
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/HdfsDirectoryFactory.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/HdfsDirectoryFactory.java
deleted file mode 100644
index e0b3a879a7b..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/HdfsDirectoryFactory.java
+++ /dev/null
@@ -1,745 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.RemovalCause;
-import com.google.common.annotations.VisibleForTesting;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.LockFactory;
-import org.apache.lucene.store.NRTCachingDirectory;
-import org.apache.lucene.store.NoLockFactory;
-import org.apache.lucene.store.SingleInstanceLockFactory;
-import org.apache.solr.cloud.api.collections.SplitShardCmd;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.EnvUtils;
-import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.core.CachingDirectoryFactory;
-import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.core.DirectoryFactory;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrPaths;
-import org.apache.solr.hdfs.store.HdfsDirectory;
-import org.apache.solr.hdfs.store.HdfsLocalityReporter;
-import org.apache.solr.hdfs.store.HdfsLockFactory;
-import org.apache.solr.hdfs.store.blockcache.BlockCache;
-import org.apache.solr.hdfs.store.blockcache.BlockDirectory;
-import org.apache.solr.hdfs.store.blockcache.BlockDirectoryCache;
-import org.apache.solr.hdfs.store.blockcache.BufferStore;
-import org.apache.solr.hdfs.store.blockcache.Cache;
-import org.apache.solr.hdfs.store.blockcache.Metrics;
-import org.apache.solr.hdfs.update.HdfsUpdateLog;
-import org.apache.solr.hdfs.util.HdfsUtil;
-import org.apache.solr.metrics.SolrMetricProducer;
-import org.apache.solr.metrics.SolrMetricsContext;
-import org.apache.solr.update.UpdateLog;
-import org.apache.solr.util.plugin.SolrCoreAware;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HdfsDirectoryFactory extends CachingDirectoryFactory
- implements SolrCoreAware, SolrMetricProducer {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public static final String LOCK_TYPE_HDFS = "hdfs";
-
- public static final String BLOCKCACHE_SLAB_COUNT = "solr.hdfs.blockcache.slab.count";
- public static final String BLOCKCACHE_DIRECT_MEMORY_ALLOCATION =
- "solr.hdfs.blockcache.direct.memory.allocation";
- public static final String BLOCKCACHE_ENABLED = "solr.hdfs.blockcache.enabled";
- public static final String BLOCKCACHE_GLOBAL = "solr.hdfs.blockcache.global";
- public static final String BLOCKCACHE_READ_ENABLED = "solr.hdfs.blockcache.read.enabled";
- public static final String BLOCKCACHE_WRITE_ENABLED =
- "solr.hdfs.blockcache.write.enabled"; // currently buggy and disabled
-
- public static final String NRTCACHINGDIRECTORY_ENABLE = "solr.hdfs.nrtcachingdirectory.enable";
- public static final String NRTCACHINGDIRECTORY_MAXMERGESIZEMB =
- "solr.hdfs.nrtcachingdirectory.maxmergesizemb";
- public static final String NRTCACHINGDIRECTORY_MAXCACHEMB =
- "solr.hdfs.nrtcachingdirectory.maxcachedmb";
- public static final String NUMBEROFBLOCKSPERBANK = "solr.hdfs.blockcache.blocksperbank";
-
- public static final String LOCALITYMETRICS_ENABLED = "solr.hdfs.locality.metrics.enabled";
-
- public static final String KERBEROS_ENABLED = "solr.hdfs.security.kerberos.enabled";
- public static final String KERBEROS_KEYTAB = "solr.hdfs.security.kerberos.keytabfile";
- public static final String KERBEROS_PRINCIPAL = "solr.hdfs.security.kerberos.principal";
-
- public static final String HDFS_HOME = "solr.hdfs.home";
-
- public static final String CONFIG_DIRECTORY = "solr.hdfs.confdir";
-
- public static final String CACHE_MERGES = "solr.hdfs.blockcache.cachemerges";
- public static final String CACHE_READONCE = "solr.hdfs.blockcache.cachereadonce";
-
- public static final String HDFS_UPDATE_LOG_CLASS_NAME = "solr.HdfsUpdateLog";
-
- private SolrParams params;
-
- private String hdfsDataDir;
-
- private String confDir;
-
- private boolean cacheReadOnce;
-
- private boolean cacheMerges;
-
- private static BlockCache globalBlockCache;
-
- public static Metrics metrics;
- private static Boolean kerberosInit;
-
- // we use this cache for FileSystem instances when we don't have access to a long lived instance
- private final com.github.benmanes.caffeine.cache.Cache<String, FileSystem> tmpFsCache =
- Caffeine.newBuilder()
- .maximumSize(1000)
- .expireAfterAccess(5, TimeUnit.MINUTES)
- .removalListener(
- (String key, FileSystem fs, RemovalCause cause) -> IOUtils.closeQuietly(fs))
- .build();
-
- private static final class MetricsHolder {
- // [JCIP SE, Goetz, 16.6] Lazy initialization
- // Won't load until MetricsHolder is referenced
- public static final Metrics metrics = new Metrics();
- }
-
- @Override
- public void close() throws IOException {
- super.close();
- Collection<FileSystem> values = tmpFsCache.asMap().values();
- for (FileSystem fs : values) {
- IOUtils.closeQuietly(fs);
- }
- tmpFsCache.invalidateAll();
- tmpFsCache.cleanUp();
- try {
- SolrMetricProducer.super.close();
- MetricsHolder.metrics.close();
- LocalityHolder.reporter.close();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- private static final class LocalityHolder {
- public static final HdfsLocalityReporter reporter = new HdfsLocalityReporter();
- }
-
- @Override
- public void init(NamedList<?> args) {
- super.init(args);
- params = args.toSolrParams();
- this.hdfsDataDir = getConfig(HDFS_HOME, null);
- if (this.hdfsDataDir != null && this.hdfsDataDir.length() == 0) {
- this.hdfsDataDir = null;
- } else {
- log.info("{}={}", HDFS_HOME, this.hdfsDataDir);
- }
- cacheMerges = getConfig(CACHE_MERGES, false);
- cacheReadOnce = getConfig(CACHE_READONCE, false);
- boolean kerberosEnabled = getConfig(KERBEROS_ENABLED, false);
- if (log.isInfoEnabled()) {
- log.info("Solr Kerberos Authentication {}", (kerberosEnabled ? "enabled" : "disabled"));
- }
- if (kerberosEnabled) {
- initKerberos();
- }
- if (StrUtils.isNullOrEmpty(
- EnvUtils.getProperty(SplitShardCmd.SHARDSPLIT_CHECKDISKSPACE_ENABLED))) {
- System.setProperty(SplitShardCmd.SHARDSPLIT_CHECKDISKSPACE_ENABLED, "false");
- }
- }
-
- @Override
- protected LockFactory createLockFactory(String rawLockType) throws IOException {
- if (null == rawLockType) {
- rawLockType = LOCK_TYPE_HDFS;
- log.warn("No lockType configured, assuming '{}'.", rawLockType);
- }
- final String lockType = rawLockType.toLowerCase(Locale.ROOT).trim();
- switch (lockType) {
- case LOCK_TYPE_HDFS:
- return HdfsLockFactory.INSTANCE;
- case DirectoryFactory.LOCK_TYPE_SINGLE:
- return new SingleInstanceLockFactory();
- case DirectoryFactory.LOCK_TYPE_NONE:
- return NoLockFactory.INSTANCE;
- default:
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Unrecognized lockType: " + rawLockType);
- }
- }
-
- @Override
- @SuppressWarnings("resource")
- protected Directory create(String path, LockFactory lockFactory, DirContext dirContext)
- throws IOException {
- assert params != null : "init must be called before create";
- log.info("creating directory factory for path {}", path);
- Configuration conf = getConf(new Path(path));
-
- if (metrics == null) {
- metrics = MetricsHolder.metrics;
- }
-
- boolean blockCacheEnabled = getConfig(BLOCKCACHE_ENABLED, true);
- boolean blockCacheGlobal = getConfig(BLOCKCACHE_GLOBAL, true);
- boolean blockCacheReadEnabled = getConfig(BLOCKCACHE_READ_ENABLED, true);
-
- final HdfsDirectory hdfsDir;
-
- final Directory dir;
- if (blockCacheEnabled && dirContext != DirContext.META_DATA) {
- int numberOfBlocksPerBank = getConfig(NUMBEROFBLOCKSPERBANK, 16384);
-
- int blockSize = BlockDirectory.BLOCK_SIZE;
-
- int bankCount = getConfig(BLOCKCACHE_SLAB_COUNT, 1);
-
- boolean directAllocation = getConfig(BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
-
- int slabSize = numberOfBlocksPerBank * blockSize;
- log.info(
- "Number of slabs of block cache [{}] with direct memory allocation set to [{}]",
- bankCount,
- directAllocation);
- if (log.isInfoEnabled()) {
- log.info(
- "Block cache target memory usage, slab size of [{}] will allocate [{}] slabs and use ~[{}] bytes",
- new Object[] {slabSize, bankCount, ((long) bankCount * (long) slabSize)});
- }
-
- int bsBufferSize = params.getInt("solr.hdfs.blockcache.bufferstore.buffersize", blockSize);
- int bsBufferCount =
- params.getInt(
- "solr.hdfs.blockcache.bufferstore.buffercount", 0); // this is actually total size
-
- BlockCache blockCache =
- getBlockDirectoryCache(
- numberOfBlocksPerBank,
- blockSize,
- bankCount,
- directAllocation,
- slabSize,
- bsBufferSize,
- bsBufferCount,
- blockCacheGlobal);
-
- Cache cache = new BlockDirectoryCache(blockCache, path, metrics, blockCacheGlobal);
- int readBufferSize = params.getInt("solr.hdfs.blockcache.read.buffersize", blockSize);
- hdfsDir = new HdfsDirectory(new Path(path), lockFactory, conf, readBufferSize);
- dir =
- new BlockDirectory(
- path, hdfsDir, cache, null, blockCacheReadEnabled, false, cacheMerges, cacheReadOnce);
- } else {
- hdfsDir = new HdfsDirectory(new Path(path), conf);
- dir = hdfsDir;
- }
- if (params.getBool(LOCALITYMETRICS_ENABLED, false)) {
- LocalityHolder.reporter.registerDirectory(hdfsDir);
- }
-
- boolean nrtCachingDirectory = getConfig(NRTCACHINGDIRECTORY_ENABLE, true);
- if (nrtCachingDirectory) {
- double nrtCacheMaxMergeSizeMB = getConfig(NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
- double nrtCacheMaxCacheMB = getConfig(NRTCACHINGDIRECTORY_MAXCACHEMB, 192);
-
- return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB, nrtCacheMaxCacheMB);
- }
- return dir;
- }
-
- boolean getConfig(String name, boolean defaultValue) {
- Boolean value = params.getBool(name);
- if (value == null) {
- String sysValue = EnvUtils.getProperty(name);
- if (sysValue != null) {
- value = Boolean.valueOf(sysValue);
- }
- }
- return value == null ? defaultValue : value;
- }
-
- int getConfig(String name, int defaultValue) {
- Integer value = params.getInt(name);
- if (value == null) {
- String sysValue = EnvUtils.getProperty(name);
- if (sysValue != null) {
- value = Integer.parseInt(sysValue);
- }
- }
- return value == null ? defaultValue : value;
- }
-
- String getConfig(String name, String defaultValue) {
- String value = params.get(name);
- if (value == null) {
- value = EnvUtils.getProperty(name);
- }
- return value == null ? defaultValue : value;
- }
-
- private BlockCache getBlockDirectoryCache(
- int numberOfBlocksPerBank,
- int blockSize,
- int bankCount,
- boolean directAllocation,
- int slabSize,
- int bufferSize,
- int bufferCount,
- boolean staticBlockCache) {
- if (!staticBlockCache) {
- log.info("Creating new single instance HDFS BlockCache");
- return createBlockCache(
- numberOfBlocksPerBank,
- blockSize,
- bankCount,
- directAllocation,
- slabSize,
- bufferSize,
- bufferCount);
- }
- synchronized (HdfsDirectoryFactory.class) {
- if (globalBlockCache == null) {
- log.info("Creating new global HDFS BlockCache");
- globalBlockCache =
- createBlockCache(
- numberOfBlocksPerBank,
- blockSize,
- bankCount,
- directAllocation,
- slabSize,
- bufferSize,
- bufferCount);
- }
- }
- return globalBlockCache;
- }
-
- private BlockCache createBlockCache(
- int numberOfBlocksPerBank,
- int blockSize,
- int bankCount,
- boolean directAllocation,
- int slabSize,
- int bufferSize,
- int bufferCount) {
- BufferStore.initNewBuffer(bufferSize, bufferCount, metrics);
- long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank * (long) blockSize;
-
- BlockCache blockCache;
- try {
- blockCache = new BlockCache(metrics, directAllocation, totalMemory, slabSize, blockSize);
- } catch (OutOfMemoryError e) {
- throw new RuntimeException(
- "The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args)"
- + " or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap,"
- + " your java heap size might not be large enough."
- + " Failed allocating ~"
- + totalMemory / 1000000.0
- + " MB.",
- e);
- }
- return blockCache;
- }
-
- @Override
- public boolean exists(String path) {
- final Path hdfsDirPath = new Path(path);
- FileSystem fileSystem = getCachedFileSystem(path);
-
- try {
- return fileSystem.exists(hdfsDirPath);
- } catch (IOException e) {
- log.error("Error checking if hdfs path exists", e);
- throw new RuntimeException("Error checking if hdfs path exists", e);
- }
- }
-
- public Configuration getConf(Path path) {
- Configuration conf = new Configuration();
- confDir = getConfig(CONFIG_DIRECTORY, null);
- HdfsUtil.addHdfsResources(conf, confDir);
-
- if (path != null) {
- String fsScheme = path.toUri().getScheme();
- if (fsScheme != null) {
- conf.setBoolean("fs." + fsScheme + ".impl.disable.cache", true);
- }
- }
- return conf;
- }
-
- @Override
- protected synchronized void removeDirectory(final CacheValue cacheValue) {
- FileSystem fileSystem = getCachedFileSystem(cacheValue.path);
-
- try {
- boolean success = fileSystem.delete(new Path(cacheValue.path), true);
- if (!success) {
- throw new RuntimeException("Could not remove directory");
- }
- } catch (Exception e) {
- log.error("Could not remove directory", e);
- throw new SolrException(ErrorCode.SERVER_ERROR, "Could not remove directory", e);
- }
- }
-
- @Override
- public boolean isAbsolute(String path) {
- if (path.startsWith("/")) {
- return false;
- }
- return new Path(path).isAbsolute();
- }
-
- @Override
- public boolean isPersistent() {
- return true;
- }
-
- @Override
- public boolean isSharedStorage() {
- return true;
- }
-
- @Override
- public boolean searchersReserveCommitPoints() {
- return true;
- }
-
- @Override
- public String getDataHome(CoreDescriptor cd) throws IOException {
- if (hdfsDataDir == null) {
- throw new SolrException(
- ErrorCode.SERVER_ERROR,
- "You must set the "
- + this.getClass().getSimpleName()
- + " param "
- + HDFS_HOME
- + " for relative dataDir paths to work");
- }
-
- String scopePath = scopePath(cd);
-
- return normalize(
- SolrPaths.normalizeDir(
- trimLeadingAndTrailingSlashes(hdfsDataDir) + "/" + scopePath + "/" + cd.getDataDir()));
- }
-
- public static String scopePath(CoreDescriptor cd) {
- // by default, we go off the instance directory
- String path;
- if (cd.getCloudDescriptor() != null) {
- path =
- URLEncoder.encode(cd.getCloudDescriptor().getCollectionName(), StandardCharsets.UTF_8)
- + "/"
- + URLEncoder.encode(
- cd.getCloudDescriptor().getCoreNodeName(), StandardCharsets.UTF_8);
- } else {
- path = cd.getName();
- }
- return path;
- }
-
- /**
- * @param directory to calculate size of
- * @return size in bytes
- * @throws IOException on low level IO error
- */
- @Override
- public long size(Directory directory) throws IOException {
- String hdfsDirPath = getPath(directory);
- return size(hdfsDirPath);
- }
-
- /**
- * @param path to calculate size of
- * @return size in bytes
- * @throws IOException on low level IO error
- */
- @Override
- public long size(String path) throws IOException {
- Path hdfsDirPath = new Path(path);
- FileSystem fileSystem = getCachedFileSystem(path);
- try {
- return fileSystem.getContentSummary(hdfsDirPath).getLength();
- } catch (IOException e) {
- log.error("Error calculating size", e);
- throw new SolrException(ErrorCode.SERVER_ERROR, "Error calculating size", e);
- }
- }
-
- private FileSystem getCachedFileSystem(String pathStr) {
- // no need to close the fs, the cache will do it
- Path path = new Path(pathStr);
- return tmpFsCache.get(
- pathStr,
- k -> {
- try {
- return FileSystem.get(path.toUri(), getConf(path));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
- }
-
- public String getConfDir() {
- return confDir;
- }
-
- private void initKerberos() {
- String keytabFile = getConfig(KERBEROS_KEYTAB, "").trim();
- if (keytabFile.length() == 0) {
- throw new IllegalArgumentException(
- KERBEROS_KEYTAB + " required because " + KERBEROS_ENABLED + " set to true");
- }
- String principal = getConfig(KERBEROS_PRINCIPAL, "");
- if (principal.length() == 0) {
- throw new IllegalArgumentException(
- KERBEROS_PRINCIPAL + " required because " + KERBEROS_ENABLED + " set to true");
- }
- synchronized (HdfsDirectoryFactory.class) {
- if (kerberosInit == null) {
- kerberosInit = Boolean.TRUE;
- final Configuration conf = getConf(null);
- final String authVal = conf.get(HADOOP_SECURITY_AUTHENTICATION);
- final String kerberos = "kerberos";
- if (authVal != null && !authVal.equals(kerberos)) {
- throw new IllegalArgumentException(
- HADOOP_SECURITY_AUTHENTICATION
- + " set to: "
- + authVal
- + ", not kerberos, but attempting to "
- + " connect to HDFS via kerberos");
- }
- // let's avoid modifying the supplied configuration, just to be conservative
- final Configuration ugiConf = new Configuration(getConf(null));
- ugiConf.set(HADOOP_SECURITY_AUTHENTICATION, kerberos);
- UserGroupInformation.setConfiguration(ugiConf);
- log.info(
- "Attempting to acquire kerberos ticket with keytab: {}, principal: {} ",
- keytabFile,
- principal);
- try {
- UserGroupInformation.loginUserFromKeytab(principal, keytabFile);
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
- }
- log.info("Got Kerberos ticket");
- }
- }
- }
-
- @Override
- public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
- MetricsHolder.metrics.initializeMetrics(parentContext, scope);
- LocalityHolder.reporter.initializeMetrics(parentContext, scope);
- }
-
- @Override
- public SolrMetricsContext getSolrMetricsContext() {
- return null;
- }
-
- @Override
- public void inform(SolrCore core) {
- setHost(core.getCoreContainer().getHostName());
- }
-
- @VisibleForTesting
- void setHost(String hostname) {
- LocalityHolder.reporter.setHost(hostname);
- }
-
- @Override
- public void cleanupOldIndexDirectories(
- final String dataDir, final String currentIndexDir, boolean afterReload) {
-
- // Get the FileSystem object
- final Path dataDirPath = new Path(dataDir);
- FileSystem fileSystem = getCachedFileSystem(dataDir);
-
- boolean pathExists = false;
- try {
- pathExists = fileSystem.exists(dataDirPath);
- } catch (IOException e) {
- log.error("Error checking if hdfs path exists {}", dataDir, e);
- }
- if (!pathExists) {
- log.warn(
- "{} does not point to a valid data directory; skipping clean-up of old index directories.",
- dataDir);
- return;
- }
-
- final Path currentIndexDirPath =
- new Path(currentIndexDir); // make sure we don't delete the current
- final FileSystem fs = fileSystem;
- FileStatus[] oldIndexDirs = null;
- try {
- oldIndexDirs =
- fileSystem.listStatus(
- dataDirPath,
- new PathFilter() {
- @Override
- public boolean accept(Path path) {
- boolean accept = false;
- String pathName = path.getName();
- try {
- accept =
- fs.getFileStatus(path).isDirectory()
- && !path.equals(currentIndexDirPath)
- && (pathName.equals("index")
- || pathName.matches(INDEX_W_TIMESTAMP_REGEX));
- } catch (IOException e) {
- log.error(
- "Error checking if path {} is an old index directory, caused by: {}",
- path,
- e);
- }
- return accept;
- }
- });
- } catch (FileNotFoundException fnfe) {
- // already deleted - ignore
- log.debug("Old index directory already deleted - skipping...", fnfe);
- } catch (IOException ioExc) {
- log.error("Error checking for old index directories to clean-up.", ioExc);
- }
-
- if (oldIndexDirs == null || oldIndexDirs.length == 0) return; // nothing to clean-up
-
- List<Path> oldIndexPaths = new ArrayList<>(oldIndexDirs.length);
- for (FileStatus ofs : oldIndexDirs) {
- oldIndexPaths.add(ofs.getPath());
- }
-
- oldIndexPaths.sort(Collections.reverseOrder());
-
- Set<String> livePaths = getLivePaths();
-
- int i = 0;
- if (afterReload) {
- log.info("Will not remove most recent old directory on reload {}", oldIndexDirs[0]);
- i = 1;
- }
- log.info(
- "Found {} old index directories to clean-up under {} afterReload={}",
- oldIndexDirs.length - i,
- dataDirPath,
- afterReload);
- for (; i < oldIndexPaths.size(); i++) {
- Path oldDirPath = oldIndexPaths.get(i);
- if (livePaths.contains(oldDirPath.toString())) {
- log.warn(
- "Cannot delete directory {} because it is still being referenced in the cache.",
- oldDirPath);
- } else {
- try {
- if (fileSystem.delete(oldDirPath, true)) {
- log.info("Deleted old index directory {}", oldDirPath);
- } else {
- log.warn("Failed to delete old index directory {}", oldDirPath);
- }
- } catch (IOException e) {
- log.error("Failed to delete old index directory {} due to: {}", oldDirPath, e);
- }
- }
- }
- }
-
- // perform an atomic rename if possible
- @Override
- public void renameWithOverwrite(Directory dir, String fileName, String toName)
- throws IOException {
- String hdfsDirPath = getPath(dir);
- FileContext fileContext = FileContext.getFileContext(getConf(new Path(hdfsDirPath)));
- fileContext.rename(
- new Path(hdfsDirPath, fileName), new Path(hdfsDirPath, toName), Options.Rename.OVERWRITE);
- }
-
- @Override
- public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext)
- throws IOException {
-
- Directory baseFromDir = getBaseDir(fromDir);
- Directory baseToDir = getBaseDir(toDir);
-
- if (baseFromDir instanceof HdfsDirectory && baseToDir instanceof HdfsDirectory) {
- Path dir1 = ((HdfsDirectory) baseFromDir).getHdfsDirPath();
- Path dir2 = ((HdfsDirectory) baseToDir).getHdfsDirPath();
- Path file1 = new Path(dir1, fileName);
- Path file2 = new Path(dir2, fileName);
- FileContext fileContext = FileContext.getFileContext(getConf(dir1));
- fileContext.rename(file1, file2);
- return;
- }
-
- super.move(fromDir, toDir, fileName, ioContext);
- }
-
- @Override
- public UpdateLog newDefaultUpdateLog() {
- return new HdfsUpdateLog(getConfDir());
- }
-
- /**
- * Utility method for trimming and leading and/or trailing slashes from its input. May return the
- * empty string. May return null if and only if the input is null.
- */
- public static String trimLeadingAndTrailingSlashes(final String in) {
- if (null == in) return in;
-
- String out = in;
- if (out.startsWith("/")) {
- out = out.substring(1);
- }
- if (out.endsWith("/")) {
- out = out.substring(0, out.length() - 1);
- }
- return out;
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/backup/repository/HdfsBackupRepository.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/backup/repository/HdfsBackupRepository.java
deleted file mode 100644
index 87b05bac59e..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/backup/repository/HdfsBackupRepository.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.hdfs.backup.repository;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Collection;
-import java.util.Objects;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.NoLockFactory;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.DirectoryFactory;
-import org.apache.solr.core.backup.repository.AbstractBackupRepository;
-import org.apache.solr.hdfs.HdfsDirectoryFactory;
-import org.apache.solr.hdfs.store.HdfsDirectory;
-import org.apache.solr.hdfs.store.HdfsDirectory.HdfsIndexInput;
-
-public class HdfsBackupRepository extends AbstractBackupRepository {
-
- private static final String HDFS_UMASK_MODE_PARAM = "solr.hdfs.permissions.umask-mode";
- private static final String HDFS_COPY_BUFFER_SIZE_PARAM = "solr.hdfs.buffer.size";
-
- private HdfsDirectoryFactory factory;
- private Configuration hdfsConfig = null;
- private FileSystem fileSystem = null;
- private Path baseHdfsPath = null;
- protected int copyBufferSize = HdfsDirectory.DEFAULT_BUFFER_SIZE;
-
- @Override
- public void init(NamedList<?> args) {
- super.init(args);
-
- // Configure the size of the buffer used for copying index files to/from HDFS, if specified.
- if (args.get(HDFS_COPY_BUFFER_SIZE_PARAM) != null) {
- this.copyBufferSize = (Integer) args.get(HDFS_COPY_BUFFER_SIZE_PARAM);
- if (this.copyBufferSize <= 0) {
- throw new IllegalArgumentException(
- "Value of " + HDFS_COPY_BUFFER_SIZE_PARAM + " must be > 0");
- }
- }
-
- String hdfsSolrHome =
- (String)
- Objects.requireNonNull(
- args.get(HdfsDirectoryFactory.HDFS_HOME),
- "Please specify " + HdfsDirectoryFactory.HDFS_HOME + " property.");
- Path path = new Path(hdfsSolrHome);
- while (path
- != null) { // Compute the path of root file-system (without requiring an additional system
- // property).
- baseHdfsPath = path;
- path = path.getParent();
- }
-
- // We don't really need this factory instance. But we want to initialize it here to
- // make sure that all HDFS related initialization is at one place (and not duplicated here).
- factory = new HdfsDirectoryFactory();
- factory.init(args);
- this.hdfsConfig = factory.getConf(new Path(hdfsSolrHome));
-
- // Configure the umask mode if specified.
- if (args.get(HDFS_UMASK_MODE_PARAM) != null) {
- String umaskVal = (String) args.get(HDFS_UMASK_MODE_PARAM);
- this.hdfsConfig.set(FsPermission.UMASK_LABEL, umaskVal);
- }
-
- try {
- this.fileSystem = FileSystem.newInstance(this.baseHdfsPath.toUri(), this.hdfsConfig);
- } catch (IOException e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, e);
- }
- }
-
- @Override
- public void close() throws IOException {
- if (this.fileSystem != null) {
- this.fileSystem.close();
- }
- if (this.factory != null) {
- this.factory.close();
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <T> T getConfigProperty(String name) {
- return (T) this.config.get(name);
- }
-
- @Override
- public URI createURI(String location) {
- Objects.requireNonNull(location);
-
- URI result = null;
- try {
- result = new URI(location);
- if (!result.isAbsolute()) {
- result = resolve(this.baseHdfsPath.toUri(), location);
- }
- } catch (URISyntaxException ex) {
- result = resolve(this.baseHdfsPath.toUri(), location);
- }
-
- return result;
- }
-
- @Override
- public URI resolve(URI baseUri, String... pathComponents) {
- if (!baseUri.isAbsolute()) {
- throw new IllegalArgumentException("baseUri must be absolute");
- }
-
- Path result = new Path(baseUri);
- for (String path : pathComponents) {
- result = new Path(result, path);
- }
-
- return result.toUri();
- }
-
- @Override
- public boolean exists(URI path) throws IOException {
- return this.fileSystem.exists(new Path(path));
- }
-
- @Override
- public PathType getPathType(URI path) throws IOException {
- return this.fileSystem.isDirectory(new Path(path)) ? PathType.DIRECTORY : PathType.FILE;
- }
-
- @Override
- public String[] listAll(URI path) throws IOException {
- FileStatus[] status = this.fileSystem.listStatus(new Path(path));
- String[] result = new String[status.length];
- for (int i = 0; i < status.length; i++) {
- result[i] = status[i].getPath().getName();
- }
- return result;
- }
-
- @Override
- public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws IOException {
- Path p = new Path(new Path(dirPath), fileName);
- return new HdfsIndexInput(fileName, this.fileSystem, p, HdfsDirectory.DEFAULT_BUFFER_SIZE);
- }
-
- @Override
- public OutputStream createOutput(URI path) throws IOException {
- return this.fileSystem.create(new Path(path));
- }
-
- @Override
- public void createDirectory(URI path) throws IOException {
- if (!this.fileSystem.mkdirs(new Path(path))) {
- throw new IOException("Unable to create a directory at following location " + path);
- }
- }
-
- @Override
- public void deleteDirectory(URI path) throws IOException {
- if (!this.fileSystem.delete(new Path(path), true)) {
- throw new IOException("Unable to delete a directory at following location " + path);
- }
- }
-
- @Override
- public void copyIndexFileFrom(
- Directory sourceDir, String sourceFileName, URI destDir, String destFileName)
- throws IOException {
- try (HdfsDirectory dir =
- new HdfsDirectory(new Path(destDir), NoLockFactory.INSTANCE, hdfsConfig, copyBufferSize)) {
- copyIndexFileFrom(sourceDir, sourceFileName, dir, destFileName);
- }
- }
-
- @Override
- public void copyIndexFileTo(
- URI sourceRepo, String sourceFileName, Directory dest, String destFileName)
- throws IOException {
- try (HdfsDirectory dir =
- new HdfsDirectory(
- new Path(sourceRepo), NoLockFactory.INSTANCE, hdfsConfig, copyBufferSize)) {
- dest.copyFrom(dir, sourceFileName, destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
- }
- }
-
- @Override
- public void delete(URI path, Collection<String> files) throws IOException {
- if (files.isEmpty()) return;
-
- for (String file : files) {
- Path filePath = new Path(new Path(path), file);
- fileSystem.delete(filePath, false);
- }
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/backup/repository/package-info.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/backup/repository/package-info.java
deleted file mode 100644
index a8f3fb29fe9..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/backup/repository/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** An HDFS backup repository implementation. */
-package org.apache.solr.hdfs.backup.repository;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/index/CheckHdfsIndex.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/index/CheckHdfsIndex.java
deleted file mode 100644
index 7545b1de326..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/index/CheckHdfsIndex.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.index;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.lucene.index.CheckIndex;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.SuppressForbidden;
-import org.apache.solr.common.util.EnvUtils;
-import org.apache.solr.hdfs.HdfsDirectoryFactory;
-import org.apache.solr.hdfs.store.HdfsDirectory;
-import org.apache.solr.hdfs.util.HdfsUtil;
-
-public class CheckHdfsIndex {
- public static void main(String[] args) throws IOException, InterruptedException {
- int exitCode = doMain(args);
- System.exit(exitCode);
- }
-
- // actual main: returns exit code instead of terminating JVM (for easy testing)
- @SuppressForbidden(reason = "System.out required: command line tool")
- protected static int doMain(String[] args) throws IOException, InterruptedException {
- CheckIndex.Options opts;
- try {
- opts = CheckIndex.parseOptions(args);
- } catch (IllegalArgumentException e) {
- System.out.println(e.getMessage());
- return 1;
- }
-
- if (!CheckIndex.assertsOn()) {
- System.out.println(
- "\nNOTE: testing will be more thorough if you run java with '-ea:org.apache.lucene...', so assertions are enabled");
- }
-
- if (opts.getDirImpl() != null) {
- System.out.println(
- "\nIgnoring specified -dir-impl, instead using " + HdfsDirectory.class.getSimpleName());
- }
-
- Path indexPath = new Path(opts.getIndexPath());
- System.out.println("\nOpening index @ " + indexPath + "\n");
-
- Directory directory;
- try {
- directory = new HdfsDirectory(indexPath, getConf(indexPath));
- } catch (IOException e) {
- System.out.println("ERROR: could not open hdfs directory \"" + indexPath + "\"; exiting");
- e.printStackTrace(System.out);
- return 1;
- }
-
- try (Directory dir = directory;
- CheckIndex checker = new CheckIndex(dir)) {
- opts.setOut(System.out);
- return checker.doCheck(opts);
- }
- }
-
- private static Configuration getConf(Path path) {
- Configuration conf = new Configuration();
- String confDir = EnvUtils.getProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY);
- HdfsUtil.addHdfsResources(conf, confDir);
-
- String fsScheme = path.toUri().getScheme();
- if (fsScheme != null) {
- conf.setBoolean("fs." + fsScheme + ".impl.disable.cache", true);
- }
- return conf;
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/index/package-info.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/index/package-info.java
deleted file mode 100644
index 51de820ed9c..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/index/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** An HDFS CheckIndex implementation. */
-package org.apache.solr.hdfs.index;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrOnHdfsSnapshotsTool.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrOnHdfsSnapshotsTool.java
deleted file mode 100644
index 92c0f73bf5c..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrOnHdfsSnapshotsTool.java
+++ /dev/null
@@ -1,393 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.hdfs.snapshots;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.fs.Path;
-import org.apache.solr.cli.CLIO;
-import org.apache.solr.cli.CommonCLIOptions;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.params.CollectionAdminParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;
-import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
-import org.apache.solr.core.snapshots.SolrSnapshotManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class provides utility functions required for Solr on HDFS specific snapshots'
- * functionality.
- *
- * <p>If HDFS remains in Solr 10, then we should migrate this to extending ToolBase
- *
- * <p>For general purpose snapshot tooling see the related classes in the {@link
- * org.apache.solr.cli} package.
- */
-public class SolrOnHdfsSnapshotsTool implements Closeable, CLIO {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final Option PREPARE_FOR_EXPORT_OPTION =
- Option.builder()
- .longOpt("prepare-snapshot-export")
- .hasArg()
- .desc(
- "The authentication mechanism to enable (basicAuth or kerberos). Defaults to 'basicAuth'.")
- .build();
- private static final Option HDFS_PATH_PREFIX_OPTION =
- Option.builder("p")
- .hasArg()
- .desc(
- "This parameter specifies the HDFS URI prefix to be used "
- + "during snapshot export preparation. This is applicable only if the Solr collection index files are stored on HDFS.")
- .build();
-
- private static final Option TEMP_DIR_OPTION =
- Option.builder("t")
- .hasArg()
- .desc(
- "This parameter specifies the path of a temporary directory on local filesystem"
- + " during prepare-snapshot-export command.")
- .build();
- private static final Option COLLECTION_OPTION =
- Option.builder("c")
- .hasArg()
- .desc(
- "This parameter specifies the name of the collection to be used during snapshot operation")
- .build();
- private static final Option DEST_DIR_OPTION =
- Option.builder("d")
- .hasArg()
- .desc(
- "This parameter specifies the path on shared file-system (e.g. HDFS) where the snapshot related"
- + " information should be stored.")
- .build();
-
- private static final List<String> OPTION_HELP_ORDER =
- Arrays.asList(
- PREPARE_FOR_EXPORT_OPTION.getLongOpt(),
- CommonCLIOptions.HELP_OPTION.getOpt(),
- CommonCLIOptions.ZK_HOST_OPTION.getOpt(),
- COLLECTION_OPTION.getOpt(),
- DEST_DIR_OPTION.getOpt(),
- TEMP_DIR_OPTION.getOpt(),
- HDFS_PATH_PREFIX_OPTION.getOpt());
-
- private final CloudSolrClient solrClient;
-
- public SolrOnHdfsSnapshotsTool(String solrZkEnsemble) {
- solrClient =
- new CloudSolrClient.Builder(Collections.singletonList(solrZkEnsemble), Optional.empty())
- .build();
- }
-
- @Override
- public void close() throws IOException {
- if (solrClient != null) {
- solrClient.close();
- }
- }
-
- private static void checkResponse(CollectionAdminResponse resp, String requestType) {
- if (resp.getStatus() != 0) {
- throw new IllegalStateException(
- "The " + requestType + " request failed. The status code is " + resp.getStatus());
- }
- }
-
- /**
- * @param pathPrefix optional
- */
- public Map<String, List<String>> getIndexFilesPathForSnapshot(
- String collectionName, String snapshotName, String pathPrefix)
- throws SolrServerException, IOException {
- Map<String, List<String>> result = new HashMap<>();
-
- Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName);
- CollectionSnapshotMetaData meta = null;
- for (CollectionSnapshotMetaData m : snaps) {
- if (snapshotName.equals(m.getName())) {
- meta = m;
- }
- }
-
- if (meta != null) {
- throw new IllegalArgumentException(
- "The snapshot named " + snapshotName + " is not found for collection " + collectionName);
- }
-
- DocCollection collectionState = solrClient.getClusterState().getCollection(collectionName);
- for (Slice s : collectionState.getSlices()) {
- List<CoreSnapshotMetaData> replicaSnaps = meta.getReplicaSnapshotsForShard(s.getName());
- // Prepare a list of *existing* replicas (since one or more replicas could have been deleted
- // after the snapshot creation).
- List<CoreSnapshotMetaData> availableReplicas = new ArrayList<>();
- for (CoreSnapshotMetaData m : replicaSnaps) {
- if (isReplicaAvailable(s, m.getCoreName())) {
- availableReplicas.add(m);
- }
- }
-
- if (availableReplicas.isEmpty()) {
- throw new IllegalArgumentException(
- "The snapshot named "
- + snapshotName
- + " not found for shard "
- + s.getName()
- + " of collection "
- + collectionName);
- }
-
- // Prefer a leader replica (at the time when the snapshot was created).
- CoreSnapshotMetaData coreSnap = availableReplicas.get(0);
- for (CoreSnapshotMetaData m : availableReplicas) {
- if (m.isLeader()) {
- coreSnap = m;
- }
- }
-
- String indexDirPath = coreSnap.getIndexDirPath();
- if (pathPrefix != null) {
- // If the path prefix is specified, rebuild the path to the index directory.
- indexDirPath = new Path(pathPrefix, coreSnap.getIndexDirPath()).toString();
- }
-
- List<String> paths = new ArrayList<>();
- for (String fileName : coreSnap.getFiles()) {
- Path p = new Path(indexDirPath, fileName);
- paths.add(p.toString());
- }
-
- result.put(s.getName(), paths);
- }
-
- return result;
- }
-
- /**
- * @param pathPrefix optional
- */
- public void buildCopyListings(
- String collectionName, String snapshotName, String localFsPath, String pathPrefix)
- throws SolrServerException, IOException {
- Map<String, List<String>> paths =
- getIndexFilesPathForSnapshot(collectionName, snapshotName, pathPrefix);
- for (Map.Entry<String, List<String>> entry : paths.entrySet()) {
- // TODO: this used to trim - check if that's needed
- // Using Paths.get instead of Path.of because of conflict with o.a.hadoop.fs.Path
- Files.write(Paths.get(localFsPath, entry.getKey()), entry.getValue());
- }
- }
-
- public void backupCollectionMetaData(String collectionName, String snapshotName, String backupLoc)
- throws SolrServerException, IOException {
- // Backup the collection meta-data
- CollectionAdminRequest.Backup backup =
- new CollectionAdminRequest.Backup(collectionName, snapshotName);
- backup.setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY);
- backup.setLocation(backupLoc);
- CollectionAdminResponse resp = backup.process(solrClient);
- checkResponse(resp, "BACKUP");
- }
-
- /**
- * HDFS specific step required before exporting a snapshot
- *
- * @param pathPrefix optional
- */
- public void prepareForExport(
- String collectionName,
- String snapshotName,
- String localFsPath,
- String pathPrefix,
- String destPath) {
- try {
- buildCopyListings(collectionName, snapshotName, localFsPath, pathPrefix);
- CLIO.out("Successfully prepared copylisting for the snapshot export.");
- } catch (Exception e) {
-
- log.error(
- "Failed to prepare a copylisting for snapshot with name {} for collection {}",
- snapshotName,
- collectionName,
- e);
-
- CLIO.out(
- "Failed to prepare a copylisting for snapshot with name "
- + snapshotName
- + " for collection "
- + collectionName
- + " due to following error : "
- + e.getLocalizedMessage());
- System.exit(1);
- }
-
- try {
- backupCollectionMetaData(collectionName, snapshotName, destPath);
- CLIO.out("Successfully backed up collection meta-data");
- } catch (Exception e) {
- log.error("Failed to backup collection meta-data for collection {}", collectionName, e);
- CLIO.out(
- "Failed to backup collection meta-data for collection "
- + collectionName
- + " due to following error : "
- + e.getLocalizedMessage());
- System.exit(1);
- }
- }
-
- public static void main(String[] args) throws IOException {
- CommandLineParser parser = new DefaultParser();
- Options options = new Options();
-
- options.addOption(PREPARE_FOR_EXPORT_OPTION);
-
- options.addOption(CommonCLIOptions.HELP_OPTION);
- options.addOption(TEMP_DIR_OPTION);
- options.addOption(DEST_DIR_OPTION);
- options.addOption(COLLECTION_OPTION);
- options.addOption(CommonCLIOptions.ZK_HOST_OPTION);
- options.addOption(HDFS_PATH_PREFIX_OPTION);
-
- CommandLine cmd = null;
- try {
- cmd = parser.parse(options, args);
- } catch (ParseException e) {
- CLIO.out(e.getLocalizedMessage());
- printHelp(options);
- System.exit(1);
- }
-
- if (cmd.hasOption(PREPARE_FOR_EXPORT_OPTION)) {
- try (SolrOnHdfsSnapshotsTool tool =
- new SolrOnHdfsSnapshotsTool(requiredArg(options, cmd, CommonCLIOptions.ZK_HOST_OPTION))) {
- if (cmd.hasOption(PREPARE_FOR_EXPORT_OPTION)) {
- String snapshotName = cmd.getOptionValue(PREPARE_FOR_EXPORT_OPTION);
- String collectionName = requiredArg(options, cmd, COLLECTION_OPTION);
- String localFsDir = requiredArg(options, cmd, TEMP_DIR_OPTION);
- String hdfsOpDir = requiredArg(options, cmd, DEST_DIR_OPTION);
- String pathPrefix = cmd.getOptionValue(HDFS_PATH_PREFIX_OPTION);
-
- if (pathPrefix != null) {
- try {
- new URI(pathPrefix);
- } catch (URISyntaxException e) {
- CLIO.out(
- "The specified File system path prefix "
- + pathPrefix
- + " is invalid. The error is "
- + e.getLocalizedMessage());
- System.exit(1);
- }
- }
- tool.prepareForExport(collectionName, snapshotName, localFsDir, pathPrefix, hdfsOpDir);
- }
- }
- } else if (cmd.hasOption(CommonCLIOptions.HELP_OPTION)) {
- printHelp(options);
- } else {
- CLIO.out("Unknown command specified.");
- printHelp(options);
- }
- }
-
- private static String requiredArg(Options options, CommandLine cmd, Option optVal) {
- if (!cmd.hasOption(optVal)) {
- CLIO.out("Please specify the value for option " + optVal);
- printHelp(options);
- System.exit(1);
- }
- return cmd.getOptionValue(optVal);
- }
-
- private static boolean isReplicaAvailable(Slice s, String coreName) {
- for (Replica r : s.getReplicas()) {
- if (coreName.equals(r.getCoreName())) {
- return true;
- }
- }
- return false;
- }
-
- private Collection<CollectionSnapshotMetaData> listCollectionSnapshots(String collectionName)
- throws SolrServerException, IOException {
- CollectionAdminRequest.ListSnapshots listSnapshots =
- new CollectionAdminRequest.ListSnapshots(collectionName);
- CollectionAdminResponse resp = listSnapshots.process(solrClient);
-
- checkResponse(resp, "LISTSNAPSHOTS");
-
- NamedList<?> apiResult =
- (NamedList<?>) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
-
- Collection<CollectionSnapshotMetaData> result = new ArrayList<>();
- for (int i = 0; i < apiResult.size(); i++) {
- result.add(new CollectionSnapshotMetaData((NamedList<?>) apiResult.getVal(i)));
- }
-
- return result;
- }
-
- private static void printHelp(Options options) {
- String helpFooter =
- "Examples: \n"
- + "prepare-snapshot-export.sh --prepare-snapshot-export snapshot-1 -c books -z localhost:2181 -b repo -l backupPath \n";
-
- HelpFormatter formatter = new HelpFormatter();
- formatter.setOptionComparator(new OptionComparator<>());
- formatter.printHelp("SolrOnHdfsSnapshotsTool", null, options, helpFooter, false);
- }
-
- private static class OptionComparator<T extends Option> implements Comparator<T> {
-
- @Override
- public int compare(T o1, T o2) {
- String s1 = o1.hasLongOpt() ? o1.getLongOpt() : o1.getOpt();
- String s2 = o2.hasLongOpt() ? o2.getLongOpt() : o2.getOpt();
- return OPTION_HELP_ORDER.indexOf(s1) - OPTION_HELP_ORDER.indexOf(s2);
- }
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/package-info.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/package-info.java
deleted file mode 100644
index 20b68d67cea..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** An HDFS snapshot implementation. */
-package org.apache.solr.hdfs.snapshots;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsDirectory.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsDirectory.java
deleted file mode 100644
index e89911cc8ac..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsDirectory.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.store;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.lucene.store.BaseDirectory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.LockFactory;
-import org.apache.solr.hdfs.store.blockcache.CustomBufferedIndexInput;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HdfsDirectory extends BaseDirectory {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final int DEFAULT_BUFFER_SIZE = 4096;
-
- private static final String LF_EXT = ".lf";
- protected final Path hdfsDirPath;
- protected final Configuration configuration;
-
- private final FileSystem fileSystem;
- private final FileContext fileContext;
-
- private final int bufferSize;
-
- /** Used to generate temp file names in {@link #createTempOutput}. */
- private final AtomicLong nextTempFileCounter = new AtomicLong();
-
- public HdfsDirectory(Path hdfsDirPath, Configuration configuration) throws IOException {
- this(hdfsDirPath, HdfsLockFactory.INSTANCE, configuration, DEFAULT_BUFFER_SIZE);
- }
-
- public HdfsDirectory(
- Path hdfsDirPath, LockFactory lockFactory, Configuration configuration, int bufferSize)
- throws IOException {
- super(lockFactory);
- this.hdfsDirPath = hdfsDirPath;
- this.configuration = configuration;
- this.bufferSize = bufferSize;
- fileSystem = FileSystem.get(hdfsDirPath.toUri(), configuration);
- fileContext = FileContext.getFileContext(hdfsDirPath.toUri(), configuration);
-
- if (fileSystem instanceof DistributedFileSystem) {
- // Make sure dfs is not in safe mode
- while (((DistributedFileSystem) fileSystem).setSafeMode(SafeModeAction.SAFEMODE_GET, true)) {
- log.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again.");
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- Thread.interrupted();
- // continue
- }
- }
- }
-
- try {
- if (!fileSystem.exists(hdfsDirPath)) {
- boolean success = fileSystem.mkdirs(hdfsDirPath);
- if (!success) {
- throw new RuntimeException("Could not create directory: " + hdfsDirPath);
- }
- }
- } catch (Exception e) {
- org.apache.solr.common.util.IOUtils.closeQuietly(fileSystem);
- throw new RuntimeException("Problem creating directory: " + hdfsDirPath, e);
- }
- }
-
- @Override
- public void close() throws IOException {
- log.info("Closing hdfs directory {}", hdfsDirPath);
- fileSystem.close();
- isOpen = false;
- }
-
- /**
- * Check whether this directory is open or closed. This check may return stale results in the form
- * of false negatives.
- *
- * @return true if the directory is definitely closed, false if the directory is open or is
- * pending closure
- */
- public boolean isClosed() {
- return !isOpen;
- }
-
- @Override
- public IndexOutput createOutput(String name, IOContext context) throws IOException {
- try {
- return new HdfsFileWriter(getFileSystem(), new Path(hdfsDirPath, name), name);
- } catch (FileAlreadyExistsException e) {
- java.nio.file.FileAlreadyExistsException ex =
- new java.nio.file.FileAlreadyExistsException(e.getMessage());
- ex.initCause(e);
- throw ex;
- }
- }
-
- @Override
- public IndexOutput createTempOutput(String prefix, String suffix, IOContext context)
- throws IOException {
- while (true) {
- try {
- String name = getTempFileName(prefix, suffix, nextTempFileCounter.getAndIncrement());
- return new HdfsFileWriter(getFileSystem(), new Path(hdfsDirPath, name), name);
- } catch (FileAlreadyExistsException faee) {
- // Retry with next incremented name
- }
- }
- }
-
- private String[] getNormalNames(List<String> files) {
- int size = files.size();
- for (int i = 0; i < size; i++) {
- String str = files.get(i);
- files.set(i, toNormalName(str));
- }
- return files.toArray(new String[] {});
- }
-
- private String toNormalName(String name) {
- if (name.endsWith(LF_EXT)) {
- return name.substring(0, name.length() - 3);
- }
- return name;
- }
-
- @Override
- public IndexInput openInput(String name, IOContext context) throws IOException {
- return new HdfsIndexInput(name, getFileSystem(), new Path(hdfsDirPath, name), bufferSize);
- }
-
- @Override
- public void deleteFile(String name) throws IOException {
- Path path = new Path(hdfsDirPath, name);
- log.debug("Deleting {}", path);
- getFileSystem().delete(path, false);
- }
-
- @Override
- public void rename(String source, String dest) throws IOException {
- Path sourcePath = new Path(hdfsDirPath, source);
- Path destPath = new Path(hdfsDirPath, dest);
- fileContext.rename(sourcePath, destPath);
- }
-
- @Override
- public void syncMetaData() throws IOException {
- // TODO: how?
- }
-
- @Override
- public long fileLength(String name) throws IOException {
- FileStatus fileStatus = fileSystem.getFileStatus(new Path(hdfsDirPath, name));
- return fileStatus.getLen();
- }
-
- public long fileModified(String name) throws IOException {
- FileStatus fileStatus = getFileSystem().getFileStatus(new Path(hdfsDirPath, name));
- return fileStatus.getModificationTime();
- }
-
- @Override
- public String[] listAll() throws IOException {
- FileStatus[] listStatus = getFileSystem().listStatus(hdfsDirPath);
- List<String> files = new ArrayList<>();
- if (listStatus == null) {
- return new String[] {};
- }
- for (FileStatus status : listStatus) {
- files.add(status.getPath().getName());
- }
- return getNormalNames(files);
- }
-
- public Path getHdfsDirPath() {
- return hdfsDirPath;
- }
-
- public FileSystem getFileSystem() {
- return fileSystem;
- }
-
- public Configuration getConfiguration() {
- return configuration;
- }
-
- @Override
- public Set<String> getPendingDeletions() {
- return Collections.emptySet();
- }
-
- public static class HdfsIndexInput extends CustomBufferedIndexInput {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private final Path path;
- private final FSDataInputStream inputStream;
- private final long length;
- private boolean clone = false;
-
- public HdfsIndexInput(String name, FileSystem fileSystem, Path path, int bufferSize)
- throws IOException {
- super(name, bufferSize);
- this.path = path;
- log.debug("Opening normal index input on {}", path);
- FileStatus fileStatus = fileSystem.getFileStatus(path);
- length = fileStatus.getLen();
- inputStream = fileSystem.open(path, bufferSize);
- }
-
- @Override
- protected void readInternal(byte[] b, int offset, int length) throws IOException {
- inputStream.readFully(getFilePointer(), b, offset, length);
- }
-
- @Override
- protected void seekInternal(long pos) throws IOException {}
-
- @Override
- protected void closeInternal() throws IOException {
- log.debug("Closing normal index input on {}", path);
- if (!clone) {
- inputStream.close();
- }
- }
-
- @Override
- public long length() {
- return length;
- }
-
- @Override
- public IndexInput clone() {
- HdfsIndexInput clone = (HdfsIndexInput) super.clone();
- clone.clone = true;
- return clone;
- }
- }
-
- @Override
- public void sync(Collection<String> names) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("Sync called on {}", Arrays.toString(names.toArray()));
- }
- }
-
- @Override
- public int hashCode() {
- return hdfsDirPath.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof HdfsDirectory)) {
- return false;
- }
- return this.hdfsDirPath.equals(((HdfsDirectory) obj).hdfsDirPath);
- }
-
- @Override
- public String toString() {
- return this.getClass().getSimpleName() + "@" + hdfsDirPath + " lockFactory=" + lockFactory;
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsFileWriter.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsFileWriter.java
deleted file mode 100644
index 4d326030f24..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsFileWriter.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.store;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.EnumSet;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FsServerDefaults;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.lucene.store.OutputStreamIndexOutput;
-
-/**
- * @lucene.experimental
- */
-public class HdfsFileWriter extends OutputStreamIndexOutput {
-
- public static final String HDFS_SYNC_BLOCK = "solr.hdfs.sync.block";
- public static final int BUFFER_SIZE = 16384;
-
- public HdfsFileWriter(FileSystem fileSystem, Path path, String name) throws IOException {
- super(
- "fileSystem=" + fileSystem + " path=" + path,
- name,
- getOutputStream(fileSystem, path),
- BUFFER_SIZE);
- }
-
- private static final OutputStream getOutputStream(FileSystem fileSystem, Path path)
- throws IOException {
- Configuration conf = fileSystem.getConf();
- FsServerDefaults fsDefaults = fileSystem.getServerDefaults(path);
- short replication = fileSystem.getDefaultReplication(path);
- EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.CREATE);
- if (Boolean.getBoolean(HDFS_SYNC_BLOCK)) {
- flags.add(CreateFlag.SYNC_BLOCK);
- }
- return fileSystem.create(
- path,
- FsPermission.getDefault().applyUMask(FsPermission.getUMask(conf)),
- flags,
- fsDefaults.getFileBufferSize(),
- replication,
- fsDefaults.getBlockSize(),
- null);
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsLocalityReporter.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsLocalityReporter.java
deleted file mode 100644
index c70675187ec..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsLocalityReporter.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.store;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.solr.core.SolrInfoBean;
-import org.apache.solr.metrics.MetricsMap;
-import org.apache.solr.metrics.SolrMetricsContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HdfsLocalityReporter implements SolrInfoBean {
- public static final String LOCALITY_BYTES_TOTAL = "locality.bytes.total";
- public static final String LOCALITY_BYTES_LOCAL = "locality.bytes.local";
- public static final String LOCALITY_BYTES_RATIO = "locality.bytes.ratio";
- public static final String LOCALITY_BLOCKS_TOTAL = "locality.blocks.total";
- public static final String LOCALITY_BLOCKS_LOCAL = "locality.blocks.local";
- public static final String LOCALITY_BLOCKS_RATIO = "locality.blocks.ratio";
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private String hostname;
- private final ConcurrentMap<HdfsDirectory, ConcurrentMap<FileStatus, BlockLocation[]>> cache;
-
- private final Set<String> metricNames = ConcurrentHashMap.newKeySet();
- private SolrMetricsContext solrMetricsContext;
-
- public HdfsLocalityReporter() {
- cache = new ConcurrentHashMap<>();
- }
-
- /**
- * Set the host name to use when determining locality
- *
- * @param hostname The name of this host; should correspond to what HDFS Data Nodes think this is.
- */
- public void setHost(String hostname) {
- this.hostname = hostname;
- }
-
- @Override
- public String getName() {
- return "hdfs-locality";
- }
-
- @Override
- public String getDescription() {
- return "Provides metrics for HDFS data locality.";
- }
-
- @Override
- public Category getCategory() {
- return Category.OTHER;
- }
-
- @Override
- public SolrMetricsContext getSolrMetricsContext() {
- return solrMetricsContext;
- }
-
- /** Provide statistics on HDFS block locality, both in terms of bytes and block counts. */
- @Override
- public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
- solrMetricsContext = parentContext.getChildContext(this);
- MetricsMap metricsMap =
- new MetricsMap(
- map -> {
- long totalBytes = 0;
- long localBytes = 0;
- int totalCount = 0;
- int localCount = 0;
-
- for (Iterator<HdfsDirectory> iterator = cache.keySet().iterator();
- iterator.hasNext(); ) {
- HdfsDirectory hdfsDirectory = iterator.next();
-
- if (hdfsDirectory.isClosed()) {
- iterator.remove();
- } else {
- try {
- refreshDirectory(hdfsDirectory);
- Map<FileStatus, BlockLocation[]> blockMap = cache.get(hdfsDirectory);
-
- // For every block in every file in this directory, count it
- for (BlockLocation[] locations : blockMap.values()) {
- for (BlockLocation bl : locations) {
- totalBytes += bl.getLength();
- totalCount++;
-
- if (Arrays.asList(bl.getHosts()).contains(hostname)) {
- localBytes += bl.getLength();
- localCount++;
- }
- }
- }
- } catch (IOException e) {
- log.warn(
- "Could not retrieve locality information for {} due to exception: {}",
- hdfsDirectory.getHdfsDirPath(),
- e);
- }
- }
- }
- map.put(LOCALITY_BYTES_TOTAL, totalBytes);
- map.put(LOCALITY_BYTES_LOCAL, localBytes);
- if (localBytes == 0) {
- map.put(LOCALITY_BYTES_RATIO, 0);
- } else {
- map.put(LOCALITY_BYTES_RATIO, localBytes / (double) totalBytes);
- }
- map.put(LOCALITY_BLOCKS_TOTAL, totalCount);
- map.put(LOCALITY_BLOCKS_LOCAL, localCount);
- if (localCount == 0) {
- map.put(LOCALITY_BLOCKS_RATIO, 0);
- } else {
- map.put(LOCALITY_BLOCKS_RATIO, localCount / (double) totalCount);
- }
- });
- solrMetricsContext.gauge(metricsMap, true, "hdfsLocality", getCategory().toString(), scope);
- }
-
- /**
- * Add a directory for block locality reporting. This directory will continue to be checked until
- * its close method has been called.
- *
- * @param dir The directory to keep metrics on.
- */
- public void registerDirectory(HdfsDirectory dir) {
- if (log.isInfoEnabled()) {
- if (log.isInfoEnabled()) {
- log.info("Registering direcotry {} for locality metrics.", dir.getHdfsDirPath());
- }
- }
- cache.put(dir, new ConcurrentHashMap<>());
- }
-
- /**
- * Update the cached block locations for the given directory. This includes deleting any files
- * that no longer exist in the file system and adding any new files that have shown up.
- *
- * @param dir The directory to refresh
- * @throws IOException If there is a problem getting info from HDFS
- */
- private void refreshDirectory(HdfsDirectory dir) throws IOException {
- Map<FileStatus, BlockLocation[]> directoryCache = cache.get(dir);
- Set<FileStatus> cachedStatuses = directoryCache.keySet();
-
- FileSystem fs = dir.getFileSystem();
- FileStatus[] statuses = fs.listStatus(dir.getHdfsDirPath());
- List<FileStatus> statusList = Arrays.asList(statuses);
-
- log.debug("Updating locality information for: {}", statusList);
-
- // Keep only the files that still exist
- cachedStatuses.retainAll(statusList);
-
- // Fill in missing entries in the cache
- for (FileStatus status : statusList) {
- if (!status.isDirectory() && !directoryCache.containsKey(status)) {
- BlockLocation[] locations = fs.getFileBlockLocations(status, 0, status.getLen());
- directoryCache.put(status, locations);
- }
- }
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsLockFactory.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsLockFactory.java
deleted file mode 100644
index 06e43b7dd2d..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsLockFactory.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.store;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockFactory;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.LockReleaseFailedException;
-import org.apache.solr.common.util.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HdfsLockFactory extends LockFactory {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public static final HdfsLockFactory INSTANCE = new HdfsLockFactory();
-
- private HdfsLockFactory() {}
-
- @Override
- public Lock obtainLock(Directory dir, String lockName) throws IOException {
- if (!(dir instanceof HdfsDirectory hdfsDir)) {
- throw new UnsupportedOperationException(
- "HdfsLockFactory can only be used with HdfsDirectory subclasses, got: " + dir);
- }
- final Configuration conf = hdfsDir.getConfiguration();
- final Path lockPath = hdfsDir.getHdfsDirPath();
- final Path lockFile = new Path(lockPath, lockName);
-
- FSDataOutputStream file = null;
- final FileSystem fs = FileSystem.get(lockPath.toUri(), conf);
- while (true) {
- try {
- if (!fs.exists(lockPath)) {
- boolean success = fs.mkdirs(lockPath);
- if (!success) {
- throw new RuntimeException("Could not create directory: " + lockPath);
- }
- } else {
- // just to check for safe mode
- fs.mkdirs(lockPath);
- }
-
- file = fs.create(lockFile, false);
- break;
- } catch (FileAlreadyExistsException e) {
- throw new LockObtainFailedException("Cannot obtain lock file: " + lockFile, e);
- } catch (RemoteException e) {
- if (e.getClassName().equals("org.apache.hadoop.hdfs.server.namenode.SafeModeException")) {
- log.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again.");
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e1) {
- Thread.interrupted();
- }
- continue;
- }
- throw new LockObtainFailedException("Cannot obtain lock file: " + lockFile, e);
- } catch (IOException e) {
- throw new LockObtainFailedException("Cannot obtain lock file: " + lockFile, e);
- } finally {
- IOUtils.closeQuietly(file);
- }
- }
-
- return new HdfsLock(conf, lockFile);
- }
-
- private static final class HdfsLock extends Lock {
-
- private final Configuration conf;
- private final Path lockFile;
- private volatile boolean closed;
-
- HdfsLock(Configuration conf, Path lockFile) {
- this.conf = conf;
- this.lockFile = lockFile;
- }
-
- @Override
- public void close() throws IOException {
- if (closed) {
- return;
- }
- final FileSystem fs = FileSystem.get(lockFile.toUri(), conf);
- try {
- if (fs.exists(lockFile) && !fs.delete(lockFile, false)) {
- throw new LockReleaseFailedException("failed to delete: " + lockFile);
- }
- } finally {
- IOUtils.closeQuietly(fs);
- }
- }
-
- @Override
- public void ensureValid() throws IOException {
- // no idea how to implement this on HDFS
- }
-
- @Override
- public String toString() {
- return "HdfsLock(lockFile=" + lockFile + ")";
- }
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/package-info.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/package-info.java
deleted file mode 100644
index e68e9336f1f..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** An HDFS blockcache implementation. */
-package org.apache.solr.hdfs.store.blockcache;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/package-info.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/package-info.java
deleted file mode 100644
index dea7244e9f9..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** An HDFS Directory implementation. */
-package org.apache.solr.hdfs.store;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/update/HdfsTransactionLog.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/update/HdfsTransactionLog.java
deleted file mode 100644
index adff96ea638..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/update/HdfsTransactionLog.java
+++ /dev/null
@@ -1,708 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.update;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.util.CollectionUtil;
-import org.apache.solr.common.util.DataInputInputStream;
-import org.apache.solr.common.util.FastInputStream;
-import org.apache.solr.common.util.FastOutputStream;
-import org.apache.solr.common.util.JavaBinCodec;
-import org.apache.solr.common.util.ObjectReleaseTracker;
-import org.apache.solr.hdfs.util.HdfsRecoverLeaseFileSystemUtils;
-import org.apache.solr.hdfs.util.HdfsRecoverLeaseFileSystemUtils.CallerInfo;
-import org.apache.solr.update.CommitUpdateCommand;
-import org.apache.solr.update.TransactionLog;
-import org.apache.solr.update.UpdateLog;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Log Format: List{Operation, Version, ...} ADD, VERSION, DOC DELETE, VERSION, ID_BYTES
- * DELETE_BY_QUERY, VERSION, String
- *
- * <p>TODO: keep two files, one for [operation, version, id] and the other for the actual document
- * data. That way we could throw away document log files more readily while retaining the smaller
- * operation log files longer (and we can retrieve the stored fields from the latest documents from
- * the index).
- *
- * <p>This would require keeping all source fields stored of course.
- *
- * <p>This would also allow to not log document data for requests with commit=true in them (since we
- * know that if the request succeeds, all docs will be committed)
- */
-public class HdfsTransactionLog extends TransactionLog {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static boolean debug = log.isDebugEnabled();
- private static boolean trace = log.isTraceEnabled();
-
- Path tlogFile;
-
- private long finalLogSize;
- private FSDataOutputStream tlogOutStream;
- private FileSystem fs;
-
- private volatile boolean isClosed = false;
-
- HdfsTransactionLog(
- FileSystem fs, Path tlogFile, Collection<String> globalStrings, Integer tlogDfsReplication) {
- this(fs, tlogFile, globalStrings, false, tlogDfsReplication);
- }
-
- HdfsTransactionLog(
- FileSystem fs,
- Path tlogFile,
- Collection<String> globalStrings,
- boolean openExisting,
- Integer tlogDfsReplication) {
- super();
- boolean success = false;
- this.fs = fs;
-
- try {
- this.tlogFile = tlogFile;
-
- if (fs.exists(tlogFile) && openExisting) {
- HdfsRecoverLeaseFileSystemUtils.recoverFileLease(
- fs,
- tlogFile,
- fs.getConf(),
- new CallerInfo() {
-
- @Override
- public boolean isCallerClosed() {
- return isClosed;
- }
- });
-
- tlogOutStream = fs.append(tlogFile);
- } else {
- fs.delete(tlogFile, false);
-
- tlogOutStream = fs.create(tlogFile, (short) tlogDfsReplication.intValue());
- tlogOutStream.hsync();
- }
-
- fos = new FastOutputStream(tlogOutStream, new byte[65536], 0);
- long start = tlogOutStream.getPos();
-
- if (openExisting) {
- if (start > 0) {
- readHeader(null);
-
- // we should already be at the end
- // raf.seek(start);
-
- // assert channel.position() == start;
- fos.setWritten(start); // reflect that we aren't starting at the beginning
- // assert fos.size() == channel.size();
- } else {
- addGlobalStrings(globalStrings);
- }
- } else {
- if (start > 0) {
- log.error(
- "New transaction log already exists:{} size={}", tlogFile, tlogOutStream.size());
- }
-
- addGlobalStrings(globalStrings);
- }
-
- success = true;
-
- assert ObjectReleaseTracker.track(this);
- log.debug("Opening new tlog {}", this);
-
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- } finally {
- if (!success && tlogOutStream != null) {
- try {
- tlogOutStream.close();
- } catch (Exception e) {
- log.error("Error closing tlog file (after error opening)", e);
- }
- }
- }
- }
-
- @Override
- public boolean endsWithCommit() throws IOException {
- ensureFlushed();
- long size = getLogSize();
-
- // the end of the file should have the end message (added during a commit) plus a 4 byte size
- byte[] buf = new byte[END_MESSAGE.length()];
- long pos = size - END_MESSAGE.length() - 4;
- if (pos < 0) return false;
-
- FSDataFastInputStream dis = new FSDataFastInputStream(fs.open(tlogFile), pos);
- try {
- dis.read(buf);
- for (int i = 0; i < buf.length; i++) {
- if (buf[i] != END_MESSAGE.charAt(i)) return false;
- }
- } finally {
- dis.close();
- }
- return true;
- }
-
- @SuppressWarnings({"unchecked"})
- private void readHeader(FastInputStream fis) throws IOException {
- // read existing header
- boolean closeFis = false;
- if (fis == null) closeFis = true;
- fis = fis != null ? fis : new FSDataFastInputStream(fs.open(tlogFile), 0);
- Map<?, ?> header = null;
- try {
- try (LogCodec codec = new LogCodec(resolver)) {
- header = (Map<?, ?>) codec.unmarshal(fis);
- }
-
- fis.readInt(); // skip size
- } finally {
- if (fis != null && closeFis) {
- fis.close();
- }
- }
- // needed to read other records
-
- synchronized (this) {
- globalStringList = (List<String>) header.get("strings");
- globalStringMap = CollectionUtil.newHashMap(globalStringList.size());
- for (int i = 0; i < globalStringList.size(); i++) {
- globalStringMap.put(globalStringList.get(i), i + 1);
- }
- }
- }
-
- @Override
- public long writeCommit(CommitUpdateCommand cmd) {
- LogCodec codec = new LogCodec(resolver);
- synchronized (this) {
- try {
- long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
-
- if (pos == 0) {
- writeLogHeader(codec);
- pos = fos.size();
- }
-
- codec.init(fos);
- codec.writeTag(JavaBinCodec.ARR, 3);
- codec.writeInt(UpdateLog.COMMIT); // should just take one byte
- codec.writeLong(cmd.getVersion());
- codec.writeStr(END_MESSAGE); // ensure these bytes are (almost) last in the file
-
- endRecord(pos);
-
- ensureFlushed(); // flush since this will be the last record in a log fill
-
- // now the commit command is written we will never write to this log again
- closeOutput();
-
- // assert fos.size() == channel.size();
-
- return pos;
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
- }
- }
-
- /* This method is thread safe */
- @Override
- public Object lookup(long pos) {
- // A negative position can result from a log replay (which does not re-log, but does
- // update the version map. This is OK since the node won't be ACTIVE when this happens.
- if (pos < 0) return null;
-
- try {
- // make sure any unflushed buffer has been flushed
- ensureFlushed();
-
- FSDataFastInputStream dis = new FSDataFastInputStream(fs.open(tlogFile), pos);
- try {
- dis.seek(pos);
- try (LogCodec codec = new LogCodec(resolver)) {
- return codec.readVal(new FastInputStream(dis));
- }
- } finally {
- dis.close();
- }
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "pos=" + pos, e);
- }
- }
-
- @Override
- public void closeOutput() {
- try {
- doCloseOutput();
- } catch (IOException e) {
- log.error("Could not close tlog output", e);
- // This situation is not fatal to the caller
- }
- }
-
- private void doCloseOutput() throws IOException {
- synchronized (this) {
- if (fos == null) return;
- if (debug) {
- log.debug("Closing output for {}", tlogFile);
- }
- fos.flushBuffer();
- finalLogSize = fos.size();
- fos = null;
- }
-
- tlogOutStream.hflush();
- tlogOutStream.close();
- tlogOutStream = null;
- }
-
- private void ensureFlushed() throws IOException {
- synchronized (this) {
- if (fos != null) {
- fos.flush();
- tlogOutStream.hflush();
- }
- }
- }
-
- @Override
- public long getLogSize() {
- synchronized (this) {
- if (fos != null) {
- return fos.size();
- } else {
- return finalLogSize;
- }
- }
- }
-
- @Override
- public void finish(UpdateLog.SyncLevel syncLevel) {
- if (syncLevel == UpdateLog.SyncLevel.NONE) return;
- try {
- synchronized (this) {
- fos.flushBuffer();
- }
-
- if (syncLevel == UpdateLog.SyncLevel.FSYNC) {
- tlogOutStream.hsync();
- } else {
- tlogOutStream.hflush();
- }
-
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
- }
-
- @Override
- public void close() {
- try {
- if (debug) {
- log.debug("Closing tlog {}", this);
- }
-
- doCloseOutput();
-
- } catch (IOException e) {
- log.error("Exception closing tlog.", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- } finally {
- isClosed = true;
- assert ObjectReleaseTracker.release(this);
- if (deleteOnClose) {
- try {
- fs.delete(tlogFile, true);
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
- }
- }
- }
-
- @Override
- public String toString() {
- return "hdfs tlog{file=" + tlogFile.toString() + " refcount=" + refcount.get() + "}";
- }
-
- /**
- * Returns a reader that can be used while a log is still in use. Currently only *one* LogReader
- * may be outstanding, and that log may only be used from a single thread.
- */
- @Override
- public LogReader getReader(long startingPos) {
- return new HDFSLogReader(startingPos);
- }
-
- @Override
- public LogReader getSortedReader(long startingPos) {
- return new HDFSSortedLogReader(startingPos);
- }
-
- /** Returns a single threaded reverse reader */
- @Override
- public ReverseReader getReverseReader() throws IOException {
- return new HDFSReverseReader();
- }
-
- public class HDFSLogReader extends LogReader {
- FSDataFastInputStream hdfsFis;
- private LogCodec codec = new LogCodec(resolver);
- private long sz;
-
- public HDFSLogReader(long startingPos) {
- super();
- incref();
- initStream(startingPos);
- }
-
- private void initStream(long pos) {
- try {
-
- synchronized (HdfsTransactionLog.this) {
- ensureFlushed();
- sz = getLogSize();
- }
-
- FSDataInputStream fdis = fs.open(tlogFile);
- hdfsFis = new FSDataFastInputStream(fdis, pos);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Returns the next object from the log, or null if none available.
- *
- * @return The log record, or null if EOF
- * @throws IOException If there is a low-level I/O error.
- */
- @Override
- public Object next() throws IOException, InterruptedException {
- long pos = hdfsFis.position();
-
- synchronized (HdfsTransactionLog.this) {
- if (trace) {
- log.trace("Reading log record. pos={} currentSize={}", pos, getLogSize());
- }
-
- if (pos >= getLogSize()) {
- return null;
- }
- }
-
- // we actually need a new reader to
- // see if any data was added by the writer
- if (pos >= sz) {
- log.info("Read available inputstream data, opening new inputstream pos={} sz={}", pos, sz);
-
- hdfsFis.close();
- initStream(pos);
- }
-
- if (pos == 0) {
- readHeader(hdfsFis);
-
- // shouldn't currently happen - header and first record are currently written at the same
- // time
- synchronized (HdfsTransactionLog.this) {
- if (hdfsFis.position() >= getLogSize()) {
- return null;
- }
- pos = hdfsFis.position();
- }
- }
-
- Object o = codec.readVal(hdfsFis);
-
- // skip over record size
- int size = hdfsFis.readInt();
- assert size == hdfsFis.position() - pos - 4;
-
- return o;
- }
-
- @Override
- public void close() {
- try {
- hdfsFis.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- decref();
- }
-
- @Override
- public String toString() {
- synchronized (HdfsTransactionLog.this) {
- return "LogReader{"
- + "file="
- + tlogFile
- + ", position="
- + hdfsFis.position()
- + ", end="
- + getLogSize()
- + "}";
- }
- }
-
- @Override
- public long currentPos() {
- return hdfsFis.position();
- }
-
- @Override
- public long currentSize() {
- return getLogSize();
- }
- }
-
- public class HDFSSortedLogReader extends HDFSLogReader {
- private long startingPos;
- private boolean inOrder = true;
- private TreeMap<Long, Long> versionToPos;
- Iterator<Long> iterator;
-
- public HDFSSortedLogReader(long startingPos) {
- super(startingPos);
- this.startingPos = startingPos;
- }
-
- @Override
- public Object next() throws IOException, InterruptedException {
- if (versionToPos == null) {
- versionToPos = new TreeMap<>();
- Object o;
- long pos = startingPos;
-
- long lastVersion = Long.MIN_VALUE;
- while ((o = super.next()) != null) {
- List<?> entry = (List<?>) o;
- long version = (Long) entry.get(UpdateLog.VERSION_IDX);
- version = Math.abs(version);
- versionToPos.put(version, pos);
- pos = currentPos();
-
- if (version < lastVersion) inOrder = false;
- lastVersion = version;
- }
- hdfsFis.seek(startingPos);
- }
-
- if (inOrder) {
- return super.next();
- } else {
- if (iterator == null) iterator = versionToPos.values().iterator();
- if (!iterator.hasNext()) return null;
- long pos = iterator.next();
- if (pos != currentPos()) hdfsFis.seek(pos);
- return super.next();
- }
- }
- }
-
- public class HDFSReverseReader extends ReverseReader {
- FSDataFastInputStream fis;
- private LogCodec codec =
- new LogCodec(resolver) {
- @Override
- public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) {
- // Given that the SolrInputDocument is last in an add record, it's OK to just skip
- // reading it completely.
- return null;
- }
- };
-
- int nextLength; // length of the next record (the next one closer to the start of the log file)
- long prevPos; // where we started reading from last time (so prevPos - nextLength == start of
-
- // next record)
-
- public HDFSReverseReader() throws IOException {
- incref();
-
- long sz;
- synchronized (HdfsTransactionLog.this) {
- ensureFlushed();
- sz = getLogSize();
- }
-
- fis = new FSDataFastInputStream(fs.open(tlogFile), 0);
-
- if (sz >= 4) {
- // readHeader(fis); // should not be needed
- prevPos = sz - 4;
- fis.seek(prevPos);
- nextLength = fis.readInt();
- }
- }
-
- /**
- * Returns the next object from the log, or null if none available.
- *
- * @return The log record, or null if EOF
- * @throws IOException If there is a low-level I/O error.
- */
- @Override
- public Object next() throws IOException {
- if (prevPos <= 0) return null;
-
- long endOfThisRecord = prevPos;
-
- int thisLength = nextLength;
-
- long recordStart = prevPos - thisLength; // back up to the beginning of the next record
- prevPos = recordStart - 4; // back up 4 more to read the length of the next record
-
- if (prevPos <= 0) return null; // this record is the header
-
- long bufferPos = fis.getBufferPos();
- if (prevPos >= bufferPos) {
- // nothing to do... we're within the current buffer
- } else {
- // Position buffer so that this record is at the end.
- // For small records, this will cause subsequent calls to next() to be within the buffer.
- long seekPos = endOfThisRecord - fis.getBufferSize();
- seekPos =
- Math.min(
- seekPos,
- prevPos); // seek to the start of the record if it's larger then the block size.
- seekPos = Math.max(seekPos, 0);
- fis.seek(seekPos);
- fis.peek(); // cause buffer to be filled
- }
-
- fis.seek(prevPos);
- nextLength =
- fis.readInt(); // this is the length of the *next* record (i.e. closer to the beginning)
-
- // TODO: optionally skip document data
- Object o = codec.readVal(fis);
-
- // assert fis.position() == prevPos + 4 + thisLength; // this is only true if we read all the
- // data (and we currently skip reading SolrInputDocument
- return o;
- }
-
- /* returns the position in the log file of the last record returned by next() */
- @Override
- public long position() {
- return prevPos + 4; // skip the length
- }
-
- @Override
- public void close() {
- try {
- fis.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- decref();
- }
-
- @Override
- public String toString() {
- synchronized (HdfsTransactionLog.this) {
- return "LogReader{"
- + "file="
- + tlogFile
- + ", position="
- + fis.position()
- + ", end="
- + getLogSize()
- + "}";
- }
- }
- }
-}
-
-class FSDataFastInputStream extends FastInputStream {
- private final FSDataInputStream hdfsFis;
-
- public FSDataFastInputStream(FSDataInputStream hdfsFis, long chPosition) {
- // super(null, new byte[10],0,0); // a small buffer size for testing purposes
- super(null);
- this.hdfsFis = hdfsFis;
- super.readFromStream = chPosition;
- }
-
- @Override
- public int readWrappedStream(byte[] target, int offset, int len) throws IOException {
- return hdfsFis.read(readFromStream, target, offset, len);
- }
-
- public void seek(long position) throws IOException {
- if (position <= readFromStream && position >= getBufferPos()) {
- // seek within buffer
- pos = (int) (position - getBufferPos());
- } else {
- // long currSize = ch.size(); // not needed - underlying read should handle (unless read
- // never done)
- // if (position > currSize) throw new EOFException("Read past EOF: seeking to " + position + "
- // on file of size " + currSize + " file=" + ch);
- readFromStream = position;
- end = pos = 0;
- }
- assert position() == position;
- }
-
- /** where is the start of the buffer relative to the whole file */
- public long getBufferPos() {
- return readFromStream - end;
- }
-
- public int getBufferSize() {
- return buf.length;
- }
-
- @Override
- public void close() throws IOException {
- hdfsFis.close();
- }
-
- @Override
- public String toString() {
- return "readFromStream="
- + readFromStream
- + " pos="
- + pos
- + " end="
- + end
- + " bufferPos="
- + getBufferPos()
- + " position="
- + position();
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/update/HdfsUpdateLog.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/update/HdfsUpdateLog.java
deleted file mode 100644
index abaf7c67eb3..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/update/HdfsUpdateLog.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.update;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Locale;
-import java.util.Objects;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FilterDirectory;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.core.DirectoryFactory;
-import org.apache.solr.core.PluginInfo;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.hdfs.HdfsDirectoryFactory;
-import org.apache.solr.hdfs.store.HdfsDirectory;
-import org.apache.solr.update.CommitUpdateCommand;
-import org.apache.solr.update.TransactionLog;
-import org.apache.solr.update.UpdateLog;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @lucene.experimental
- */
-public class HdfsUpdateLog extends UpdateLog {
-
- private FileSystem fs;
- private volatile Path hdfsTlogDir;
- private final String confDir;
- private Integer tlogDfsReplication;
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private boolean debug = log.isDebugEnabled();
-
- // used internally by tests to track total count of failed tran log loads in init
- public static AtomicLong INIT_FAILED_LOGS_COUNT = new AtomicLong();
-
- public HdfsUpdateLog() {
- this(null);
- }
-
- public HdfsUpdateLog(String confDir) {
- this.confDir = confDir;
- }
-
- @Override
- public void init(PluginInfo info) {
- super.init(info);
-
- tlogDfsReplication = (Integer) info.initArgs.get("tlogDfsReplication");
- if (tlogDfsReplication == null) tlogDfsReplication = 3;
-
- log.info("Initializing HdfsUpdateLog: tlogDfsReplication={}", tlogDfsReplication);
- }
-
- @Override
- protected void initTlogDir(SolrCore core) {
- assert fs == null;
- URI ulog = new Path(dataDir).toUri();
- URI coreData = new Path(core.getDataDir()).toUri();
- if (!ulog.getScheme().equals(coreData.getScheme())
- || !Objects.equals(ulog.getAuthority(), coreData.getAuthority())
- || !java.nio.file.Path.of(ulog.getPath()).startsWith(coreData.getPath())) {
- // ulog is hdfs, but is not scoped under the core data dir (core data dir may
- // be not hdfs, or is a different host/port, or an external path within the same
- // host/port); either way we must scope it analogous to how the data dir is
- // scoped. This inherits the same issue as SOLR-7187, but we're not making
- // anything worse than it already is.
- String scopePath = HdfsDirectoryFactory.scopePath(core.getCoreDescriptor());
- hdfsTlogDir = new Path(dataDir, scopePath);
- } else {
- hdfsTlogDir = new Path(dataDir, TLOG_NAME);
- }
-
- maybeClearLog(core);
-
- try {
- // usage of tlog dir almost entirely bypasses `Directory` API; we only need to do this so that
- // we can remove the tlog dir via `DirectoryFactory.remove()`, which understands how to delete
- // on hdfs.
- DirectoryFactory df = core.getDirectoryFactory();
- Directory tlogDir =
- df.get(
- hdfsTlogDir.toUri().toString(),
- DirectoryFactory.DirContext.DEFAULT,
- DirectoryFactory.LOCK_TYPE_NONE);
- try {
- // here we assume that Hdfs update log will only be configured in conjunction with
- // Hdfs DirectoryFactory.
- fs = ((HdfsDirectory) FilterDirectory.unwrap(tlogDir)).getFileSystem();
- } catch (Throwable t) {
- df.release(tlogDir);
- throw t;
- }
- this.releaseTlogDir = () -> df.release(tlogDir);
- } catch (IOException e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, e);
- }
-
- while (true) {
- try {
- if (!fs.exists(hdfsTlogDir)) {
- boolean success = fs.mkdirs(hdfsTlogDir);
- if (!success) {
- throw new RuntimeException("Could not create directory:" + hdfsTlogDir);
- }
- } else {
- fs.mkdirs(hdfsTlogDir); // To check for safe mode
- }
- break;
- } catch (RemoteException e) {
- if (e.getClassName().equals("org.apache.hadoop.hdfs.server.namenode.SafeModeException")) {
- log.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again.");
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e1) {
- Thread.interrupted();
- }
- continue;
- }
- throw new RuntimeException("Problem creating directory: " + hdfsTlogDir, e);
- } catch (IOException e) {
- throw new RuntimeException("Problem creating directory: " + hdfsTlogDir, e);
- }
- }
-
- String[] oldBufferTlog = getBufferLogList(fs, hdfsTlogDir);
- if (oldBufferTlog != null && oldBufferTlog.length != 0) {
- existOldBufferLog = true;
- }
-
- tlogFiles = getLogList(fs, hdfsTlogDir);
- id = getLastLogId() + 1; // add 1 since we will create a new log for the
- // next update
-
- if (debug) {
- log.debug(
- "UpdateHandler init: tlogDir={}, existing tlogs={}, next id={}",
- hdfsTlogDir,
- Arrays.asList(tlogFiles),
- id);
- }
-
- TransactionLog oldLog = null;
- for (String oldLogName : tlogFiles) {
- Path f = new Path(hdfsTlogDir, oldLogName);
- try {
- oldLog = new HdfsTransactionLog(fs, f, null, true, tlogDfsReplication);
- // don't remove old logs on startup since more than one may be uncapped.
- addOldLog(oldLog, false);
- } catch (Exception e) {
- INIT_FAILED_LOGS_COUNT.incrementAndGet();
- log.error("Failure to open existing log file (non fatal) {}", f, e);
- try {
- fs.delete(f, false);
- } catch (IOException e1) {
- throw new RuntimeException(e1);
- }
- }
- }
-
- // Record first two logs (oldest first) at startup for potential tlog recovery.
- // It's possible that at abnormal close both "tlog" and "prevTlog" were uncapped.
- for (TransactionLog ll : logs) {
- if (newestLogsOnStartup.size() < 2) {
- newestLogsOnStartup.addFirst(ll);
- } else {
- // We're never going to modify old non-recovery logs - no need to hold their output open
- log.info("Closing output for old non-recovery log {}", ll);
- ll.closeOutput();
- }
- }
- }
-
- @Override
- protected String resolveDataDir(SolrCore core, String path) {
- DirectoryFactory df = core.getDirectoryFactory();
- if (!(df instanceof HdfsDirectoryFactory)) {
- // Given that UpdateLog prioritizes low-latency, and that network filesystems are
- // higher-latency, for simplicity we require that HdfsUpdateLog can only be configured
- // if the DirectoryFactory is also Hdfs.
- throw new IllegalStateException(
- HdfsUpdateLog.class.getName() + " incompatible with " + df.getClass().getName());
- }
- if (df.isAbsolute(path)) {
- return path;
- }
- try {
- // For `HdfsUpdateLog` it doesn't make sense to resolve relative to `core.instanceDir()`
- // (which is a regular filesystem path); any relative path will be resolved wrt the core
- // data home.
- String dataHome = df.getDataHome(core.getCoreDescriptor());
- return new Path(dataHome, path).toString();
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- @Override
- public String getTlogDir() {
- return hdfsTlogDir.toUri().toString();
- }
-
- public static String[] getBufferLogList(FileSystem fs, Path tlogDir) {
- final String prefix = BUFFER_TLOG_NAME + '.';
- assert fs != null;
- FileStatus[] fileStatuses;
- try {
- fileStatuses = fs.listStatus(tlogDir, path -> path.getName().startsWith(prefix));
- } catch (IOException e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "Failed on listing old buffer tlog", e);
- }
-
- String[] names = new String[fileStatuses.length];
- for (int i = 0; i < fileStatuses.length; i++) {
- names[i] = fileStatuses[i].getPath().getName();
- }
- return names;
- }
-
- public static String[] getLogList(FileSystem fs, Path tlogDir) {
- final String prefix = TLOG_NAME + '.';
- assert fs != null;
- FileStatus[] fileStatuses;
- try {
- fileStatuses =
- fs.listStatus(
- tlogDir,
- new PathFilter() {
-
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith(prefix);
- }
- });
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- String[] names = new String[fileStatuses.length];
- for (int i = 0; i < fileStatuses.length; i++) {
- names[i] = fileStatuses[i].getPath().getName();
- }
- Arrays.sort(names);
-
- return names;
- }
-
- @Override
- public void close(boolean committed) {
- close(committed, false);
- }
-
- @Override
- protected void ensureBufferTlog() {
- if (bufferTlog != null) return;
- String newLogName =
- String.format(Locale.ROOT, LOG_FILENAME_PATTERN, BUFFER_TLOG_NAME, System.nanoTime());
- bufferTlog =
- new HdfsTransactionLog(
- fs, new Path(hdfsTlogDir, newLogName), globalStrings, tlogDfsReplication);
- bufferTlog.isBuffer = true;
- }
-
- @Override
- protected void deleteBufferLogs() {
- // Delete old buffer logs
- String[] oldBufferTlog = getBufferLogList(fs, hdfsTlogDir);
- if (oldBufferTlog != null && oldBufferTlog.length != 0) {
- for (String oldBufferLogName : oldBufferTlog) {
- Path f = new Path(hdfsTlogDir, oldBufferLogName);
- try {
- boolean s = fs.delete(f, false);
- if (!s) {
- log.error("Could not remove old buffer tlog file:{}", f);
- }
- } catch (IOException e) {
- // No need to bubble up this exception, because it won't cause any problems on recovering
- log.error("Could not remove old buffer tlog file:{}", f, e);
- }
- }
- }
- }
-
- @Override
- protected void ensureLog() {
- if (tlog == null) {
- String newLogName = String.format(Locale.ROOT, LOG_FILENAME_PATTERN, TLOG_NAME, id);
- HdfsTransactionLog ntlog =
- new HdfsTransactionLog(
- fs, new Path(hdfsTlogDir, newLogName), globalStrings, tlogDfsReplication);
- tlog = ntlog;
- }
- }
-
- /** Clears the logs on the file system. Only call before init. */
- @Override
- public void clearLog() {
- Path tlogDir = hdfsTlogDir;
- try {
- if (fs != null && fs.exists(tlogDir)) {
- String[] files = getLogList(tlogDir);
- for (String file : files) {
- Path f = new Path(tlogDir, file);
- boolean s = fs.delete(f, false);
- if (!s) {
- log.error("Could not remove tlog file:{}", f);
- }
- }
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void preSoftCommit(CommitUpdateCommand cmd) {
- debug = log.isDebugEnabled();
- super.preSoftCommit(cmd);
- }
-
- public String[] getLogList(Path tlogDir) throws FileNotFoundException, IOException {
- final String prefix = TLOG_NAME + '.';
- FileStatus[] files =
- fs.listStatus(
- tlogDir,
- new PathFilter() {
-
- @Override
- public boolean accept(Path name) {
- return name.getName().startsWith(prefix);
- }
- });
- List<String> fileList = new ArrayList<>(files.length);
- for (FileStatus file : files) {
- fileList.add(file.getPath().getName());
- }
- return fileList.toArray(new String[0]);
- }
-
- /** Returns true if we were able to drop buffered updates and return to the ACTIVE state */
- // public boolean dropBufferedUpdates() {
- // versionInfo.blockUpdates();
- // try {
- // if (state != State.BUFFERING) return false;
- //
- // if (log.isInfoEnabled()) {
- // log.info("Dropping buffered updates " + this);
- // }
- //
- // // since we blocked updates, this synchronization shouldn't strictly be
- // necessary.
- // synchronized (this) {
- // if (tlog != null) {
- // tlog.rollback(recoveryInfo.positionOfStart);
- // }
- // }
- //
- // state = State.ACTIVE;
- // operationFlags &= ~FLAG_GAP;
- // } catch (IOException e) {
- // log.error("Error attempting to roll back log", e);
- // return false;
- // }
- // finally {
- // versionInfo.unblockUpdates();
- // }
- // return true;
- // }
-
- @Override
- public String toString() {
- return "HDFSUpdateLog{state=" + getState() + ", tlog=" + tlog + "}";
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/update/package-info.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/update/package-info.java
deleted file mode 100644
index 461612bbb63..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/update/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** An HDFS update log and transaction log implementation. */
-package org.apache.solr.hdfs.update;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/HdfsRecoverLeaseFileSystemUtils.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/HdfsRecoverLeaseFileSystemUtils.java
deleted file mode 100644
index 2d5cd6a06ac..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/HdfsRecoverLeaseFileSystemUtils.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.util;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.lang.invoke.MethodHandles;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Borrowed from Apache HBase to recover an HDFS lease.
- * https://github.com/apache/hbase/blob/master/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/util/RecoverLeaseFSUtils.java
- */
-public class HdfsRecoverLeaseFileSystemUtils {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- // internal, for tests
- public static final AtomicLong RECOVER_LEASE_SUCCESS_COUNT = new AtomicLong();
-
- public interface CallerInfo {
- boolean isCallerClosed();
- }
-
- /** Recover the lease from HDFS, retrying multiple times. */
- public static void recoverFileLease(
- final FileSystem fs, final Path p, Configuration conf, CallerInfo callerInfo)
- throws IOException {
- // lease recovery not needed for local file system case.
- if (!(fs instanceof DistributedFileSystem)) return;
- recoverDFSFileLease((DistributedFileSystem) fs, p, conf, callerInfo);
- }
-
- /*
- * Run the dfs recover lease. recoverLease is asynchronous. It returns:
- * - false when it starts the lease recovery (i.e. lease recovery not *yet* done)
- * - true when the lease recovery has succeeded or the file is closed.
- * But, we have to be careful. Each time we call recoverLease, it starts the recover lease
- * process over from the beginning. We could put ourselves in a situation where we are
- * doing nothing but starting a recovery, interrupting it to start again, and so on.
- * The findings over in HBASE-8354 have it that the namenode will try to recover the lease
- * on the file's primary node. If all is well, it should return near immediately. But,
- * as is common, it is the very primary node that has crashed and so the namenode will be
- * stuck waiting on a socket timeout before it will ask another datanode to start the
- * recovery. It does not help if we call recoverLease in the meantime and in particular,
- * subsequent to the socket timeout, a recoverLease invocation will cause us to start
- * over from square one (possibly waiting on socket timeout against primary node). So,
- * in the below, we do the following:
- * 1. Call recoverLease.
- * 2. If it returns true, break.
- * 3. If it returns false, wait a few seconds and then call it again.
- * 4. If it returns true, break.
- * 5. If it returns false, wait for what we think the datanode socket timeout is
- * (configurable) and then try again.
- * 6. If it returns true, break.
- * 7. If it returns false, repeat starting at step 5. above.
- *
- * If HDFS-4525 is available, call it every second and we might be able to exit early.
- */
- static boolean recoverDFSFileLease(
- final DistributedFileSystem dfs,
- final Path p,
- final Configuration conf,
- CallerInfo callerInfo)
- throws IOException {
- log.info("Recovering lease on dfs file {}", p);
- long startWaiting = System.nanoTime();
- // Default is 15 minutes. It's huge, but the idea is that if we have a major issue, HDFS
- // usually needs 10 minutes before marking the nodes as dead. So we're putting ourselves
- // beyond that limit 'to be safe'.
- long recoveryTimeout =
- TimeUnit.NANOSECONDS.convert(
- conf.getInt("solr.hdfs.lease.recovery.timeout", 900000), TimeUnit.MILLISECONDS)
- + startWaiting;
- // This setting should be a little bit above what the cluster dfs heartbeat is set to.
- long firstPause = conf.getInt("solr.hdfs.lease.recovery.first.pause", 4000);
- // This should be set to how long it'll take for us to timeout against primary datanode if it
- // is dead. We set it to 61 seconds, 1 second than the default READ_TIMEOUT in HDFS, the
- // default value for DFS_CLIENT_SOCKET_TIMEOUT_KEY.
- long subsequentPause =
- TimeUnit.NANOSECONDS.convert(
- conf.getInt("solr.hdfs.lease.recovery.dfs.timeout", 61 * 1000), TimeUnit.MILLISECONDS);
-
- if (dfs.isFileClosed(p)) {
- return true;
- }
-
- boolean recovered = false;
- // We break the loop if we succeed the lease recovery, timeout, or we throw an exception.
- for (int nbAttempt = 0; !recovered; nbAttempt++) {
- recovered = recoverLease(dfs, nbAttempt, p, startWaiting);
- if (recovered) break;
- if (checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting)
- || callerInfo.isCallerClosed()) break;
- try {
- // On the first time through wait the short 'firstPause'.
- if (nbAttempt == 0) {
- Thread.sleep(firstPause);
- } else {
- // Cycle here until subsequentPause elapses. While spinning, check isFileClosed
- long localStartWaiting = System.nanoTime();
- while ((System.nanoTime() - localStartWaiting) < subsequentPause
- && !callerInfo.isCallerClosed()) {
- Thread.sleep(conf.getInt("solr.hdfs.lease.recovery.pause", 1000));
-
- if (dfs.isFileClosed(p)) {
- recovered = true;
- break;
- }
- }
- }
- } catch (InterruptedException ie) {
- InterruptedIOException iioe = new InterruptedIOException();
- iioe.initCause(ie);
- throw iioe;
- }
- }
- if (recovered) {
- RECOVER_LEASE_SUCCESS_COUNT.incrementAndGet();
- }
- return recovered;
- }
-
- static boolean checkIfTimedout(
- final Configuration conf,
- final long recoveryTimeout,
- final int nbAttempt,
- final Path p,
- final long startWaiting) {
- if (recoveryTimeout < System.nanoTime()) {
- log.warn(
- "Cannot recoverLease after trying for {}ms (solr.hdfs.lease.recovery.timeout); continuing, but may be DATALOSS!!!; {}",
- conf.getInt("solr.hdfs.lease.recovery.timeout", 900000),
- getLogMessageDetail(nbAttempt, p, startWaiting));
- return true;
- }
- return false;
- }
-
- /**
- * Try to recover the lease.
- *
- * @return True if dfs#recoverLease came by true.
- */
- static boolean recoverLease(
- final DistributedFileSystem dfs, final int nbAttempt, final Path p, final long startWaiting)
- throws FileNotFoundException {
- boolean recovered = false;
- try {
- recovered = dfs.recoverLease(p);
- if (log.isInfoEnabled()) {
- log.info("recoverLease={}, {}", recovered, getLogMessageDetail(nbAttempt, p, startWaiting));
- }
- } catch (IOException e) {
- if (e.getMessage().contains("File does not exist")) {
- // This exception comes out instead of FNFE, fix it
- throw new FileNotFoundException("The given transactionlog file wasn't found at " + p);
- } else if (e instanceof FileNotFoundException) {
- throw (FileNotFoundException) e;
- }
- log.warn(getLogMessageDetail(nbAttempt, p, startWaiting), e);
- }
- return recovered;
- }
-
- /**
- * @return Detail to append to any log message around lease recovering.
- */
- private static String getLogMessageDetail(
- final int nbAttempt, final Path p, final long startWaiting) {
- return "attempt="
- + nbAttempt
- + " on file="
- + p
- + " after "
- + TimeUnit.MILLISECONDS.convert(System.nanoTime() - startWaiting, TimeUnit.NANOSECONDS)
- + "ms";
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/HdfsUtil.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/HdfsUtil.java
deleted file mode 100644
index f930284cc5b..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/HdfsUtil.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.util;
-
-import java.io.File;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
-
-public class HdfsUtil {
-
- // Allows tests to easily add additional conf
- public static Configuration TEST_CONF = null;
-
- private static final String[] HADOOP_CONF_FILES = {
- "core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml", "hadoop-site.xml"
- };
-
- public static void addHdfsResources(Configuration conf, String confDir) {
-
- if (confDir != null && confDir.length() > 0) {
- File confDirFile = new File(confDir);
- if (!confDirFile.exists()) {
- throw new SolrException(
- ErrorCode.SERVER_ERROR,
- "Resource directory does not exist: " + confDirFile.getAbsolutePath());
- }
- if (!confDirFile.isDirectory()) {
- throw new SolrException(
- ErrorCode.SERVER_ERROR,
- "Specified resource directory is not a directory" + confDirFile.getAbsolutePath());
- }
- if (!confDirFile.canRead()) {
- throw new SolrException(
- ErrorCode.SERVER_ERROR,
- "Resource directory must be readable by the Solr process: "
- + confDirFile.getAbsolutePath());
- }
- for (String file : HADOOP_CONF_FILES) {
- if (new File(confDirFile, file).exists()) {
- conf.addResource(new Path(confDir, file));
- }
- }
- }
-
- if (TEST_CONF != null) {
- conf.addResource(TEST_CONF);
- }
- }
-}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/package-info.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/package-info.java
deleted file mode 100644
index fec20561321..00000000000
--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** An HDFS util implementation. */
-package org.apache.solr.hdfs.util;
diff --git a/solr/modules/hdfs/src/test-files/core-site.xml b/solr/modules/hdfs/src/test-files/core-site.xml
deleted file mode 100644
index cfc6d134d76..00000000000
--- a/solr/modules/hdfs/src/test-files/core-site.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<configuration>
- <property>
- <name>hadoop.security.group.mapping</name>
- <value>org.apache.solr.hdfs.cloud.HdfsFakeGroupMapping</value>
- </property>
-
- <!-- SSL KeyStore reloading causes thread leaks - HADOOP-16524 introduced in Hadoop 3.3.1 -->
- <property>
- <name>hadoop.ssl.keystores.factory.class</name>
- <value>org.apache.solr.hdfs.cloud.DisableKeyStoresFactory</value>
- </property>
-</configuration>
diff --git a/solr/modules/hdfs/src/test-files/log4j2.xml b/solr/modules/hdfs/src/test-files/log4j2.xml
deleted file mode 100644
index 96f69f1dc8b..00000000000
--- a/solr/modules/hdfs/src/test-files/log4j2.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<!-- We're configuring testing to be synchronous due to "logging polution", see SOLR-13268 -->
-<Configuration>
- <Appenders>
- <Console name="STDERR" target="SYSTEM_ERR">
- <PatternLayout>
- <Pattern>
- %maxLen{%-4r %-5p (%t) [%notEmpty{n:%X{node_name}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{
- =>%ex{short}}}{10240}%n
- </Pattern>
- </PatternLayout>
- </Console>
- </Appenders>
- <Loggers>
- <!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
- <Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
- <Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
- <Logger name="org.eclipse.jetty" level="INFO"/>
-
- <Root level="INFO">
- <AppenderRef ref="STDERR"/>
- </Root>
- </Loggers>
-</Configuration>
diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java b/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java
deleted file mode 100644
index 29c08364fe3..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java
+++ /dev/null
@@ -1,1917 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs;
-
-import org.apache.commons.collections4.map.CaseInsensitiveMap;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.net.InetAddress;
-import java.net.URI;
-import java.net.UnknownHostException;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.AccessDeniedException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.jar.Attributes;
-import java.util.jar.JarOutputStream;
-import java.util.jar.Manifest;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
-import java.util.zip.ZipInputStream;
-
-/**
- * A collection of file-processing util methods
- */
-public class FileUtil {
- public static final Object SOLR_HACK_FOR_CLASS_VERIFICATION = new Object();
-
- // Apparently the Hadoop code expectes upper-case LOG, so...
- private static final Logger LOG = LoggerFactory.getLogger(FileUtil.class); // nowarn_valid_logger
-
- /* The error code is defined in winutils to indicate insufficient
- * privilege to create symbolic links. This value need to keep in
- * sync with the constant of the same name in:
- * "src\winutils\common.h"
- * */
- public static final int SYMLINK_NO_PRIVILEGE = 2;
-
- /**
- * Buffer size for copy the content of compressed file to new file.
- */
- private static final int BUFFER_SIZE = 8_192;
-
- /**
- * convert an array of FileStatus to an array of Path
- *
- * @param stats
- * an array of FileStatus objects
- * @return an array of paths corresponding to the input
- */
- public static Path[] stat2Paths(FileStatus[] stats) {
- if (stats == null)
- return null;
- Path[] ret = new Path[stats.length];
- for (int i = 0; i < stats.length; ++i) {
- ret[i] = stats[i].getPath();
- }
- return ret;
- }
-
- /**
- * convert an array of FileStatus to an array of Path.
- * If stats if null, return path
- * @param stats
- * an array of FileStatus objects
- * @param path
- * default path to return in stats is null
- * @return an array of paths corresponding to the input
- */
- public static Path[] stat2Paths(FileStatus[] stats, Path path) {
- if (stats == null)
- return new Path[]{path};
- else
- return stat2Paths(stats);
- }
-
- /**
- * Register all files recursively to be deleted on exit.
- * @param file File/directory to be deleted
- */
- public static void fullyDeleteOnExit(final File file) {
- file.deleteOnExit();
- if (file.isDirectory()) {
- File[] files = file.listFiles();
- if (files != null) {
- for (File child : files) {
- fullyDeleteOnExit(child);
- }
- }
- }
- }
-
- /**
- * Delete a directory and all its contents. If
- * we return false, the directory may be partially-deleted.
- * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
- * to by the symlink is not deleted.
- * (2) If dir is symlink to a directory, symlink is deleted. The directory
- * pointed to by symlink is not deleted.
- * (3) If dir is a normal file, it is deleted.
- * (4) If dir is a normal directory, then dir and all its contents recursively
- * are deleted.
- */
- public static boolean fullyDelete(final File dir) {
- return fullyDelete(dir, false);
- }
-
- /**
- * Delete a directory and all its contents. If
- * we return false, the directory may be partially-deleted.
- * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
- * to by the symlink is not deleted.
- * (2) If dir is symlink to a directory, symlink is deleted. The directory
- * pointed to by symlink is not deleted.
- * (3) If dir is a normal file, it is deleted.
- * (4) If dir is a normal directory, then dir and all its contents recursively
- * are deleted.
- * @param dir the file or directory to be deleted
- * @param tryGrantPermissions true if permissions should be modified to delete a file.
- * @return true on success false on failure.
- */
- public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
- if (tryGrantPermissions) {
- // try to chmod +rwx the parent folder of the 'dir':
- File parent = dir.getParentFile();
- grantPermissions(parent);
- }
- if (deleteImpl(dir, false)) {
- // dir is (a) normal file, (b) symlink to a file, (c) empty directory or
- // (d) symlink to a directory
- return true;
- }
- // handle nonempty directory deletion
- if (!fullyDeleteContents(dir, tryGrantPermissions)) {
- return false;
- }
- return deleteImpl(dir, true);
- }
-
- /**
- * Returns the target of the given symlink. Returns the empty string if
- * the given path does not refer to a symlink or there is an error
- * accessing the symlink.
- * @param f File representing the symbolic link.
- * @return The target of the symbolic link, empty string on error or if not
- * a symlink.
- */
- public static String readLink(File f) {
- /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
- * use getCanonicalPath in File to get the target of the symlink but that
- * does not indicate if the given path refers to a symlink.
- */
-
- if (f == null) {
- LOG.warn("Can not read a null symLink");
- return "";
- }
-
- try {
- return Shell.execCommand(
- Shell.getReadlinkCommand(f.toString())).trim();
- } catch (IOException x) {
- return "";
- }
- }
-
- /*
- * Pure-Java implementation of "chmod +rwx f".
- */
- private static void grantPermissions(final File f) {
- FileUtil.setExecutable(f, true);
- FileUtil.setReadable(f, true);
- FileUtil.setWritable(f, true);
- }
-
- private static boolean deleteImpl(final File f, final boolean doLog) {
- if (f == null) {
- LOG.warn("null file argument.");
- return false;
- }
- final boolean wasDeleted = f.delete();
- if (wasDeleted) {
- return true;
- }
- final boolean ex = f.exists();
- if (doLog && ex) {
- LOG.warn("Failed to delete file or dir [{}]: it still exists.", f.getAbsolutePath());
- }
- return !ex;
- }
-
- /**
- * Delete the contents of a directory, not the directory itself. If
- * we return false, the directory may be partially-deleted.
- * If dir is a symlink to a directory, all the contents of the actual
- * directory pointed to by dir will be deleted.
- */
- public static boolean fullyDeleteContents(final File dir) {
- return fullyDeleteContents(dir, false);
- }
-
- /**
- * Delete the contents of a directory, not the directory itself. If
- * we return false, the directory may be partially-deleted.
- * If dir is a symlink to a directory, all the contents of the actual
- * directory pointed to by dir will be deleted.
- * @param tryGrantPermissions if 'true', try grant +rwx permissions to this
- * and all the underlying directories before trying to delete their contents.
- */
- public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
- if (tryGrantPermissions) {
- // to be able to list the dir and delete files from it
- // we must grant the dir rwx permissions:
- grantPermissions(dir);
- }
- boolean deletionSucceeded = true;
- final File[] contents = dir.listFiles();
- if (contents != null) {
- for (int i = 0; i < contents.length; i++) {
- if (contents[i].isFile()) {
- if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file
- deletionSucceeded = false;
- continue; // continue deletion of other files/dirs under dir
- }
- } else {
- // Either directory or symlink to another directory.
- // Try deleting the directory as this might be a symlink
- boolean b = false;
- b = deleteImpl(contents[i], false);
- if (b){
- //this was indeed a symlink or an empty directory
- continue;
- }
- // if not an empty directory or symlink let
- // fullydelete handle it.
- if (!fullyDelete(contents[i], tryGrantPermissions)) {
- deletionSucceeded = false;
- // continue deletion of other files/dirs under dir
- }
- }
- }
- }
- return deletionSucceeded;
- }
-
- /**
- * Recursively delete a directory.
- *
- * @param fs {@link FileSystem} on which the path is present
- * @param dir directory to recursively delete
- * @throws IOException Exception on delete
- * @deprecated Use {@link FileSystem#delete(Path, boolean)}
- */
- @Deprecated
- public static void fullyDelete(FileSystem fs, Path dir)
- throws IOException {
- fs.delete(dir, true);
- }
-
- //
- // If the destination is a subdirectory of the source, then
- // generate exception
- //
- private static void checkDependencies(FileSystem srcFS,
- Path src,
- FileSystem dstFS,
- Path dst)
- throws IOException {
- if (srcFS == dstFS) {
- String srcq = srcFS.makeQualified(src).toString() + Path.SEPARATOR;
- String dstq = dstFS.makeQualified(dst).toString() + Path.SEPARATOR;
- if (dstq.startsWith(srcq)) {
- if (srcq.length() == dstq.length()) {
- throw new IOException("Cannot copy " + src + " to itself.");
- } else {
- throw new IOException("Cannot copy " + src + " to its subdirectory " +
- dst);
- }
- }
- }
- }
-
- /** Copy files between FileSystems. */
- public static boolean copy(FileSystem srcFS, Path src,
- FileSystem dstFS, Path dst,
- boolean deleteSource,
- Configuration conf) throws IOException {
- return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
- }
-
- public static boolean copy(FileSystem srcFS, Path[] srcs,
- FileSystem dstFS, Path dst,
- boolean deleteSource,
- boolean overwrite, Configuration conf)
- throws IOException {
- boolean gotException = false;
- boolean returnVal = true;
- StringBuilder exceptions = new StringBuilder();
-
- if (srcs.length == 1)
- return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf);
-
- // Check if dest is directory
- try {
- FileStatus sdst = dstFS.getFileStatus(dst);
- if (!sdst.isDirectory())
- throw new IOException("copying multiple files, but last argument `" +
- dst + "' is not a directory");
- } catch (FileNotFoundException e) {
- throw new IOException(
- "`" + dst + "': specified destination directory " +
- "does not exist", e);
- }
-
- for (Path src : srcs) {
- try {
- if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf))
- returnVal = false;
- } catch (IOException e) {
- gotException = true;
- exceptions.append(e.getMessage())
- .append("\n");
- }
- }
- if (gotException) {
- throw new IOException(exceptions.toString());
- }
- return returnVal;
- }
-
- /** Copy files between FileSystems. */
- public static boolean copy(FileSystem srcFS, Path src,
- FileSystem dstFS, Path dst,
- boolean deleteSource,
- boolean overwrite,
- Configuration conf) throws IOException {
- FileStatus fileStatus = srcFS.getFileStatus(src);
- return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf);
- }
-
- /** Copy files between FileSystems. */
- public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
- FileSystem dstFS, Path dst,
- boolean deleteSource,
- boolean overwrite,
- Configuration conf) throws IOException {
- Path src = srcStatus.getPath();
- dst = checkDest(src.getName(), dstFS, dst, overwrite);
- if (srcStatus.isDirectory()) {
- checkDependencies(srcFS, src, dstFS, dst);
- if (!dstFS.mkdirs(dst)) {
- return false;
- }
- FileStatus contents[] = srcFS.listStatus(src);
- for (int i = 0; i < contents.length; i++) {
- copy(srcFS, contents[i], dstFS,
- new Path(dst, contents[i].getPath().getName()),
- deleteSource, overwrite, conf);
- }
- } else {
- InputStream in=null;
- OutputStream out = null;
- try {
- in = srcFS.open(src);
- out = dstFS.create(dst, overwrite);
- IOUtils.copyBytes(in, out, conf, true);
- } catch (IOException e) {
- IOUtils.closeStream(out);
- IOUtils.closeStream(in);
- throw e;
- }
- }
- if (deleteSource) {
- return srcFS.delete(src, true);
- } else {
- return true;
- }
-
- }
-
- /** Copy local files to a FileSystem. */
- public static boolean copy(File src,
- FileSystem dstFS, Path dst,
- boolean deleteSource,
- Configuration conf) throws IOException {
- dst = checkDest(src.getName(), dstFS, dst, false);
-
- if (src.isDirectory()) {
- if (!dstFS.mkdirs(dst)) {
- return false;
- }
- File contents[] = listFiles(src);
- for (int i = 0; i < contents.length; i++) {
- copy(contents[i], dstFS, new Path(dst, contents[i].getName()),
- deleteSource, conf);
- }
- } else if (src.isFile()) {
- InputStream in = null;
- OutputStream out =null;
- try {
- in = Files.newInputStream(src.toPath());
- out = dstFS.create(dst);
- IOUtils.copyBytes(in, out, conf);
- } catch (IOException e) {
- IOUtils.closeStream( out );
- IOUtils.closeStream( in );
- throw e;
- }
- } else if (!src.canRead()) {
- throw new IOException(src.toString() +
- ": Permission denied");
-
- } else {
- throw new IOException(src.toString() +
- ": No such file or directory");
- }
- if (deleteSource) {
- return FileUtil.fullyDelete(src);
- } else {
- return true;
- }
- }
-
- /** Copy FileSystem files to local files. */
- public static boolean copy(FileSystem srcFS, Path src,
- File dst, boolean deleteSource,
- Configuration conf) throws IOException {
- FileStatus filestatus = srcFS.getFileStatus(src);
- return copy(srcFS, filestatus, dst, deleteSource, conf);
- }
-
- /** Copy FileSystem files to local files. */
- private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
- File dst, boolean deleteSource,
- Configuration conf) throws IOException {
- Path src = srcStatus.getPath();
- if (srcStatus.isDirectory()) {
- if (!dst.mkdirs()) {
- return false;
- }
- FileStatus contents[] = srcFS.listStatus(src);
- for (int i = 0; i < contents.length; i++) {
- copy(srcFS, contents[i],
- new File(dst, contents[i].getPath().getName()),
- deleteSource, conf);
- }
- } else {
- InputStream in = srcFS.open(src);
- IOUtils.copyBytes(in, Files.newOutputStream(dst.toPath()), conf);
- }
- if (deleteSource) {
- return srcFS.delete(src, true);
- } else {
- return true;
- }
- }
-
- private static Path checkDest(String srcName, FileSystem dstFS, Path dst,
- boolean overwrite) throws IOException {
- FileStatus sdst;
- try {
- sdst = dstFS.getFileStatus(dst);
- } catch (FileNotFoundException e) {
- sdst = null;
- }
- if (null != sdst) {
- if (sdst.isDirectory()) {
- if (null == srcName) {
- if (overwrite) {
- return dst;
- }
- throw new PathIsDirectoryException(dst.toString());
- }
- return checkDest(null, dstFS, new Path(dst, srcName), overwrite);
- } else if (!overwrite) {
- throw new PathExistsException(dst.toString(),
- "Target " + dst + " already exists");
- }
- }
- return dst;
- }
-
- /**
- * Convert a os-native filename to a path that works for the shell.
- * @param filename The filename to convert
- * @return The unix pathname
- * @throws IOException on windows, there can be problems with the subprocess
- */
- public static String makeShellPath(String filename) throws IOException {
- return filename;
- }
-
- /**
- * Convert a os-native filename to a path that works for the shell.
- * @param file The filename to convert
- * @return The unix pathname
- * @throws IOException on windows, there can be problems with the subprocess
- */
- public static String makeShellPath(File file) throws IOException {
- return makeShellPath(file, false);
- }
-
- /**
- * Convert a os-native filename to a path that works for the shell
- * and avoids script injection attacks.
- * @param file The filename to convert
- * @return The unix pathname
- * @throws IOException on windows, there can be problems with the subprocess
- */
- public static String makeSecureShellPath(File file) throws IOException {
- if (Shell.WINDOWS) {
- // Currently it is never called, but it might be helpful in the future.
- throw new UnsupportedOperationException("Not implemented for Windows");
- } else {
- return makeShellPath(file, false).replace("'", "'\\''");
- }
- }
-
- /**
- * Convert a os-native filename to a path that works for the shell.
- * @param file The filename to convert
- * @param makeCanonicalPath
- * Whether to make canonical path for the file passed
- * @return The unix pathname
- * @throws IOException on windows, there can be problems with the subprocess
- */
- public static String makeShellPath(File file, boolean makeCanonicalPath)
- throws IOException {
- if (makeCanonicalPath) {
- return makeShellPath(file.getCanonicalPath());
- } else {
- return makeShellPath(file.toString());
- }
- }
-
- /**
- * Takes an input dir and returns the du on that local directory. Very basic
- * implementation.
- *
- * @param dir
- * The input dir to get the disk space of this local dir
- * @return The total disk space of the input local directory
- */
- public static long getDU(File dir) {
- long size = 0;
- if (!dir.exists())
- return 0;
- if (!dir.isDirectory()) {
- return dir.length();
- } else {
- File[] allFiles = dir.listFiles();
- if (allFiles != null) {
- for (File f : allFiles) {
- if (!org.apache.commons.io.FileUtils.isSymlink(f)) {
- size += getDU(f);
- }
- }
- }
- return size;
- }
- }
-
- /**
- * Given a stream input it will unzip the it in the unzip directory.
- * passed as the second parameter
- * @param inputStream The zip file as input
- * @param toDir The unzip directory where to unzip the zip file.
- * @throws IOException an exception occurred
- */
- public static void unZip(InputStream inputStream, File toDir)
- throws IOException {
- try (ZipInputStream zip = new ZipInputStream(inputStream)) {
- int numOfFailedLastModifiedSet = 0;
- String targetDirPath = toDir.getCanonicalPath() + File.separator;
- for(ZipEntry entry = zip.getNextEntry();
- entry != null;
- entry = zip.getNextEntry()) {
- if (!entry.isDirectory()) {
- File file = new File(toDir, entry.getName());
- if (!file.getCanonicalPath().startsWith(targetDirPath)) {
- throw new IOException("expanding " + entry.getName()
- + " would create file outside of " + toDir);
- }
- File parent = file.getParentFile();
- if (!parent.mkdirs() &&
- !parent.isDirectory()) {
- throw new IOException("Mkdirs failed to create " +
- parent.getAbsolutePath());
- }
- try (OutputStream out = Files.newOutputStream(file.toPath())) {
- IOUtils.copyBytes(zip, out, BUFFER_SIZE);
- }
- if (!file.setLastModified(entry.getTime())) {
- numOfFailedLastModifiedSet++;
- }
- }
- }
- if (numOfFailedLastModifiedSet > 0) {
- LOG.warn("Could not set last modfied time for {} file(s)",
- numOfFailedLastModifiedSet);
- }
- }
- }
-
- /**
- * Given a File input it will unzip it in the unzip directory.
- * passed as the second parameter
- * @param inFile The zip file as input
- * @param unzipDir The unzip directory where to unzip the zip file.
- * @throws IOException An I/O exception has occurred
- */
- public static void unZip(File inFile, File unzipDir) throws IOException {
- Enumeration<? extends ZipEntry> entries;
- ZipFile zipFile = new ZipFile(inFile);
-
- try {
- entries = zipFile.entries();
- String targetDirPath = unzipDir.getCanonicalPath() + File.separator;
- while (entries.hasMoreElements()) {
- ZipEntry entry = entries.nextElement();
- if (!entry.isDirectory()) {
- InputStream in = zipFile.getInputStream(entry);
- try {
- File file = new File(unzipDir, entry.getName());
- if (!file.getCanonicalPath().startsWith(targetDirPath)) {
- throw new IOException("expanding " + entry.getName()
- + " would create file outside of " + unzipDir);
- }
- if (!file.getParentFile().mkdirs()) {
- if (!file.getParentFile().isDirectory()) {
- throw new IOException("Mkdirs failed to create " +
- file.getParentFile().toString());
- }
- }
- OutputStream out = Files.newOutputStream(file.toPath());
- try {
- byte[] buffer = new byte[8192];
- int i;
- while ((i = in.read(buffer)) != -1) {
- out.write(buffer, 0, i);
- }
- } finally {
- out.close();
- }
- } finally {
- in.close();
- }
- }
- }
- } finally {
- zipFile.close();
- }
- }
-
- /**
- * Run a command and send the contents of an input stream to it.
- * @param inputStream Input stream to forward to the shell command
- * @param command shell command to run
- * @throws IOException read or write failed
- * @throws InterruptedException command interrupted
- * @throws ExecutionException task submit failed
- */
- private static void runCommandOnStream(
- InputStream inputStream, String command)
- throws IOException, InterruptedException, ExecutionException {
- ExecutorService executor = null;
- ProcessBuilder builder = new ProcessBuilder();
- builder.command(
- Shell.WINDOWS ? "cmd" : "bash",
- Shell.WINDOWS ? "/c" : "-c",
- command);
- Process process = builder.start();
- int exitCode;
- try {
- // Consume stdout and stderr, to avoid blocking the command
- executor = Executors.newFixedThreadPool(2);
- Future<?> output = executor.submit(() -> {
- try {
- // Read until the output stream receives an EOF and closed.
- if (LOG.isDebugEnabled()) {
- // Log directly to avoid out of memory errors
- try (BufferedReader reader =
- new BufferedReader(
- new InputStreamReader(process.getInputStream(),
- StandardCharsets.UTF_8))) {
- String line;
- while((line = reader.readLine()) != null) {
- LOG.debug(line);
- }
- }
- } else {
- org.apache.commons.io.IOUtils.copy(
- process.getInputStream(),
- new IOUtils.NullOutputStream());
- }
- } catch (IOException e) {
- LOG.debug(e.getMessage());
- }
- });
- Future<?> error = executor.submit(() -> {
- try {
- // Read until the error stream receives an EOF and closed.
- if (LOG.isDebugEnabled()) {
- // Log directly to avoid out of memory errors
- try (BufferedReader reader =
- new BufferedReader(
- new InputStreamReader(process.getErrorStream(),
- StandardCharsets.UTF_8))) {
- String line;
- while((line = reader.readLine()) != null) {
- LOG.debug(line);
- }
- }
- } else {
- org.apache.commons.io.IOUtils.copy(
- process.getErrorStream(),
- new IOUtils.NullOutputStream());
- }
- } catch (IOException e) {
- LOG.debug(e.getMessage());
- }
- });
-
- // Pass the input stream to the command to process
- try {
- org.apache.commons.io.IOUtils.copy(
- inputStream, process.getOutputStream());
- } finally {
- process.getOutputStream().close();
- }
-
- // Wait for both stdout and stderr futures to finish
- error.get();
- output.get();
- } finally {
- // Clean up the threads
- if (executor != null) {
- executor.shutdown();
- }
- // Wait to avoid leaking the child process
- exitCode = process.waitFor();
- }
-
- if (exitCode != 0) {
- throw new IOException(
- String.format(
- "Error executing command. %s " +
- "Process exited with exit code %d.",
- command, exitCode));
- }
- }
-
- /**
- * Given a Tar File as input it will untar the file in a the untar directory
- * passed as the second parameter
- *
- * This utility will untar ".tar" files and ".tar.gz","tgz" files.
- *
- * @param inputStream The tar file as input.
- * @param untarDir The untar directory where to untar the tar file.
- * @param gzipped The input stream is gzipped
- * TODO Use magic number and PusbackInputStream to identify
- * @throws IOException an exception occurred
- * @throws InterruptedException command interrupted
- * @throws ExecutionException task submit failed
- */
- public static void unTar(InputStream inputStream, File untarDir,
- boolean gzipped)
- throws IOException, InterruptedException, ExecutionException {
- if (!untarDir.mkdirs()) {
- if (!untarDir.isDirectory()) {
- throw new IOException("Mkdirs failed to create " + untarDir);
- }
- }
-
- if(Shell.WINDOWS) {
- // Tar is not native to Windows. Use simple Java based implementation for
- // tests and simple tar archives
- unTarUsingJava(inputStream, untarDir, gzipped);
- } else {
- // spawn tar utility to untar archive for full fledged unix behavior such
- // as resolving symlinks in tar archives
- unTarUsingTar(inputStream, untarDir, gzipped);
- }
- }
-
- /**
- * Given a Tar File as input it will untar the file in a the untar directory
- * passed as the second parameter
- *
- * This utility will untar ".tar" files and ".tar.gz","tgz" files.
- *
- * @param inFile The tar file as input.
- * @param untarDir The untar directory where to untar the tar file.
- * @throws IOException exception untarring
- */
- public static void unTar(File inFile, File untarDir) throws IOException {
- if (!untarDir.mkdirs()) {
- if (!untarDir.isDirectory()) {
- throw new IOException("Mkdirs failed to create " + untarDir);
- }
- }
-
- boolean gzipped = inFile.toString().endsWith("gz");
- if(Shell.WINDOWS) {
- // Tar is not native to Windows. Use simple Java based implementation for
- // tests and simple tar archives
- unTarUsingJava(inFile, untarDir, gzipped);
- }
- else {
- // spawn tar utility to untar archive for full fledged unix behavior such
- // as resolving symlinks in tar archives
- unTarUsingTar(inFile, untarDir, gzipped);
- }
- }
-
- private static void unTarUsingTar(InputStream inputStream, File untarDir,
- boolean gzipped)
- throws IOException, InterruptedException, ExecutionException {
- StringBuilder untarCommand = new StringBuilder();
- if (gzipped) {
- untarCommand.append("gzip -dc | (");
- }
- untarCommand.append("cd '")
- .append(FileUtil.makeSecureShellPath(untarDir))
- .append("' && ")
- .append("tar -x ");
-
- if (gzipped) {
- untarCommand.append(")");
- }
- runCommandOnStream(inputStream, untarCommand.toString());
- }
-
- private static void unTarUsingTar(File inFile, File untarDir,
- boolean gzipped) throws IOException {
- StringBuilder untarCommand = new StringBuilder();
- // not using canonical path here; this postpones relative path
- // resolution until bash is executed.
- final String source = "'" + FileUtil.makeSecureShellPath(inFile) + "'";
- if (gzipped) {
- untarCommand.append(" gzip -dc ")
- .append(source)
- .append(" | (");
- }
- untarCommand.append("cd '")
- .append(FileUtil.makeSecureShellPath(untarDir))
- .append("' && ")
- .append("tar -xf ");
-
- if (gzipped) {
- untarCommand.append(" -)");
- } else {
- untarCommand.append(source);
- }
- LOG.debug("executing [{}]", untarCommand);
- String[] shellCmd = { "bash", "-c", untarCommand.toString() };
- ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
- shexec.execute();
- int exitcode = shexec.getExitCode();
- if (exitcode != 0) {
- throw new IOException("Error untarring file " + inFile +
- ". Tar process exited with exit code " + exitcode
- + " from command " + untarCommand);
- }
- }
-
- static void unTarUsingJava(File inFile, File untarDir,
- boolean gzipped) throws IOException {
- InputStream inputStream = null;
- TarArchiveInputStream tis = null;
- try {
- if (gzipped) {
- inputStream =
- new GZIPInputStream(Files.newInputStream(inFile.toPath()));
- } else {
- inputStream = Files.newInputStream(inFile.toPath());
- }
-
- inputStream = new BufferedInputStream(inputStream);
-
- tis = new TarArchiveInputStream(inputStream);
-
- for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
- unpackEntries(tis, entry, untarDir);
- entry = tis.getNextTarEntry();
- }
- } finally {
- IOUtils.cleanupWithLogger(LOG, tis, inputStream);
- }
- }
-
- private static void unTarUsingJava(InputStream inputStream, File untarDir,
- boolean gzipped) throws IOException {
- TarArchiveInputStream tis = null;
- try {
- if (gzipped) {
- inputStream = new GZIPInputStream(inputStream);
- }
- inputStream = new BufferedInputStream(inputStream);
- tis = new TarArchiveInputStream(inputStream);
-
- for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
- unpackEntries(tis, entry, untarDir);
- entry = tis.getNextTarEntry();
- }
- } finally {
- IOUtils.cleanupWithLogger(LOG, tis, inputStream);
- }
- }
-
- private static void unpackEntries(TarArchiveInputStream tis,
- TarArchiveEntry entry, File outputDir) throws IOException {
- String targetDirPath = outputDir.getCanonicalPath() + File.separator;
- File outputFile = new File(outputDir, entry.getName());
- if (!outputFile.getCanonicalPath().startsWith(targetDirPath)) {
- throw new IOException("expanding " + entry.getName()
- + " would create entry outside of " + outputDir);
- }
-
- if (entry.isSymbolicLink() || entry.isLink()) {
- String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
- if (!canonicalTargetPath.startsWith(targetDirPath)) {
- throw new IOException(
- "expanding " + entry.getName() + " would create entry outside of " + outputDir);
- }
- }
-
- if (entry.isDirectory()) {
- File subDir = new File(outputDir, entry.getName());
- if (!subDir.mkdirs() && !subDir.isDirectory()) {
- throw new IOException("Mkdirs failed to create tar internal dir "
- + outputDir);
- }
-
- for (TarArchiveEntry e : entry.getDirectoryEntries()) {
- unpackEntries(tis, e, subDir);
- }
-
- return;
- }
-
- if (entry.isSymbolicLink()) {
- // Create symlink with canonical target path to ensure that we don't extract
- // outside targetDirPath
- String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
- Files.createSymbolicLink(
- FileSystems.getDefault().getPath(outputDir.getPath(), entry.getName()),
- FileSystems.getDefault().getPath(canonicalTargetPath));
- return;
- }
-
- if (!outputFile.getParentFile().exists()) {
- if (!outputFile.getParentFile().mkdirs()) {
- throw new IOException("Mkdirs failed to create tar internal dir "
- + outputDir);
- }
- }
-
- if (entry.isLink()) {
- String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
- File src = new File(canonicalTargetPath);
- HardLink.createHardLink(src, outputFile);
- return;
- }
-
- org.apache.commons.io.FileUtils.copyToFile(tis, outputFile);
- }
-
- /**
- * Gets the canonical path for the given path.
- *
- * @param path The path for which the canonical path needs to be computed.
- * @param parentDir The parent directory to use if the path is a relative path.
- * @return The canonical path of the given path.
- */
- private static String getCanonicalPath(String path, File parentDir) throws IOException {
- java.nio.file.Path targetPath = Paths.get(path);
- return (targetPath.isAbsolute() ?
- new File(path) :
- new File(parentDir, path)).getCanonicalPath();
- }
-
- /**
- * Class for creating hardlinks.
- * Supports Unix, WindXP.
- * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
- */
- @Deprecated
- public static class HardLink extends org.apache.hadoop.fs.HardLink {
- // This is a stub to assist with coordinated change between
- // COMMON and HDFS projects. It will be removed after the
- // corresponding change is committed to HDFS.
- }
-
- /**
- * Create a soft link between a src and destination
- * only on a local disk. HDFS does not support this.
- * On Windows, when symlink creation fails due to security
- * setting, we will log a warning. The return code in this
- * case is 2.
- *
- * @param target the target for symlink
- * @param linkname the symlink
- * @return 0 on success
- */
- public static int symLink(String target, String linkname) throws IOException{
-
- if (target == null || linkname == null) {
- LOG.warn("Can not create a symLink with a target = {} and link = {}", target, linkname);
- return 1;
- }
-
- // Run the input paths through Java's File so that they are converted to the
- // native OS form
- File targetFile = new File(
- Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString());
- File linkFile = new File(
- Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
-
- String[] cmd = Shell.getSymlinkCommand(
- targetFile.toString(),
- linkFile.toString());
-
- ShellCommandExecutor shExec;
- try {
- if (Shell.WINDOWS &&
- linkFile.getParentFile() != null &&
- !new Path(target).isAbsolute()) {
- // Relative links on Windows must be resolvable at the time of
- // creation. To ensure this we run the shell command in the directory
- // of the link.
- //
- shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile());
- } else {
- shExec = new ShellCommandExecutor(cmd);
- }
- shExec.execute();
- } catch (Shell.ExitCodeException ec) {
- int returnVal = ec.getExitCode();
- if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
- LOG.warn("Fail to create symbolic links on Windows. "
- + "The default security settings in Windows disallow non-elevated "
- + "administrators and all non-administrators from creating symbolic links. "
- + "This behavior can be changed in the Local Security Policy management console");
- } else if (returnVal != 0) {
- LOG.warn("Command '{}' failed {} with: {}",StringUtils.join(" ", cmd)
- , returnVal, ec.getMessage());
- }
- return returnVal;
- } catch (IOException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Error while create symlink {} to {}. Exception: {}", linkname, target, StringUtils.stringifyException(e));
- }
- throw e;
- }
- return shExec.getExitCode();
- }
-
- /**
- * Change the permissions on a filename.
- * @param filename the name of the file to change
- * @param perm the permission string
- * @return the exit code from the command
- * @throws IOException exception on chmod
- */
- public static int chmod(String filename, String perm
- ) throws IOException {
- return chmod(filename, perm, false);
- }
-
- /**
- * Change the permissions on a file / directory, recursively, if
- * needed.
- * @param filename name of the file whose permissions are to change
- * @param perm permission string
- * @param recursive true, if permissions should be changed recursively
- * @return the exit code from the command.
- * @throws IOException exception on chmod
- */
- public static int chmod(String filename, String perm, boolean recursive)
- throws IOException {
- String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
- String[] args = new String[cmd.length + 1];
- System.arraycopy(cmd, 0, args, 0, cmd.length);
- args[cmd.length] = new File(filename).getPath();
- ShellCommandExecutor shExec = new ShellCommandExecutor(args);
- try {
- shExec.execute();
- }catch(IOException e) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Error while changing permission : {} Exception: {}", filename, StringUtils.stringifyException(e));
- }
- }
- return shExec.getExitCode();
- }
-
- /**
- * Set the ownership on a file / directory. User name and group name
- * cannot both be null.
- * @param file the file to change
- * @param username the new user owner name
- * @param groupname the new group owner name
- * @throws IOException exception on setOwner
- */
- public static void setOwner(File file, String username,
- String groupname) throws IOException {
- if (username == null && groupname == null) {
- throw new IOException("username == null && groupname == null");
- }
- String arg = (username == null ? "" : username)
- + (groupname == null ? "" : ":" + groupname);
- String [] cmd = Shell.getSetOwnerCommand(arg);
- execCommand(file, cmd);
- }
-
- /**
- * Platform independent implementation for {@link File#setReadable(boolean)}
- * File#setReadable does not work as expected on Windows.
- * @param f input file
- * @param readable set to readable or not
- * @return true on success, false otherwise
- */
- public static boolean setReadable(File f, boolean readable) {
- if (Shell.WINDOWS) {
- try {
- String permission = readable ? "u+r" : "u-r";
- FileUtil.chmod(f.getCanonicalPath(), permission, false);
- return true;
- } catch (IOException ex) {
- return false;
- }
- } else {
- return f.setReadable(readable);
- }
- }
-
- /**
- * Platform independent implementation for {@link File#setWritable(boolean)}
- * File#setWritable does not work as expected on Windows.
- * @param f input file
- * @param writable set to writable or not
- * @return true on success, false otherwise
- */
- public static boolean setWritable(File f, boolean writable) {
- if (Shell.WINDOWS) {
- try {
- String permission = writable ? "u+w" : "u-w";
- FileUtil.chmod(f.getCanonicalPath(), permission, false);
- return true;
- } catch (IOException ex) {
- return false;
- }
- } else {
- return f.setWritable(writable);
- }
- }
-
- /**
- * Platform independent implementation for {@link File#setExecutable(boolean)}
- * File#setExecutable does not work as expected on Windows.
- * Note: revoking execute permission on folders does not have the same
- * behavior on Windows as on Unix platforms. Creating, deleting or renaming
- * a file within that folder will still succeed on Windows.
- * @param f input file
- * @param executable set to executable or not
- * @return true on success, false otherwise
- */
- public static boolean setExecutable(File f, boolean executable) {
- if (Shell.WINDOWS) {
- try {
- String permission = executable ? "u+x" : "u-x";
- FileUtil.chmod(f.getCanonicalPath(), permission, false);
- return true;
- } catch (IOException ex) {
- return false;
- }
- } else {
- return f.setExecutable(executable);
- }
- }
-
- /**
- * Platform independent implementation for {@link File#canRead()}
- * @param f input file
- * @return On Unix, same as {@link File#canRead()}
- * On Windows, true if process has read access on the path
- */
- public static boolean canRead(File f) {
- if (Shell.WINDOWS) {
- try {
- return NativeIO.Windows.access(f.getCanonicalPath(),
- NativeIO.Windows.AccessRight.ACCESS_READ);
- } catch (IOException e) {
- return false;
- }
- } else {
- return f.canRead();
- }
- }
-
- /**
- * Platform independent implementation for {@link File#canWrite()}
- * @param f input file
- * @return On Unix, same as {@link File#canWrite()}
- * On Windows, true if process has write access on the path
- */
- public static boolean canWrite(File f) {
- if (Shell.WINDOWS) {
- try {
- return NativeIO.Windows.access(f.getCanonicalPath(),
- NativeIO.Windows.AccessRight.ACCESS_WRITE);
- } catch (IOException e) {
- return false;
- }
- } else {
- return f.canWrite();
- }
- }
-
- /**
- * Platform independent implementation for {@link File#canExecute()}
- * @param f input file
- * @return On Unix, same as {@link File#canExecute()}
- * On Windows, true if process has execute access on the path
- */
- public static boolean canExecute(File f) {
- if (Shell.WINDOWS) {
- try {
- return NativeIO.Windows.access(f.getCanonicalPath(),
- NativeIO.Windows.AccessRight.ACCESS_EXECUTE);
- } catch (IOException e) {
- return false;
- }
- } else {
- try {
- return f.canExecute();
- } catch (SecurityException e) {
- return false;
- }
- }
- }
-
- /**
- * Set permissions to the required value. Uses the java primitives instead
- * of forking if group == other.
- * @param f the file to change
- * @param permission the new permissions
- * @throws IOException exception on setPermission
- */
- public static void setPermission(File f, FsPermission permission
- ) throws IOException {
- FsAction user = permission.getUserAction();
- FsAction group = permission.getGroupAction();
- FsAction other = permission.getOtherAction();
-
- // use the native/fork if the group/other permissions are different
- // or if the native is available or on Windows
- if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
- execSetPermission(f, permission);
- return;
- }
-
- boolean rv = true;
-
- // read perms
- rv = f.setReadable(group.implies(FsAction.READ), false);
- checkReturnValue(rv, f, permission);
- if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
- rv = f.setReadable(user.implies(FsAction.READ), true);
- checkReturnValue(rv, f, permission);
- }
-
- // write perms
- rv = f.setWritable(group.implies(FsAction.WRITE), false);
- checkReturnValue(rv, f, permission);
- if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
- rv = f.setWritable(user.implies(FsAction.WRITE), true);
- checkReturnValue(rv, f, permission);
- }
-
- // exec perms
- rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
- checkReturnValue(rv, f, permission);
- if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
- rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
- checkReturnValue(rv, f, permission);
- }
- }
-
- private static void checkReturnValue(boolean rv, File p,
- FsPermission permission
- ) throws IOException {
- if (!rv) {
- throw new IOException("Failed to set permissions of path: " + p +
- " to " +
- String.format("%04o", permission.toShort()));
- }
- }
-
- private static void execSetPermission(File f,
- FsPermission permission
- ) throws IOException {
- if (NativeIO.isAvailable()) {
- NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
- } else {
- execCommand(f, Shell.getSetPermissionCommand(
- String.format("%04o", permission.toShort()), false));
- }
- }
-
- static String execCommand(File f, String... cmd) throws IOException {
- String[] args = new String[cmd.length + 1];
- System.arraycopy(cmd, 0, args, 0, cmd.length);
- args[cmd.length] = f.getCanonicalPath();
- String output = Shell.execCommand(args);
- return output;
- }
-
- /**
- * Create a tmp file for a base file.
- * @param basefile the base file of the tmp
- * @param prefix file name prefix of tmp
- * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
- * @return a newly created tmp file
- * @exception IOException If a tmp file cannot created
- * @see java.io.File#createTempFile(String, String, File)
- * @see java.io.File#deleteOnExit()
- */
- public static final File createLocalTempFile(final File basefile,
- final String prefix,
- final boolean isDeleteOnExit)
- throws IOException {
- File tmp = File.createTempFile(prefix + basefile.getName(),
- "", basefile.getParentFile());
- if (isDeleteOnExit) {
- tmp.deleteOnExit();
- }
- return tmp;
- }
-
- /**
- * Move the src file to the name specified by target.
- * @param src the source file
- * @param target the target file
- * @exception IOException If this operation fails
- */
- public static void replaceFile(File src, File target) throws IOException {
- /* renameTo() has two limitations on Windows platform.
- * src.renameTo(target) fails if
- * 1) If target already exists OR
- * 2) If target is already open for reading/writing.
- */
- if (!src.renameTo(target)) {
- int retries = 5;
- while (target.exists() && !target.delete() && retries-- >= 0) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- throw new IOException("replaceFile interrupted.");
- }
- }
- if (!src.renameTo(target)) {
- throw new IOException("Unable to rename " + src +
- " to " + target);
- }
- }
- }
-
- /**
- * A wrapper for {@link File#listFiles()}. This java.io API returns null
- * when a dir is not a directory or for any I/O error. Instead of having
- * null check everywhere File#listFiles() is used, we will add utility API
- * to get around this problem. For the majority of cases where we prefer
- * an IOException to be thrown.
- * @param dir directory for which listing should be performed
- * @return list of files or empty list
- * @exception IOException for invalid directory or for a bad disk.
- */
- public static File[] listFiles(File dir) throws IOException {
- File[] files = dir.listFiles();
- if(files == null) {
- throw new IOException("Invalid directory or I/O error occurred for dir: "
- + dir.toString());
- }
- return files;
- }
-
- /**
- * A wrapper for {@link File#list()}. This java.io API returns null
- * when a dir is not a directory or for any I/O error. Instead of having
- * null check everywhere File#list() is used, we will add utility API
- * to get around this problem. For the majority of cases where we prefer
- * an IOException to be thrown.
- * @param dir directory for which listing should be performed
- * @return list of file names or empty string list
- * @exception AccessDeniedException for unreadable directory
- * @exception IOException for invalid directory or for bad disk
- */
- public static String[] list(File dir) throws IOException {
- if (!canRead(dir)) {
- throw new AccessDeniedException(dir.toString(), null,
- FSExceptionMessages.PERMISSION_DENIED);
- }
- String[] fileNames = dir.list();
- if(fileNames == null) {
- throw new IOException("Invalid directory or I/O error occurred for dir: "
- + dir.toString());
- }
- return fileNames;
- }
-
- public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
- Map<String, String> callerEnv) throws IOException {
- return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv);
- }
-
- /**
- * Create a jar file at the given path, containing a manifest with a classpath
- * that references all specified entries.
- *
- * Some platforms may have an upper limit on command line length. For example,
- * the maximum command line length on Windows is 8191 characters, but the
- * length of the classpath may exceed this. To work around this limitation,
- * use this method to create a small intermediate jar with a manifest that
- * contains the full classpath. It returns the absolute path to the new jar,
- * which the caller may set as the classpath for a new process.
- *
- * Environment variable evaluation is not supported within a jar manifest, so
- * this method expands environment variables before inserting classpath entries
- * to the manifest. The method parses environment variables according to
- * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows,
- * environment variables are case-insensitive. For example, %VAR% and %var%
- * evaluate to the same value.
- *
- * Specifying the classpath in a jar manifest does not support wildcards, so
- * this method expands wildcards internally. Any classpath entry that ends
- * with * is translated to all files at that path with extension .jar or .JAR.
- *
- * @param inputClassPath String input classpath to bundle into the jar manifest
- * @param pwd Path to working directory to save jar
- * @param targetDir path to where the jar execution will have its working dir
- * @param callerEnv Map {@literal <}String, String{@literal >} caller's
- * environment variables to use for expansion
- * @return String[] with absolute path to new jar in position 0 and
- * unexpanded wild card entry path in position 1
- * @throws IOException if there is an I/O error while writing the jar file
- */
- public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
- Path targetDir,
- Map<String, String> callerEnv) throws IOException {
- // Replace environment variables, case-insensitive on Windows
- @SuppressWarnings("unchecked")
- Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap<>(callerEnv) :
- callerEnv;
- String[] classPathEntries = inputClassPath.split(File.pathSeparator);
- for (int i = 0; i < classPathEntries.length; ++i) {
- classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
- StringUtils.ENV_VAR_PATTERN, env);
- }
- File workingDir = new File(pwd.toString());
- if (!workingDir.mkdirs()) {
- // If mkdirs returns false because the working directory already exists,
- // then this is acceptable. If it returns false due to some other I/O
- // error, then this method will fail later with an IOException while saving
- // the jar.
- LOG.debug("mkdirs false for {}, execution will continue", workingDir);
- }
-
- StringBuilder unexpandedWildcardClasspath = new StringBuilder();
- // Append all entries
- List<String> classPathEntryList = new ArrayList<String>(
- classPathEntries.length);
- for (String classPathEntry: classPathEntries) {
- if (classPathEntry.length() == 0) {
- continue;
- }
- if (classPathEntry.endsWith("*")) {
- // Append all jars that match the wildcard
- List<Path> jars = getJarsInDirectory(classPathEntry);
- if (!jars.isEmpty()) {
- for (Path jar: jars) {
- classPathEntryList.add(jar.toUri().toURL().toExternalForm());
- }
- } else {
- unexpandedWildcardClasspath.append(File.pathSeparator)
- .append(classPathEntry);
- }
- } else {
- // Append just this entry
- File fileCpEntry = null;
- if(!new Path(classPathEntry).isAbsolute()) {
- fileCpEntry = new File(targetDir.toString(), classPathEntry);
- }
- else {
- fileCpEntry = new File(classPathEntry);
- }
- String classPathEntryUrl = fileCpEntry.toURI().toURL()
- .toExternalForm();
-
- // File.toURI only appends trailing '/' if it can determine that it is a
- // directory that already exists. (See JavaDocs.) If this entry had a
- // trailing '/' specified by the caller, then guarantee that the
- // classpath entry in the manifest has a trailing '/', and thus refers to
- // a directory instead of a file. This can happen if the caller is
- // creating a classpath jar referencing a directory that hasn't been
- // created yet, but will definitely be created before running.
- if (classPathEntry.endsWith(Path.SEPARATOR) &&
- !classPathEntryUrl.endsWith(Path.SEPARATOR)) {
- classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR;
- }
- classPathEntryList.add(classPathEntryUrl);
- }
- }
- String jarClassPath = StringUtils.join(" ", classPathEntryList);
-
- // Create the manifest
- Manifest jarManifest = new Manifest();
- jarManifest.getMainAttributes().putValue(
- Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
- jarManifest.getMainAttributes().putValue(
- Attributes.Name.CLASS_PATH.toString(), jarClassPath);
-
- // Write the manifest to output JAR file
- File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
- try (OutputStream fos = Files.newOutputStream(classPathJar.toPath());
- BufferedOutputStream bos = new BufferedOutputStream(fos)) {
- JarOutputStream jos = new JarOutputStream(bos, jarManifest);
- jos.close();
- }
- String[] jarCp = {classPathJar.getCanonicalPath(),
- unexpandedWildcardClasspath.toString()};
- return jarCp;
- }
-
- /**
- * Returns all jars that are in the directory. It is useful in expanding a
- * wildcard path to return all jars from the directory to use in a classpath.
- * It operates only on local paths.
- *
- * @param path the path to the directory. The path may include the wildcard.
- * @return the list of jars as URLs, or an empty list if there are no jars, or
- * the directory does not exist locally
- */
- public static List<Path> getJarsInDirectory(String path) {
- return getJarsInDirectory(path, true);
- }
-
- /**
- * Returns all jars that are in the directory. It is useful in expanding a
- * wildcard path to return all jars from the directory to use in a classpath.
- *
- * @param path the path to the directory. The path may include the wildcard.
- * @return the list of jars as URLs, or an empty list if there are no jars, or
- * the directory does not exist
- */
- public static List<Path> getJarsInDirectory(String path, boolean useLocal) {
- List<Path> paths = new ArrayList<>();
- try {
- // add the wildcard if it is not provided
- if (!path.endsWith("*")) {
- path += File.separator + "*";
- }
- Path globPath = new Path(path).suffix("{.jar,.JAR}");
- FileContext context = useLocal ?
- FileContext.getLocalFSFileContext() :
- FileContext.getFileContext(globPath.toUri());
- FileStatus[] files = context.util().globStatus(globPath);
- if (files != null) {
- for (FileStatus file: files) {
- paths.add(file.getPath());
- }
- }
- } catch (IOException ignore) {} // return the empty list
- return paths;
- }
-
- public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
- if (srcFs==null || destFs==null) {
- return false;
- }
- URI srcUri = srcFs.getUri();
- URI dstUri = destFs.getUri();
- if (srcUri.getScheme()==null) {
- return false;
- }
- if (!srcUri.getScheme().equals(dstUri.getScheme())) {
- return false;
- }
- String srcHost = srcUri.getHost();
- String dstHost = dstUri.getHost();
- if ((srcHost!=null) && (dstHost!=null)) {
- if (srcHost.equals(dstHost)) {
- return srcUri.getPort()==dstUri.getPort();
- }
- try {
- srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
- dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
- } catch (UnknownHostException ue) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Could not compare file-systems. Unknown host: ", ue);
- }
- return false;
- }
- if (!srcHost.equals(dstHost)) {
- return false;
- }
- } else if (srcHost==null && dstHost!=null) {
- return false;
- } else if (srcHost!=null) {
- return false;
- }
- // check for ports
- return srcUri.getPort()==dstUri.getPort();
- }
-
- /**
- * Writes bytes to a file. This utility method opens the file for writing,
- * creating the file if it does not exist, or overwrites an existing file. All
- * bytes in the byte array are written to the file.
- *
- * @param fs the file system with which to create the file
- * @param path the path to the file
- * @param bytes the byte array with the bytes to write
- *
- * @return the file system
- *
- * @throws NullPointerException if any of the arguments are {@code null}
- * @throws IOException if an I/O error occurs creating or writing to the file
- */
- public static FileSystem write(final FileSystem fs, final Path path,
- final byte[] bytes) throws IOException {
-
- Objects.requireNonNull(path);
- Objects.requireNonNull(bytes);
-
- try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build()) {
- out.write(bytes);
- }
-
- return fs;
- }
-
- /**
- * Writes bytes to a file. This utility method opens the file for writing,
- * creating the file if it does not exist, or overwrites an existing file. All
- * bytes in the byte array are written to the file.
- *
- * @param fileContext the file context with which to create the file
- * @param path the path to the file
- * @param bytes the byte array with the bytes to write
- *
- * @return the file context
- *
- * @throws NullPointerException if any of the arguments are {@code null}
- * @throws IOException if an I/O error occurs creating or writing to the file
- */
- public static FileContext write(final FileContext fileContext,
- final Path path, final byte[] bytes) throws IOException {
-
- Objects.requireNonNull(path);
- Objects.requireNonNull(bytes);
-
- try (FSDataOutputStream out =
- fileContext.create(path).overwrite(true).build()) {
- out.write(bytes);
- }
-
- return fileContext;
- }
-
- /**
- * Write lines of text to a file. Each line is a char sequence and is written
- * to the file in sequence with each line terminated by the platform's line
- * separator, as defined by the system property {@code
- * line.separator}. Characters are encoded into bytes using the specified
- * charset. This utility method opens the file for writing, creating the file
- * if it does not exist, or overwrites an existing file.
- *
- * @param fs the file system with which to create the file
- * @param path the path to the file
- * @param lines a Collection to iterate over the char sequences
- * @param cs the charset to use for encoding
- *
- * @return the file system
- *
- * @throws NullPointerException if any of the arguments are {@code null}
- * @throws IOException if an I/O error occurs creating or writing to the file
- */
- public static FileSystem write(final FileSystem fs, final Path path,
- final Iterable<? extends CharSequence> lines, final Charset cs)
- throws IOException {
-
- Objects.requireNonNull(path);
- Objects.requireNonNull(lines);
- Objects.requireNonNull(cs);
-
- CharsetEncoder encoder = cs.newEncoder();
- try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build();
- BufferedWriter writer =
- new BufferedWriter(new OutputStreamWriter(out, encoder))) {
- for (CharSequence line : lines) {
- writer.append(line);
- writer.newLine();
- }
- }
- return fs;
- }
-
- /**
- * Write lines of text to a file. Each line is a char sequence and is written
- * to the file in sequence with each line terminated by the platform's line
- * separator, as defined by the system property {@code
- * line.separator}. Characters are encoded into bytes using the specified
- * charset. This utility method opens the file for writing, creating the file
- * if it does not exist, or overwrites an existing file.
- *
- * @param fileContext the file context with which to create the file
- * @param path the path to the file
- * @param lines a Collection to iterate over the char sequences
- * @param cs the charset to use for encoding
- *
- * @return the file context
- *
- * @throws NullPointerException if any of the arguments are {@code null}
- * @throws IOException if an I/O error occurs creating or writing to the file
- */
- public static FileContext write(final FileContext fileContext,
- final Path path, final Iterable<? extends CharSequence> lines,
- final Charset cs) throws IOException {
-
- Objects.requireNonNull(path);
- Objects.requireNonNull(lines);
- Objects.requireNonNull(cs);
-
- CharsetEncoder encoder = cs.newEncoder();
- try (FSDataOutputStream out = fileContext.create(path).overwrite(true).build();
- BufferedWriter writer =
- new BufferedWriter(new OutputStreamWriter(out, encoder))) {
- for (CharSequence line : lines) {
- writer.append(line);
- writer.newLine();
- }
- }
- return fileContext;
- }
-
- /**
- * Write a line of text to a file. Characters are encoded into bytes using the
- * specified charset. This utility method opens the file for writing, creating
- * the file if it does not exist, or overwrites an existing file.
- *
- * @param fs the file system with which to create the file
- * @param path the path to the file
- * @param charseq the char sequence to write to the file
- * @param cs the charset to use for encoding
- *
- * @return the file system
- *
- * @throws NullPointerException if any of the arguments are {@code null}
- * @throws IOException if an I/O error occurs creating or writing to the file
- */
- public static FileSystem write(final FileSystem fs, final Path path,
- final CharSequence charseq, final Charset cs) throws IOException {
-
- Objects.requireNonNull(path);
- Objects.requireNonNull(charseq);
- Objects.requireNonNull(cs);
-
- CharsetEncoder encoder = cs.newEncoder();
- try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build();
- BufferedWriter writer =
- new BufferedWriter(new OutputStreamWriter(out, encoder))) {
- writer.append(charseq);
- }
- return fs;
- }
-
- /**
- * Write a line of text to a file. Characters are encoded into bytes using the
- * specified charset. This utility method opens the file for writing, creating
- * the file if it does not exist, or overwrites an existing file.
- *
- * @param fs the file context with which to create the file
- * @param path the path to the file
- * @param charseq the char sequence to write to the file
- * @param cs the charset to use for encoding
- *
- * @return the file context
- *
- * @throws NullPointerException if any of the arguments are {@code null}
- * @throws IOException if an I/O error occurs creating or writing to the file
- */
- public static FileContext write(final FileContext fs, final Path path,
- final CharSequence charseq, final Charset cs) throws IOException {
-
- Objects.requireNonNull(path);
- Objects.requireNonNull(charseq);
- Objects.requireNonNull(cs);
-
- CharsetEncoder encoder = cs.newEncoder();
- try (FSDataOutputStream out = fs.create(path).overwrite(true).build();
- BufferedWriter writer =
- new BufferedWriter(new OutputStreamWriter(out, encoder))) {
- writer.append(charseq);
- }
- return fs;
- }
-
- /**
- * Write a line of text to a file. Characters are encoded into bytes using
- * UTF-8. This utility method opens the file for writing, creating the file if
- * it does not exist, or overwrites an existing file.
- *
- * @param fs the files system with which to create the file
- * @param path the path to the file
- * @param charseq the char sequence to write to the file
- *
- * @return the file system
- *
- * @throws NullPointerException if any of the arguments are {@code null}
- * @throws IOException if an I/O error occurs creating or writing to the file
- */
- public static FileSystem write(final FileSystem fs, final Path path,
- final CharSequence charseq) throws IOException {
- return write(fs, path, charseq, StandardCharsets.UTF_8);
- }
-
- /**
- * Write a line of text to a file. Characters are encoded into bytes using
- * UTF-8. This utility method opens the file for writing, creating the file if
- * it does not exist, or overwrites an existing file.
- *
- * @param fileContext the files system with which to create the file
- * @param path the path to the file
- * @param charseq the char sequence to write to the file
- *
- * @return the file context
- *
- * @throws NullPointerException if any of the arguments are {@code null}
- * @throws IOException if an I/O error occurs creating or writing to the file
- */
- public static FileContext write(final FileContext fileContext,
- final Path path, final CharSequence charseq) throws IOException {
- return write(fileContext, path, charseq, StandardCharsets.UTF_8);
- }
-
- /**
- * Used in ViewDistributedFileSystem rename API to get access to the protected
- * API of FileSystem interface. Even though Rename with options API
- * deprecated, we are still using as part of trash. If any filesystem provided
- * implementation to this protected FileSystem API, we can't invoke it with
- * out casting to the specific filesystem. This util method is proposed to get
- * the access to FileSystem#rename with options.
- */
- @InterfaceAudience.LimitedPrivate({"ViewDistributedFileSystem"})
- @InterfaceStability.Unstable
- @SuppressWarnings("deprecation")
- public static void rename(FileSystem srcFs, Path src, Path dst,
- final Options.Rename... options) throws IOException {
- srcFs.rename(src, dst, options);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/HardLink.java b/solr/modules/hdfs/src/test/org/apache/hadoop/fs/HardLink.java
deleted file mode 100644
index 2532d283d9c..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/HardLink.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.file.Files;
-
-import static java.nio.file.Files.createLink;
-
-/**
- * Class for creating hardlinks.
- * Supports Unix/Linux, Windows via winutils , and Mac OS X.
- *
- * The HardLink class was formerly a static inner class of FSUtil,
- * and the methods provided were blatantly non-thread-safe.
- * To enable volume-parallel Update snapshots, we now provide static
- * threadsafe methods that allocate new buffer string arrays
- * upon each call. We also provide an API to hardlink all files in a
- * directory with a single command, which is up to 128 times more
- * efficient - and minimizes the impact of the extra buffer creations.
- */
-public class HardLink {
- public static final Object SOLR_HACK_FOR_CLASS_VERIFICATION = new Object();
-
- public final LinkStats linkStats; //not static
-
- public HardLink() {
- linkStats = new LinkStats();
- }
-
- /*
- * ****************************************************
- * Complexity is above. User-visible functionality is below
- * ****************************************************
- */
-
- /**
- * Creates a hardlink
- * @param file - existing source file
- * @param linkName - desired target link file
- */
- public static void createHardLink(File file, File linkName)
- throws IOException {
- if (file == null) {
- throw new IOException(
- "invalid arguments to createHardLink: source file is null");
- }
- if (linkName == null) {
- throw new IOException(
- "invalid arguments to createHardLink: link name is null");
- }
- createLink(linkName.toPath(), file.toPath());
- }
-
- /**
- * Creates hardlinks from multiple existing files within one parent
- * directory, into one target directory.
- * @param parentDir - directory containing source files
- * @param fileBaseNames - list of path-less file names, as returned by
- * parentDir.list()
- * @param linkDir - where the hardlinks should be put. It must already exist.
- */
- public static void createHardLinkMult(File parentDir, String[] fileBaseNames,
- File linkDir) throws IOException {
- if (parentDir == null) {
- throw new IOException(
- "invalid arguments to createHardLinkMult: parent directory is null");
- }
- if (linkDir == null) {
- throw new IOException(
- "invalid arguments to createHardLinkMult: link directory is null");
- }
- if (fileBaseNames == null) {
- throw new IOException(
- "invalid arguments to createHardLinkMult: "
- + "filename list can be empty but not null");
- }
- if (!linkDir.exists()) {
- throw new FileNotFoundException(linkDir + " not found.");
- }
- for (String name : fileBaseNames) {
- createLink(linkDir.toPath().resolve(name),
- parentDir.toPath().resolve(name));
- }
- }
-
- /**
- * Retrieves the number of links to the specified file.
- */
- public static int getLinkCount(File fileName) throws IOException {
- if (fileName == null) {
- throw new IOException(
- "invalid argument to getLinkCount: file name is null");
- }
- if (!fileName.exists()) {
- throw new FileNotFoundException(fileName + " not found.");
- }
-
- return (Integer)Files.getAttribute(fileName.toPath(), "unix:nlink");
- }
-
- /* Create an IOException for failing to get link count. */
- private static IOException createIOException(File f, String message,
- String error, int exitvalue, Exception cause) {
-
- final String s = "Failed to get link count on file " + f
- + ": message=" + message
- + "; error=" + error
- + "; exit value=" + exitvalue;
- return (cause == null) ? new IOException(s) : new IOException(s, cause);
- }
-
-
- /**
- * HardLink statistics counters and methods.
- * Not multi-thread safe, obviously.
- * Init is called during HardLink instantiation, above.
- *
- * These are intended for use by knowledgeable clients, not internally,
- * because many of the internal methods are static and can't update these
- * per-instance counters.
- */
- public static class LinkStats {
- public int countDirs = 0;
- public int countSingleLinks = 0;
- public int countMultLinks = 0;
- public int countFilesMultLinks = 0;
- public int countEmptyDirs = 0;
- public int countPhysicalFileCopies = 0;
-
- public void clear() {
- countDirs = 0;
- countSingleLinks = 0;
- countMultLinks = 0;
- countFilesMultLinks = 0;
- countEmptyDirs = 0;
- countPhysicalFileCopies = 0;
- }
-
- public String report() {
- return "HardLinkStats: " + countDirs + " Directories, including "
- + countEmptyDirs + " Empty Directories, "
- + countSingleLinks
- + " single Link operations, " + countMultLinks
- + " multi-Link operations, linking " + countFilesMultLinks
- + " files, total " + (countSingleLinks + countFilesMultLinks)
- + " linkable files. Also physically copied "
- + countPhysicalFileCopies + " other files.";
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/RawLocalFileSystem.java b/solr/modules/hdfs/src/test/org/apache/hadoop/fs/RawLocalFileSystem.java
deleted file mode 100644
index 52b6a18b142..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ /dev/null
@@ -1,1178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.impl.StoreImplementationUtils;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.statistics.BufferedIOStatisticsOutputStream;
-import org.apache.hadoop.fs.statistics.IOStatistics;
-import org.apache.hadoop.fs.statistics.IOStatisticsSource;
-import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.solr.common.util.EnvUtils;
-
-import java.io.BufferedOutputStream;
-import java.io.DataOutput;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileDescriptor;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.LinkOption;
-import java.nio.file.NoSuchFileException;
-import java.nio.file.Paths;
-import java.nio.file.attribute.BasicFileAttributeView;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.nio.file.attribute.FileTime;
-import java.nio.file.attribute.PosixFileAttributes;
-import java.nio.file.attribute.PosixFilePermissions;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.Locale;
-import java.util.Optional;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
-import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_BYTES;
-import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_EXCEPTIONS;
-import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_SEEK_OPERATIONS;
-import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_SKIP_BYTES;
-import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_SKIP_OPERATIONS;
-import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_WRITE_BYTES;
-import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_WRITE_EXCEPTIONS;
-import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore;
-
-/****************************************************************
- * Implement the FileSystem API for the raw local filesystem.
- *
- *****************************************************************/
-public class RawLocalFileSystem extends FileSystem {
- public static final Object SOLR_HACK_FOR_CLASS_VERIFICATION = new Object();
-
- static final URI NAME = URI.create("file:///");
- private Path workingDir;
- private long defaultBlockSize;
- // Temporary workaround for HADOOP-9652.
- private static boolean useDeprecatedFileStatus = true;
-
- @VisibleForTesting
- public static void useStatIfAvailable() {
- useDeprecatedFileStatus = !Stat.isAvailable();
- }
-
- public RawLocalFileSystem() {
- workingDir = getInitialWorkingDirectory();
- }
-
- private Path makeAbsolute(Path f) {
- if (f.isAbsolute()) {
- return f;
- } else {
- return new Path(workingDir, f);
- }
- }
-
- /** Convert a path to a File. */
- public File pathToFile(Path path) {
- checkPath(path);
- if (!path.isAbsolute()) {
- path = new Path(getWorkingDirectory(), path);
- }
- return new File(path.toUri().getPath());
- }
-
- @Override
- public URI getUri() { return NAME; }
-
- @Override
- public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
- setConf(conf);
- defaultBlockSize = getDefaultBlockSize(new Path(uri));
- }
-
- /*******************************************************
- * For open()'s FSInputStream.
- *******************************************************/
- class LocalFSFileInputStream extends FSInputStream implements
- HasFileDescriptor, IOStatisticsSource, StreamCapabilities {
- private FileInputStream fis;
- private long position;
-
- /**
- * Minimal set of counters.
- */
- private final IOStatisticsStore ioStatistics = iostatisticsStore()
- .withCounters(
- STREAM_READ_BYTES,
- STREAM_READ_EXCEPTIONS,
- STREAM_READ_SEEK_OPERATIONS,
- STREAM_READ_SKIP_OPERATIONS,
- STREAM_READ_SKIP_BYTES)
- .build();
-
- /** Reference to the bytes read counter for slightly faster counting. */
- private final AtomicLong bytesRead;
-
- public LocalFSFileInputStream(Path f) throws IOException {
- fis = new FileInputStream(pathToFile(f));
- bytesRead = ioStatistics.getCounterReference(
- STREAM_READ_BYTES);
- }
-
- @Override
- public void seek(long pos) throws IOException {
- if (pos < 0) {
- throw new EOFException(
- FSExceptionMessages.NEGATIVE_SEEK);
- }
- fis.getChannel().position(pos);
- this.position = pos;
- }
-
- @Override
- public long getPos() throws IOException {
- return this.position;
- }
-
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-
- /*
- * Just forward to the fis
- */
- @Override
- public int available() throws IOException { return fis.available(); }
- @Override
- public void close() throws IOException { fis.close(); }
- @Override
- public boolean markSupported() { return false; }
-
- @Override
- public int read() throws IOException {
- try {
- int value = fis.read();
- if (value >= 0) {
- this.position++;
- statistics.incrementBytesRead(1);
- bytesRead.addAndGet(1);
- }
- return value;
- } catch (IOException e) { // unexpected exception
- ioStatistics.incrementCounter(STREAM_READ_EXCEPTIONS);
- throw new FSError(e); // assume native fs error
- }
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- // parameter check
- validatePositionedReadArgs(position, b, off, len);
- try {
- int value = fis.read(b, off, len);
- if (value > 0) {
- this.position += value;
- statistics.incrementBytesRead(value);
- bytesRead.addAndGet(value);
- }
- return value;
- } catch (IOException e) { // unexpected exception
- ioStatistics.incrementCounter(STREAM_READ_EXCEPTIONS);
- throw new FSError(e); // assume native fs error
- }
- }
-
- @Override
- public int read(long position, byte[] b, int off, int len)
- throws IOException {
- // parameter check
- validatePositionedReadArgs(position, b, off, len);
- if (len == 0) {
- return 0;
- }
-
- ByteBuffer bb = ByteBuffer.wrap(b, off, len);
- try {
- int value = fis.getChannel().read(bb, position);
- if (value > 0) {
- statistics.incrementBytesRead(value);
- ioStatistics.incrementCounter(STREAM_READ_BYTES, value);
- }
- return value;
- } catch (IOException e) {
- ioStatistics.incrementCounter(STREAM_READ_EXCEPTIONS);
- throw new FSError(e);
- }
- }
-
- @Override
- public long skip(long n) throws IOException {
- ioStatistics.incrementCounter(STREAM_READ_SKIP_OPERATIONS);
- long value = fis.skip(n);
- if (value > 0) {
- this.position += value;
- ioStatistics.incrementCounter(STREAM_READ_SKIP_BYTES, value);
- }
- return value;
- }
-
- @Override
- public FileDescriptor getFileDescriptor() throws IOException {
- return fis.getFD();
- }
-
- @Override
- public boolean hasCapability(String capability) {
- // a bit inefficient, but intended to make it easier to add
- // new capabilities.
- switch (capability.toLowerCase(Locale.ENGLISH)) {
- case StreamCapabilities.IOSTATISTICS:
- return true;
- default:
- return false;
- }
- }
-
- @Override
- public IOStatistics getIOStatistics() {
- return ioStatistics;
- }
- }
-
- @Override
- public FSDataInputStream open(Path f, int bufferSize) throws IOException {
- getFileStatus(f);
- return new FSDataInputStream(new BufferedFSInputStream(
- new LocalFSFileInputStream(f), bufferSize));
- }
-
- @Override
- public FSDataInputStream open(PathHandle fd, int bufferSize)
- throws IOException {
- if (!(fd instanceof LocalFileSystemPathHandle)) {
- fd = new LocalFileSystemPathHandle(fd.bytes());
- }
- LocalFileSystemPathHandle id = (LocalFileSystemPathHandle) fd;
- id.verify(getFileStatus(new Path(id.getPath())));
- return new FSDataInputStream(new BufferedFSInputStream(
- new LocalFSFileInputStream(new Path(id.getPath())), bufferSize));
- }
-
- /*********************************************************
- * For create()'s FSOutputStream.
- *********************************************************/
- final class LocalFSFileOutputStream extends OutputStream implements
- IOStatisticsSource, StreamCapabilities, Syncable {
- private FileOutputStream fos;
-
- /**
- * Minimal set of counters.
- */
- private final IOStatisticsStore ioStatistics = iostatisticsStore()
- .withCounters(
- STREAM_WRITE_BYTES,
- STREAM_WRITE_EXCEPTIONS)
- .build();
-
- private LocalFSFileOutputStream(Path f, boolean append,
- FsPermission permission) throws IOException {
- File file = pathToFile(f);
- if (!append && permission == null) {
- permission = FsPermission.getFileDefault();
- }
- if (permission == null) {
- this.fos = new FileOutputStream(file, append);
- } else {
- permission = permission.applyUMask(FsPermission.getUMask(getConf()));
- if (Shell.WINDOWS && NativeIO.isAvailable()) {
- this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
- append, permission.toShort());
- } else {
- this.fos = new FileOutputStream(file, append);
- boolean success = false;
- try {
- setPermission(f, permission);
- success = true;
- } finally {
- if (!success) {
- IOUtils.cleanupWithLogger(LOG, this.fos);
- }
- }
- }
- }
- }
-
- /*
- * Just forward to the fos
- */
- @Override
- public void close() throws IOException { fos.close(); }
- @Override
- public void flush() throws IOException { fos.flush(); }
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- try {
- fos.write(b, off, len);
- ioStatistics.incrementCounter(STREAM_WRITE_BYTES, len);
- } catch (IOException e) { // unexpected exception
- ioStatistics.incrementCounter(STREAM_WRITE_EXCEPTIONS);
- throw new FSError(e); // assume native fs error
- }
- }
-
- @Override
- public void write(int b) throws IOException {
- try {
- fos.write(b);
- ioStatistics.incrementCounter(STREAM_WRITE_BYTES);
- } catch (IOException e) { // unexpected exception
- ioStatistics.incrementCounter(STREAM_WRITE_EXCEPTIONS);
- throw new FSError(e); // assume native fs error
- }
- }
-
- @Override
- public void hflush() throws IOException {
- flush();
- }
-
- /**
- * HSync calls sync on fhe file descriptor after a local flush() call.
- * @throws IOException failure
- */
- @Override
- public void hsync() throws IOException {
- flush();
- fos.getFD().sync();
- }
-
- @Override
- public boolean hasCapability(String capability) {
- // a bit inefficient, but intended to make it easier to add
- // new capabilities.
- switch (capability.toLowerCase(Locale.ENGLISH)) {
- case StreamCapabilities.IOSTATISTICS:
- return true;
- default:
- return StoreImplementationUtils.isProbeForSyncable(capability);
- }
- }
-
- @Override
- public IOStatistics getIOStatistics() {
- return ioStatistics;
- }
- }
-
- @Override
- public FSDataOutputStream append(Path f, int bufferSize,
- Progressable progress) throws IOException {
- FileStatus status = getFileStatus(f);
- if (status.isDirectory()) {
- throw new IOException("Cannot append to a diretory (=" + f + " )");
- }
- return new FSDataOutputStream(new BufferedOutputStream(
- createOutputStreamWithMode(f, true, null), bufferSize), statistics,
- status.getLen());
- }
-
- @Override
- public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
- short replication, long blockSize, Progressable progress)
- throws IOException {
- return create(f, overwrite, true, bufferSize, replication, blockSize,
- progress, null);
- }
-
- private FSDataOutputStream create(Path f, boolean overwrite,
- boolean createParent, int bufferSize, short replication, long blockSize,
- Progressable progress, FsPermission permission) throws IOException {
- if (exists(f) && !overwrite) {
- throw new FileAlreadyExistsException("File already exists: " + f);
- }
- Path parent = f.getParent();
- if (parent != null && !mkdirs(parent)) {
- throw new IOException("Mkdirs failed to create " + parent.toString());
- }
- return new FSDataOutputStream(new BufferedIOStatisticsOutputStream(
- createOutputStreamWithMode(f, false, permission), bufferSize, true),
- statistics);
- }
-
- protected OutputStream createOutputStream(Path f, boolean append)
- throws IOException {
- return createOutputStreamWithMode(f, append, null);
- }
-
- protected OutputStream createOutputStreamWithMode(Path f, boolean append,
- FsPermission permission) throws IOException {
- return new LocalFSFileOutputStream(f, append, permission);
- }
-
- @Override
- public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
- EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
- if (exists(f) && !flags.contains(CreateFlag.OVERWRITE)) {
- throw new FileAlreadyExistsException("File already exists: " + f);
- }
- return new FSDataOutputStream(new BufferedIOStatisticsOutputStream(
- createOutputStreamWithMode(f, false, permission), bufferSize, true),
- statistics);
- }
-
- @Override
- public FSDataOutputStream create(Path f, FsPermission permission,
- boolean overwrite, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
-
- FSDataOutputStream out = create(f, overwrite, true, bufferSize, replication,
- blockSize, progress, permission);
- return out;
- }
-
- @Override
- public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
- boolean overwrite,
- int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
- FSDataOutputStream out = create(f, overwrite, false, bufferSize, replication,
- blockSize, progress, permission);
- return out;
- }
-
- @Override
- public void concat(final Path trg, final Path [] psrcs) throws IOException {
- final int bufferSize = 4096;
- try(FSDataOutputStream out = create(trg)) {
- for (Path src : psrcs) {
- try(FSDataInputStream in = open(src)) {
- IOUtils.copyBytes(in, out, bufferSize, false);
- }
- }
- }
- }
-
- @Override
- public boolean rename(Path src, Path dst) throws IOException {
- // Attempt rename using Java API.
- File srcFile = pathToFile(src);
- File dstFile = pathToFile(dst);
- if (srcFile.renameTo(dstFile)) {
- return true;
- }
-
- // Else try POSIX style rename on Windows only
- if (Shell.WINDOWS &&
- handleEmptyDstDirectoryOnWindows(src, srcFile, dst, dstFile)) {
- return true;
- }
-
- // The fallback behavior accomplishes the rename by a full copy.
- if (LOG.isDebugEnabled()) {
- LOG.debug("Falling through to a copy of " + src + " to " + dst);
- }
- return FileUtil.copy(this, src, this, dst, true, getConf());
- }
-
- @VisibleForTesting
- public final boolean handleEmptyDstDirectoryOnWindows(Path src, File srcFile,
- Path dst, File dstFile) throws IOException {
-
- // Enforce POSIX rename behavior that a source directory replaces an
- // existing destination if the destination is an empty directory. On most
- // platforms, this is already handled by the Java API call above. Some
- // platforms (notably Windows) do not provide this behavior, so the Java API
- // call renameTo(dstFile) fails. Delete destination and attempt rename
- // again.
- try {
- FileStatus sdst = this.getFileStatus(dst);
- String[] dstFileList = dstFile.list();
- if (dstFileList != null) {
- if (sdst.isDirectory() && dstFileList.length == 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleting empty destination and renaming " + src +
- " to " + dst);
- }
- if (this.delete(dst, false) && srcFile.renameTo(dstFile)) {
- return true;
- }
- }
- }
- } catch (FileNotFoundException ignored) {
- }
- return false;
- }
-
- @Override
- public boolean truncate(Path f, final long newLength) throws IOException {
- FileStatus status = getFileStatus(f);
- if(status == null) {
- throw new FileNotFoundException("File " + f + " not found");
- }
- if(status.isDirectory()) {
- throw new IOException("Cannot truncate a directory (=" + f + ")");
- }
- long oldLength = status.getLen();
- if(newLength > oldLength) {
- throw new IllegalArgumentException(
- "Cannot truncate to a larger file size. Current size: " + oldLength +
- ", truncate size: " + newLength + ".");
- }
- try (FileOutputStream out = new FileOutputStream(pathToFile(f), true)) {
- try {
- out.getChannel().truncate(newLength);
- } catch(IOException e) {
- throw new FSError(e);
- }
- }
- return true;
- }
-
- /**
- * Delete the given path to a file or directory.
- * @param p the path to delete
- * @param recursive to delete sub-directories
- * @return true if the file or directory and all its contents were deleted
- * @throws IOException if p is non-empty and recursive is false
- */
- @Override
- public boolean delete(Path p, boolean recursive) throws IOException {
- File f = pathToFile(p);
- if (!f.exists()) {
- //no path, return false "nothing to delete"
- return false;
- }
- if (f.isFile()) {
- return f.delete();
- } else if (!recursive && f.isDirectory() &&
- (FileUtil.listFiles(f).length != 0)) {
- throw new IOException("Directory " + f.toString() + " is not empty");
- }
- return FileUtil.fullyDelete(f);
- }
-
- /**
- * {@inheritDoc}
- *
- * (<b>Note</b>: Returned list is not sorted in any given order,
- * due to reliance on Java's {@link File#list()} API.)
- */
- @Override
- public FileStatus[] listStatus(Path f) throws IOException {
- File localf = pathToFile(f);
- FileStatus[] results;
-
- if (!localf.exists()) {
- throw new FileNotFoundException("File " + f + " does not exist");
- }
-
- if (localf.isDirectory()) {
- String[] names = FileUtil.list(localf);
- results = new FileStatus[names.length];
- int j = 0;
- for (int i = 0; i < names.length; i++) {
- try {
- // Assemble the path using the Path 3 arg constructor to make sure
- // paths with colon are properly resolved on Linux
- results[j] = getFileStatus(new Path(f, new Path(null, null,
- names[i])));
- j++;
- } catch (FileNotFoundException e) {
- // ignore the files not found since the dir list may have have
- // changed since the names[] list was generated.
- }
- }
- if (j == names.length) {
- return results;
- }
- return Arrays.copyOf(results, j);
- }
-
- if (!useDeprecatedFileStatus) {
- return new FileStatus[] { getFileStatus(f) };
- }
- return new FileStatus[] {
- new DeprecatedRawLocalFileStatus(localf,
- defaultBlockSize, this) };
- }
-
- @Override
- public boolean exists(Path f) throws IOException {
- return pathToFile(f).exists();
- }
-
- protected boolean mkOneDir(File p2f) throws IOException {
- return mkOneDirWithMode(new Path(p2f.getAbsolutePath()), p2f, null);
- }
-
- protected boolean mkOneDirWithMode(Path p, File p2f, FsPermission permission)
- throws IOException {
- if (permission == null) {
- permission = FsPermission.getDirDefault();
- }
- permission = permission.applyUMask(FsPermission.getUMask(getConf()));
- if (Shell.WINDOWS && NativeIO.isAvailable()) {
- try {
- NativeIO.Windows.createDirectoryWithMode(p2f, permission.toShort());
- return true;
- } catch (IOException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format(
- "NativeIO.createDirectoryWithMode error, path = %s, mode = %o",
- p2f, permission.toShort()), e);
- }
- return false;
- }
- } else {
- boolean b = p2f.mkdir();
- if (b) {
- setPermission(p, permission);
- }
- return b;
- }
- }
-
- /**
- * Creates the specified directory hierarchy. Does not
- * treat existence as an error.
- */
- @Override
- public boolean mkdirs(Path f) throws IOException {
- return mkdirsWithOptionalPermission(f, null);
- }
-
- @Override
- public boolean mkdirs(Path f, FsPermission permission) throws IOException {
- return mkdirsWithOptionalPermission(f, permission);
- }
-
- private boolean mkdirsWithOptionalPermission(Path f, FsPermission permission)
- throws IOException {
- if(f == null) {
- throw new IllegalArgumentException("mkdirs path arg is null");
- }
- Path parent = f.getParent();
- File p2f = pathToFile(f);
- File parent2f = null;
- if(parent != null) {
- parent2f = pathToFile(parent);
- if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
- throw new ParentNotDirectoryException("Parent path is not a directory: "
- + parent);
- }
- }
- if (p2f.exists() && !p2f.isDirectory()) {
- throw new FileAlreadyExistsException("Destination exists" +
- " and is not a directory: " + p2f.getCanonicalPath());
- }
- return (parent == null || parent2f.exists() || mkdirs(parent)) &&
- (mkOneDirWithMode(f, p2f, permission) || p2f.isDirectory());
- }
-
-
- @Override
- public Path getHomeDirectory() {
- return this.makeQualified(new Path(EnvUtils.getProperty("user.home")));
- }
-
- /**
- * Set the working directory to the given directory.
- */
- @Override
- public void setWorkingDirectory(Path newDir) {
- workingDir = makeAbsolute(newDir);
- checkPath(workingDir);
- }
-
- @Override
- public Path getWorkingDirectory() {
- return workingDir;
- }
-
- @Override
- protected Path getInitialWorkingDirectory() {
- return this.makeQualified(new Path(EnvUtils.getProperty("user.dir")));
- }
-
- @Override
- public FsStatus getStatus(Path p) throws IOException {
- File partition = pathToFile(p == null ? new Path("/") : p);
- //File provides getUsableSpace() and getFreeSpace()
- //File provides no API to obtain used space, assume used = total - free
- return new FsStatus(partition.getTotalSpace(),
- partition.getTotalSpace() - partition.getFreeSpace(),
- partition.getFreeSpace());
- }
-
- // In the case of the local filesystem, we can just rename the file.
- @Override
- public void moveFromLocalFile(Path src, Path dst) throws IOException {
- rename(src, dst);
- }
-
- // We can write output directly to the final location
- @Override
- public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
- return fsOutputFile;
- }
-
- // It's in the right place - nothing to do.
- @Override
- public void completeLocalOutput(Path fsWorkingFile, Path tmpLocalFile)
- throws IOException {
- }
-
- @Override
- public void close() throws IOException {
- super.close();
- }
-
- @Override
- public String toString() {
- return "LocalFS";
- }
-
- @Override
- public FileStatus getFileStatus(Path f) throws IOException {
- return getFileLinkStatusInternal(f, true);
- }
-
- @Deprecated
- private FileStatus deprecatedGetFileStatus(Path f) throws IOException {
- File path = pathToFile(f);
- if (path.exists()) {
- return new DeprecatedRawLocalFileStatus(pathToFile(f),
- defaultBlockSize, this);
- } else {
- throw new FileNotFoundException("File " + f + " does not exist");
- }
- }
-
- @Deprecated
- static class DeprecatedRawLocalFileStatus extends FileStatus {
- /* We can add extra fields here. It breaks at least CopyFiles.FilePair().
- * We recognize if the information is already loaded by check if
- * onwer.equals("").
- */
- private boolean isPermissionLoaded() {
- return !super.getOwner().isEmpty();
- }
-
- private static long getLastAccessTime(File f) throws IOException {
- long accessTime;
- try {
- accessTime = Files.readAttributes(f.toPath(),
- BasicFileAttributes.class).lastAccessTime().toMillis();
- } catch (NoSuchFileException e) {
- throw new FileNotFoundException("File " + f + " does not exist");
- }
- return accessTime;
- }
-
- DeprecatedRawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs)
- throws IOException {
- super(f.length(), f.isDirectory(), 1, defaultBlockSize,
- f.lastModified(), getLastAccessTime(f),
- null, null, null,
- new Path(f.getPath()).makeQualified(fs.getUri(),
- fs.getWorkingDirectory()));
- }
-
- @Override
- public FsPermission getPermission() {
- if (!isPermissionLoaded()) {
- loadPermissionInfo();
- }
- return super.getPermission();
- }
-
- @Override
- public String getOwner() {
- if (!isPermissionLoaded()) {
- loadPermissionInfo();
- }
- return super.getOwner();
- }
-
- @Override
- public String getGroup() {
- if (!isPermissionLoaded()) {
- loadPermissionInfo();
- }
- return super.getGroup();
- }
-
- /**
- * Load file permission information (UNIX symbol rwxrwxrwx, sticky bit info).
- *
- * To improve peformance, give priority to native stat() call. First try get
- * permission information by using native JNI call then fall back to use non
- * native (ProcessBuilder) call in case native lib is not loaded or native
- * call is not successful
- */
- private synchronized void loadPermissionInfo() {
- if (!isPermissionLoaded() && NativeIO.isAvailable()) {
- try {
- loadPermissionInfoByNativeIO();
- } catch (IOException ex) {
- LOG.debug("Native call failed", ex);
- }
- }
-
- if (!isPermissionLoaded()) {
- loadPermissionInfoByNonNativeIO();
- }
- }
-
- /// loads permissions, owner, and group from `ls -ld`
- @VisibleForTesting
- void loadPermissionInfoByNonNativeIO() {
- IOException e = null;
- try {
- java.nio.file.Path path = Paths.get(getPath().toUri());
- String permission = '-' + PosixFilePermissions.toString(Files.getPosixFilePermissions(path));
- setPermission(FsPermission.valueOf(permission));
-
- String owner = Files.getOwner(path).getName();
- String group = Files.readAttributes(path, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS).group().getName();
- // If on windows domain, token format is DOMAIN\\user and we want to
- // extract only the user name
- // same as to the group name
- if (Shell.WINDOWS) {
- owner = removeDomain(owner);
- group = removeDomain(group);
- }
- setOwner(owner);
- setGroup(group);
- } catch (IOException ioe) {
- e = ioe;
- } finally {
- if (e != null) {
- throw new RuntimeException("Error while running command to get " +
- "file permissions : " +
- StringUtils.stringifyException(e));
- }
- }
- }
-
- // In Windows, domain name is added.
- // For example, given machine name (domain name) dname, user name i, then
- // the result for user is dname\\i and for group is dname\\None. So we need
- // remove domain name as follows:
- // DOMAIN\\user => user, DOMAIN\\group => group
- private String removeDomain(String str) {
- int index = str.indexOf('\\');
- if (index != -1) {
- str = str.substring(index + 1);
- }
- return str;
- }
-
- // loads permissions, owner, and group from `ls -ld`
- // but use JNI to more efficiently get file mode (permission, owner, group)
- // by calling file stat() in *nix or some similar calls in Windows
- @VisibleForTesting
- void loadPermissionInfoByNativeIO() throws IOException {
- Path path = getPath();
- String pathName = path.toUri().getPath();
- // remove leading slash for Windows path
- if (Shell.WINDOWS && pathName.startsWith("/")) {
- pathName = pathName.substring(1);
- }
- try {
- NativeIO.POSIX.Stat stat = NativeIO.POSIX.getStat(pathName);
- String owner = stat.getOwner();
- String group = stat.getGroup();
- int mode = stat.getMode();
- setOwner(owner);
- setGroup(group);
- setPermission(new FsPermission(mode));
- } catch (IOException e) {
- setOwner(null);
- setGroup(null);
- setPermission(null);
- throw e;
- }
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- if (!isPermissionLoaded()) {
- loadPermissionInfo();
- }
- super.write(out);
- }
- }
-
- /**
- * Use the command chown to set owner.
- */
- @Override
- public void setOwner(Path p, String username, String groupname)
- throws IOException {
- FileUtil.setOwner(pathToFile(p), username, groupname);
- }
-
- /**
- * Use the command chmod to set permission.
- */
- @Override
- public void setPermission(Path p, FsPermission permission)
- throws IOException {
- if (NativeIO.isAvailable()) {
- NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
- permission.toShort());
- } else {
- Files.setPosixFilePermissions(Paths.get(p.toUri()),
- PosixFilePermissions.fromString(permission.toString()));
- }
- }
-
- /**
- * Sets the {@link Path}'s last modified time and last access time to
- * the given valid times.
- *
- * @param mtime the modification time to set (only if no less than zero).
- * @param atime the access time to set (only if no less than zero).
- * @throws IOException if setting the times fails.
- */
- @Override
- public void setTimes(Path p, long mtime, long atime) throws IOException {
- try {
- BasicFileAttributeView view = Files.getFileAttributeView(
- pathToFile(p).toPath(), BasicFileAttributeView.class);
- FileTime fmtime = (mtime >= 0) ? FileTime.fromMillis(mtime) : null;
- FileTime fatime = (atime >= 0) ? FileTime.fromMillis(atime) : null;
- view.setTimes(fmtime, fatime, null);
- } catch (NoSuchFileException e) {
- throw new FileNotFoundException("File " + p + " does not exist");
- }
- }
-
- /**
- * Hook to implement support for {@link PathHandle} operations.
- * @param stat Referent in the target FileSystem
- * @param opts Constraints that determine the validity of the
- * {@link PathHandle} reference.
- */
- @Override
- protected PathHandle createPathHandle(FileStatus stat,
- Options.HandleOpt... opts) {
- if (stat.isDirectory() || stat.isSymlink()) {
- throw new IllegalArgumentException("PathHandle only available for files");
- }
- String authority = stat.getPath().toUri().getAuthority();
- if (authority != null && !authority.equals("file://")) {
- throw new IllegalArgumentException("Wrong FileSystem: " + stat.getPath());
- }
- Options.HandleOpt.Data data =
- Options.HandleOpt.getOpt(Options.HandleOpt.Data.class, opts)
- .orElse(Options.HandleOpt.changed(false));
- Options.HandleOpt.Location loc =
- Options.HandleOpt.getOpt(Options.HandleOpt.Location.class, opts)
- .orElse(Options.HandleOpt.moved(false));
- if (loc.allowChange()) {
- throw new UnsupportedOperationException("Tracking file movement in " +
- "basic FileSystem is not supported");
- }
- final Path p = stat.getPath();
- final Optional<Long> mtime = !data.allowChange()
- ? Optional.of(stat.getModificationTime())
- : Optional.empty();
- return new LocalFileSystemPathHandle(p.toString(), mtime);
- }
-
- @Override
- public boolean supportsSymlinks() {
- return true;
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public void createSymlink(Path target, Path link, boolean createParent)
- throws IOException {
- if (!FileSystem.areSymlinksEnabled()) {
- throw new UnsupportedOperationException("Symlinks not supported");
- }
- final String targetScheme = target.toUri().getScheme();
- if (targetScheme != null && !"file".equals(targetScheme)) {
- throw new IOException("Unable to create symlink to non-local file "+
- "system: "+target.toString());
- }
- if (createParent) {
- mkdirs(link.getParent());
- }
-
- // NB: Use createSymbolicLink in java.nio.file.Path once available
- int result = FileUtil.symLink(target.toString(),
- makeAbsolute(link).toString());
- if (result != 0) {
- throw new IOException("Error " + result + " creating symlink " +
- link + " to " + target);
- }
- }
-
- /**
- * Return a FileStatus representing the given path. If the path refers
- * to a symlink return a FileStatus representing the link rather than
- * the object the link refers to.
- */
- @Override
- public FileStatus getFileLinkStatus(final Path f) throws IOException {
- FileStatus fi = getFileLinkStatusInternal(f, false);
- // getFileLinkStatus is supposed to return a symlink with a
- // qualified path
- if (fi.isSymlink()) {
- Path targetQual = FSLinkResolver.qualifySymlinkTarget(this.getUri(),
- fi.getPath(), fi.getSymlink());
- fi.setSymlink(targetQual);
- }
- return fi;
- }
-
- /**
- * Public {@link FileStatus} methods delegate to this function, which in turn
- * either call the new {@link Stat} based implementation or the deprecated
- * methods based on platform support.
- *
- * @param f Path to stat
- * @param dereference whether to dereference the final path component if a
- * symlink
- * @return FileStatus of f
- * @throws IOException Exception on getFileLinkStatusInternal
- */
- private FileStatus getFileLinkStatusInternal(final Path f,
- boolean dereference) throws IOException {
- if (!useDeprecatedFileStatus) {
- return getNativeFileLinkStatus(f, dereference);
- } else if (dereference) {
- return deprecatedGetFileStatus(f);
- } else {
- return deprecatedGetFileLinkStatusInternal(f);
- }
- }
-
- /**
- * Deprecated. Remains for legacy support. Should be removed when {@link Stat}
- * gains support for Windows and other operating systems.
- */
- @Deprecated
- private FileStatus deprecatedGetFileLinkStatusInternal(final Path f)
- throws IOException {
- String target = FileUtil.readLink(new File(f.toString()));
-
- try {
- FileStatus fs = getFileStatus(f);
- // If f refers to a regular file or directory
- if (target.isEmpty()) {
- return fs;
- }
- // Otherwise f refers to a symlink
- return new FileStatus(fs.getLen(),
- false,
- fs.getReplication(),
- fs.getBlockSize(),
- fs.getModificationTime(),
- fs.getAccessTime(),
- fs.getPermission(),
- fs.getOwner(),
- fs.getGroup(),
- new Path(target),
- f);
- } catch (FileNotFoundException e) {
- /* The exists method in the File class returns false for dangling
- * links so we can get a FileNotFoundException for links that exist.
- * It's also possible that we raced with a delete of the link. Use
- * the readBasicFileAttributes method in java.nio.file.attributes
- * when available.
- */
- if (!target.isEmpty()) {
- return new FileStatus(0, false, 0, 0, 0, 0, FsPermission.getDefault(),
- "", "", new Path(target), f);
- }
- // f refers to a file or directory that does not exist
- throw e;
- }
- }
- /**
- * Calls out to platform's native stat(1) implementation to get file metadata
- * (permissions, user, group, atime, mtime, etc). This works around the lack
- * of lstat(2) in Java 6.
- *
- * Currently, the {@link Stat} class used to do this only supports Linux
- * and FreeBSD, so the old {@link #deprecatedGetFileLinkStatusInternal(Path)}
- * implementation (deprecated) remains further OS support is added.
- *
- * @param f File to stat
- * @param dereference whether to dereference symlinks
- * @return FileStatus of f
- * @throws IOException Exception on getNativeFileLinkStatus
- */
- private FileStatus getNativeFileLinkStatus(final Path f,
- boolean dereference) throws IOException {
- checkPath(f);
- Stat stat = new Stat(f, defaultBlockSize, dereference, this);
- FileStatus status = stat.getFileStatus();
- return status;
- }
-
- @Override
- public Path getLinkTarget(Path f) throws IOException {
- FileStatus fi = getFileLinkStatusInternal(f, false);
- // return an unqualified symlink target
- return fi.getSymlink();
- }
-
- @Override
- public boolean hasPathCapability(final Path path, final String capability)
- throws IOException {
- switch (validatePathCapabilityArgs(makeQualified(path), capability)) {
- case CommonPathCapabilities.FS_APPEND:
- case CommonPathCapabilities.FS_CONCAT:
- case CommonPathCapabilities.FS_PATHHANDLES:
- case CommonPathCapabilities.FS_PERMISSIONS:
- case CommonPathCapabilities.FS_TRUNCATE:
- return true;
- case CommonPathCapabilities.FS_SYMLINKS:
- return FileSystem.areSymlinksEnabled();
- default:
- return super.hasPathCapability(path, capability);
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/solr/modules/hdfs/src/test/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
deleted file mode 100644
index 93dc499e8f9..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ /dev/null
@@ -1,981 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtilClient;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
-import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportReplica;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
-import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
-import org.apache.hadoop.hdfs.server.datanode.DataSetLockManager;
-import org.apache.hadoop.hdfs.server.datanode.DataStorage;
-import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
-import org.apache.hadoop.hdfs.server.datanode.FileIoProvider;
-import org.apache.hadoop.hdfs.server.datanode.ReplicaBuilder;
-import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaTracker.RamDiskReplica;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.MultipleIOException;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.util.DataChecksum;
-import org.apache.hadoop.util.DataChecksum.Type;
-import org.apache.hadoop.util.DiskChecker;
-import org.apache.hadoop.util.DiskChecker.DiskErrorException;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.Timer;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedInputStream;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.RandomAccessFile;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.Scanner;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.ForkJoinTask;
-import java.util.concurrent.RecursiveAction;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * A block pool slice represents a portion of a block pool stored on a volume.
- * Taken together, all BlockPoolSlices sharing a block pool ID across a
- * cluster represent a single block pool.
- *
- * This class is synchronized by {@link FsVolumeImpl}.
- */
-public class BlockPoolSlice {
- public static final Object SOLR_HACK_FOR_CLASS_VERIFICATION = new Object();
-
- // Apparently the Hadoop code expectes upper-case LOG, so...
- static final Logger LOG = LoggerFactory.getLogger(BlockPoolSlice.class); // nowarn_valid_logger
-
- private final String bpid;
- private final FsVolumeImpl volume; // volume to which this BlockPool belongs to
- private final File currentDir; // StorageDirectory/current/bpid/current
- // directory where finalized replicas are stored
- private final File finalizedDir;
- private final File lazypersistDir;
- private final File rbwDir; // directory store RBW replica
- private final File tmpDir; // directory store Temporary replica
- private final int ioFileBufferSize;
- @VisibleForTesting
- public static final String DU_CACHE_FILE = "dfsUsed";
- private final Runnable shutdownHook;
- private volatile boolean dfsUsedSaved = false;
- private static final int SHUTDOWN_HOOK_PRIORITY = 30;
- private final boolean deleteDuplicateReplicas;
- private static final String REPLICA_CACHE_FILE = "replicas";
- private final long replicaCacheExpiry;
- private final File replicaCacheDir;
- private AtomicLong numOfBlocks = new AtomicLong();
- private final long cachedDfsUsedCheckTime;
- private final Timer timer;
- private final int maxDataLength;
- private final FileIoProvider fileIoProvider;
-
- private static ForkJoinPool addReplicaThreadPool = null;
- private static final int VOLUMES_REPLICA_ADD_THREADPOOL_SIZE = Runtime
- .getRuntime().availableProcessors();
- private static final Comparator<File> FILE_COMPARATOR =
- new Comparator<File>() {
- @Override
- public int compare(File f1, File f2) {
- return f1.getName().compareTo(f2.getName());
- }
- };
-
- /**
- * Create a blook pool slice
- * @param bpid Block pool Id
- * @param volume {@link FsVolumeImpl} to which this BlockPool belongs to
- * @param bpDir directory corresponding to the BlockPool
- * @param conf configuration
- * @param timer include methods for getting time
- * @throws IOException Error making directories
- */
- BlockPoolSlice(String bpid, FsVolumeImpl volume, File bpDir,
- Configuration conf, Timer timer) throws IOException {
- this.bpid = bpid;
- this.volume = volume;
- this.fileIoProvider = volume.getFileIoProvider();
- this.currentDir = new File(bpDir, DataStorage.STORAGE_DIR_CURRENT);
- this.finalizedDir = new File(
- currentDir, DataStorage.STORAGE_DIR_FINALIZED);
- this.lazypersistDir = new File(currentDir, DataStorage.STORAGE_DIR_LAZY_PERSIST);
- if (!this.finalizedDir.exists()) {
- if (!this.finalizedDir.mkdirs()) {
- throw new IOException("Failed to mkdirs " + this.finalizedDir);
- }
- }
-
- this.ioFileBufferSize = DFSUtilClient.getIoFileBufferSize(conf);
-
- this.deleteDuplicateReplicas = conf.getBoolean(
- DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION,
- DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION_DEFAULT);
-
- this.cachedDfsUsedCheckTime =
- conf.getLong(
- DFSConfigKeys.DFS_DN_CACHED_DFSUSED_CHECK_INTERVAL_MS,
- DFSConfigKeys.DFS_DN_CACHED_DFSUSED_CHECK_INTERVAL_DEFAULT_MS);
-
- this.maxDataLength = conf.getInt(
- CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH,
- CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT);
-
- this.timer = timer;
-
- // Files that were being written when the datanode was last shutdown
- // are now moved back to the data directory. It is possible that
- // in the future, we might want to do some sort of datanode-local
- // recovery for these blocks. For example, crc validation.
- //
- this.tmpDir = new File(bpDir, DataStorage.STORAGE_DIR_TMP);
- if (tmpDir.exists()) {
- fileIoProvider.fullyDelete(volume, tmpDir);
- }
- this.rbwDir = new File(currentDir, DataStorage.STORAGE_DIR_RBW);
-
- // create the rbw and tmp directories if they don't exist.
- fileIoProvider.mkdirs(volume, rbwDir);
- fileIoProvider.mkdirs(volume, tmpDir);
-
- String cacheDirRoot = conf.get(
- DFSConfigKeys.DFS_DATANODE_REPLICA_CACHE_ROOT_DIR_KEY);
- if (cacheDirRoot != null && !cacheDirRoot.isEmpty()) {
- this.replicaCacheDir = new File(cacheDirRoot,
- currentDir.getCanonicalPath());
- if (!this.replicaCacheDir.exists()) {
- if (!this.replicaCacheDir.mkdirs()) {
- throw new IOException("Failed to mkdirs " + this.replicaCacheDir);
- }
- }
- } else {
- this.replicaCacheDir = currentDir;
- }
- this.replicaCacheExpiry = conf.getTimeDuration(
- DFSConfigKeys.DFS_DATANODE_REPLICA_CACHE_EXPIRY_TIME_KEY,
- DFSConfigKeys.DFS_DATANODE_REPLICA_CACHE_EXPIRY_TIME_DEFAULT,
- TimeUnit.MILLISECONDS);
-
- if (addReplicaThreadPool == null) {
- // initialize add replica fork join pool
- initializeAddReplicaPool(conf, (FsDatasetImpl) volume.getDataset());
- }
- // Make the dfs usage to be saved during shutdown.
- shutdownHook = new Runnable() {
- @Override
- public void run() {
- addReplicaThreadPool.shutdownNow();
- }
- };
- ShutdownHookManager.get().addShutdownHook(shutdownHook,
- SHUTDOWN_HOOK_PRIORITY);
- }
-
- private synchronized static void initializeAddReplicaPool(Configuration conf,
- FsDatasetImpl dataset) {
- if (addReplicaThreadPool == null) {
- int numberOfBlockPoolSlice = dataset.getVolumeCount()
- * dataset.getBPServiceCount();
- int poolsize = Math.max(numberOfBlockPoolSlice,
- VOLUMES_REPLICA_ADD_THREADPOOL_SIZE);
- // Default pool sizes is max of (volume * number of bp_service) and
- // number of processor.
- int parallelism = conf.getInt(
- DFSConfigKeys.DFS_DATANODE_VOLUMES_REPLICA_ADD_THREADPOOL_SIZE_KEY,
- poolsize);
-
- // Needed for SOLR-9515 and HDFS-14251
- ForkJoinPool.ForkJoinWorkerThreadFactory threadFactory = new HdfsTestUtil.HDFSForkJoinThreadFactory();
- addReplicaThreadPool = new ForkJoinPool(parallelism, threadFactory, null, false);
- }
- }
-
- File getDirectory() {
- return currentDir.getParentFile();
- }
-
- File getFinalizedDir() {
- return finalizedDir;
- }
-
- File getLazypersistDir() {
- return lazypersistDir;
- }
-
- File getRbwDir() {
- return rbwDir;
- }
-
- File getTmpDir() {
- return tmpDir;
- }
-
- /** Run DU on local drives. It must be synchronized from caller. */
- void decDfsUsed(long value) {
- }
-
- long getDfsUsed() throws IOException {
- return 0L;
- }
-
- void incDfsUsed(long value) {
- }
-
- /**
- * Temporary files. They get moved to the finalized block directory when
- * the block is finalized.
- */
- File createTmpFile(Block b) throws IOException {
- File f = new File(tmpDir, b.getBlockName());
- File tmpFile = DatanodeUtil.createFileWithExistsCheck(
- volume, b, f, fileIoProvider);
- // If any exception during creation, its expected that counter will not be
- // incremented, So no need to decrement
- incrNumBlocks();
- return tmpFile;
- }
-
- /**
- * RBW files. They get moved to the finalized block directory when
- * the block is finalized.
- */
- File createRbwFile(Block b) throws IOException {
- File f = new File(rbwDir, b.getBlockName());
- File rbwFile = DatanodeUtil.createFileWithExistsCheck(
- volume, b, f, fileIoProvider);
- // If any exception during creation, its expected that counter will not be
- // incremented, So no need to decrement
- incrNumBlocks();
- return rbwFile;
- }
-
- File addFinalizedBlock(Block b, ReplicaInfo replicaInfo) throws IOException {
- File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId());
- fileIoProvider.mkdirsWithExistsCheck(volume, blockDir);
- return FsDatasetImpl.moveBlockFiles(b, replicaInfo, blockDir);
- }
-
- /**
- * Move a persisted replica from lazypersist directory to a subdirectory
- * under finalized.
- */
- ReplicaInfo activateSavedReplica(ReplicaInfo replicaInfo,
- RamDiskReplica replicaState) throws IOException {
- File metaFile = replicaState.getSavedMetaFile();
- File blockFile = replicaState.getSavedBlockFile();
- final long blockId = replicaInfo.getBlockId();
- final File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId);
- final File targetBlockFile = new File(blockDir, blockFile.getName());
- final File targetMetaFile = new File(blockDir, metaFile.getName());
- fileIoProvider.moveFile(volume, blockFile, targetBlockFile);
- FsDatasetImpl.LOG.info("Moved " + blockFile + " to " + targetBlockFile);
- fileIoProvider.moveFile(volume, metaFile, targetMetaFile);
- FsDatasetImpl.LOG.info("Moved " + metaFile + " to " + targetMetaFile);
-
- ReplicaInfo newReplicaInfo =
- new ReplicaBuilder(ReplicaState.FINALIZED)
- .setBlockId(blockId)
- .setLength(replicaInfo.getBytesOnDisk())
- .setGenerationStamp(replicaInfo.getGenerationStamp())
- .setFsVolume(replicaState.getLazyPersistVolume())
- .setDirectoryToUse(targetBlockFile.getParentFile())
- .build();
- return newReplicaInfo;
- }
-
- void checkDirs() throws DiskErrorException {
- DiskChecker.checkDir(finalizedDir);
- DiskChecker.checkDir(tmpDir);
- DiskChecker.checkDir(rbwDir);
- }
-
-
-
- void getVolumeMap(ReplicaMap volumeMap,
- final RamDiskReplicaTracker lazyWriteReplicaMap)
- throws IOException {
- // Recover lazy persist replicas, they will be added to the volumeMap
- // when we scan the finalized directory.
- if (lazypersistDir.exists()) {
- int numRecovered = moveLazyPersistReplicasToFinalized(lazypersistDir);
- FsDatasetImpl.LOG.info(
- "Recovered " + numRecovered + " replicas from " + lazypersistDir);
- }
-
- boolean success = readReplicasFromCache(volumeMap, lazyWriteReplicaMap);
- if (!success) {
- List<IOException> exceptions = Collections
- .synchronizedList(new ArrayList<IOException>());
- Queue<RecursiveAction> subTaskQueue =
- new ConcurrentLinkedQueue<RecursiveAction>();
-
- // add finalized replicas
- AddReplicaProcessor task = new AddReplicaProcessor(volumeMap,
- finalizedDir, lazyWriteReplicaMap, true, exceptions, subTaskQueue);
- ForkJoinTask<Void> finalizedTask = addReplicaThreadPool.submit(task);
-
- // add rbw replicas
- task = new AddReplicaProcessor(volumeMap, rbwDir, lazyWriteReplicaMap,
- false, exceptions, subTaskQueue);
- ForkJoinTask<Void> rbwTask = addReplicaThreadPool.submit(task);
-
- try {
- finalizedTask.get();
- rbwTask.get();
- } catch (InterruptedException | ExecutionException e) {
- exceptions.add(new IOException(
- "Failed to start sub tasks to add replica in replica map :"
- + e.getMessage()));
- }
-
- //wait for all the tasks to finish.
- waitForSubTaskToFinish(subTaskQueue, exceptions);
- }
- }
-
- /**
- * Wait till all the recursive task for add replica to volume completed.
- *
- * @param subTaskQueue
- * {@link AddReplicaProcessor} tasks list.
- * @param exceptions
- * exceptions occurred in sub tasks.
- * @throws IOException
- * throw if any sub task or multiple sub tasks failed.
- */
- private void waitForSubTaskToFinish(Queue<RecursiveAction> subTaskQueue,
- List<IOException> exceptions) throws IOException {
- while (!subTaskQueue.isEmpty()) {
- RecursiveAction task = subTaskQueue.poll();
- if (task != null) {
- task.join();
- }
- }
- if (!exceptions.isEmpty()) {
- throw MultipleIOException.createIOException(exceptions);
- }
- }
-
- /**
- * Recover an unlinked tmp file on datanode restart. If the original block
- * does not exist, then the tmp file is renamed to be the
- * original file name and the original name is returned; otherwise the tmp
- * file is deleted and null is returned.
- */
- File recoverTempUnlinkedBlock(File unlinkedTmp) throws IOException {
- File blockFile = FsDatasetUtil.getOrigFile(unlinkedTmp);
- if (blockFile.exists()) {
- // If the original block file still exists, then no recovery is needed.
- if (!fileIoProvider.delete(volume, unlinkedTmp)) {
- throw new IOException("Unable to cleanup unlinked tmp file " +
- unlinkedTmp);
- }
- return null;
- } else {
- fileIoProvider.rename(volume, unlinkedTmp, blockFile);
- return blockFile;
- }
- }
-
-
- /**
- * Move replicas in the lazy persist directory to their corresponding locations
- * in the finalized directory.
- * @return number of replicas recovered.
- */
- private int moveLazyPersistReplicasToFinalized(File source)
- throws IOException {
- File[] files = fileIoProvider.listFiles(volume, source);
- int numRecovered = 0;
- for (File file : files) {
- if (file.isDirectory()) {
- numRecovered += moveLazyPersistReplicasToFinalized(file);
- }
-
- if (Block.isMetaFilename(file.getName())) {
- File metaFile = file;
- File blockFile = Block.metaToBlockFile(metaFile);
- long blockId = Block.filename2id(blockFile.getName());
- File targetDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId);
-
- if (blockFile.exists()) {
-
- try {
- fileIoProvider.mkdirsWithExistsCheck(volume, targetDir);
- } catch(IOException ioe) {
- LOG.warn("Failed to mkdirs {}", targetDir);
- continue;
- }
-
- final File targetMetaFile = new File(targetDir, metaFile.getName());
- try {
- fileIoProvider.rename(volume, metaFile, targetMetaFile);
- } catch (IOException e) {
- LOG.warn("Failed to move meta file from {} to {}", metaFile, targetMetaFile, e);
- continue;
- }
-
- final File targetBlockFile = new File(targetDir, blockFile.getName());
- try {
- fileIoProvider.rename(volume, blockFile, targetBlockFile);
- } catch (IOException e) {
- LOG.warn("Failed to move block file from {} to {}", blockFile, targetBlockFile, e);
- continue;
- }
-
- if (targetBlockFile.exists() && targetMetaFile.exists()) {
- ++numRecovered;
- } else {
- // Failure should be rare.
- LOG.warn("Failed to move {} to {}", blockFile, targetDir);
- }
- }
- }
- }
-
- fileIoProvider.fullyDelete(volume, source);
- return numRecovered;
- }
-
- private void addReplicaToReplicasMap(Block block, ReplicaMap volumeMap,
- final RamDiskReplicaTracker lazyWriteReplicaMap,boolean isFinalized)
- throws IOException {
- ReplicaInfo newReplica = null;
- long blockId = block.getBlockId();
- long genStamp = block.getGenerationStamp();
- if (isFinalized) {
- newReplica = new ReplicaBuilder(ReplicaState.FINALIZED)
- .setBlockId(blockId)
- .setLength(block.getNumBytes())
- .setGenerationStamp(genStamp)
- .setFsVolume(volume)
- .setDirectoryToUse(DatanodeUtil.idToBlockDir(finalizedDir, blockId))
- .build();
- } else {
- File file = new File(rbwDir, block.getBlockName());
- boolean loadRwr = true;
- File restartMeta = new File(file.getParent() +
- File.pathSeparator + "." + file.getName() + ".restart");
- Scanner sc = null;
- try {
- sc = new Scanner(restartMeta, StandardCharsets.UTF_8);
- // The restart meta file exists
- if (sc.hasNextLong() && (sc.nextLong() > timer.now())) {
- // It didn't expire. Load the replica as a RBW.
- // We don't know the expected block length, so just use 0
- // and don't reserve any more space for writes.
- newReplica = new ReplicaBuilder(ReplicaState.RBW)
- .setBlockId(blockId)
- .setLength(validateIntegrityAndSetLength(file, genStamp))
- .setGenerationStamp(genStamp)
- .setFsVolume(volume)
- .setDirectoryToUse(file.getParentFile())
- .setWriterThread(null)
- .setBytesToReserve(0)
- .build();
- loadRwr = false;
- }
- sc.close();
- if (!fileIoProvider.delete(volume, restartMeta)) {
- FsDatasetImpl.LOG.warn("Failed to delete restart meta file: " +
- restartMeta.getPath());
- }
- } catch (FileNotFoundException fnfe) {
- // nothing to do hereFile dir =
- } finally {
- if (sc != null) {
- sc.close();
- }
- }
- // Restart meta doesn't exist or expired.
- if (loadRwr) {
- ReplicaBuilder builder = new ReplicaBuilder(ReplicaState.RWR)
- .setBlockId(blockId)
- .setLength(validateIntegrityAndSetLength(file, genStamp))
- .setGenerationStamp(genStamp)
- .setFsVolume(volume)
- .setDirectoryToUse(file.getParentFile());
- newReplica = builder.build();
- }
- }
-
- ReplicaInfo tmpReplicaInfo = volumeMap.addAndGet(bpid, newReplica);
- ReplicaInfo oldReplica = (tmpReplicaInfo.equals(newReplica)) ? null
- : tmpReplicaInfo;
- if (oldReplica != null) {
- // We have multiple replicas of the same block so decide which one
- // to keep.
- newReplica = resolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
- }
-
- // If we are retaining a replica on transient storage make sure
- // it is in the lazyWriteReplicaMap so it can be persisted
- // eventually.
- if (newReplica.getVolume().isTransientStorage()) {
- lazyWriteReplicaMap.addReplica(bpid, blockId,
- (FsVolumeImpl) newReplica.getVolume(), 0);
- } else {
- lazyWriteReplicaMap.discardReplica(bpid, blockId, false);
- }
- if (oldReplica == null) {
- incrNumBlocks();
- }
- }
-
-
- /**
- * Add replicas under the given directory to the volume map
- * @param volumeMap the replicas map
- * @param dir an input directory
- * @param lazyWriteReplicaMap Map of replicas on transient
- * storage.
- * @param isFinalized true if the directory has finalized replicas;
- * false if the directory has rbw replicas
- * @param exceptions list of exception which need to return to parent thread.
- * @param subTaskQueue queue of sub tasks
- */
- void addToReplicasMap(ReplicaMap volumeMap, File dir,
- final RamDiskReplicaTracker lazyWriteReplicaMap, boolean isFinalized,
- List<IOException> exceptions, Queue<RecursiveAction> subTaskQueue)
- throws IOException {
- File[] files = fileIoProvider.listFiles(volume, dir);
- Arrays.sort(files, FILE_COMPARATOR);
- for (int i = 0; i < files.length; i++) {
- File file = files[i];
- if (file.isDirectory()) {
- // Launch new sub task.
- AddReplicaProcessor subTask = new AddReplicaProcessor(volumeMap, file,
- lazyWriteReplicaMap, isFinalized, exceptions, subTaskQueue);
- subTask.fork();
- subTaskQueue.add(subTask);
- }
-
- if (isFinalized && FsDatasetUtil.isUnlinkTmpFile(file)) {
- file = recoverTempUnlinkedBlock(file);
- if (file == null) { // the original block still exists, so we cover it
- // in another iteration and can continue here
- continue;
- }
- }
- if (!Block.isBlockFilename(file)) {
- continue;
- }
-
- long genStamp = FsDatasetUtil.getGenerationStampFromFile(
- files, file, i);
- long blockId = Block.filename2id(file.getName());
- Block block = new Block(blockId, file.length(), genStamp);
- addReplicaToReplicasMap(block, volumeMap, lazyWriteReplicaMap,
- isFinalized);
- }
- }
-
- /**
- * This method is invoked during DN startup when volumes are scanned to
- * build up the volumeMap.
- *
- * Given two replicas, decide which one to keep. The preference is as
- * follows:
- * 1. Prefer the replica with the higher generation stamp.
- * 2. If generation stamps are equal, prefer the replica with the
- * larger on-disk length.
- * 3. If on-disk length is the same, prefer the replica on persistent
- * storage volume.
- * 4. All other factors being equal, keep replica1.
- *
- * The other replica is removed from the volumeMap and is deleted from
- * its storage volume.
- *
- * @param replica1 first replica
- * @param replica2 second replica
- * @param volumeMap volume map to update
- * @return the replica that is retained.
- */
- ReplicaInfo resolveDuplicateReplicas(
- final ReplicaInfo replica1, final ReplicaInfo replica2,
- final ReplicaMap volumeMap) {
-
- if (!deleteDuplicateReplicas) {
- // Leave both block replicas in place.
- return replica1;
- }
- final ReplicaInfo replicaToDelete =
- selectReplicaToDelete(replica1, replica2);
- final ReplicaInfo replicaToKeep =
- (!Objects.equals(replicaToDelete, replica1)) ? replica1 : replica2;
- // Update volumeMap and delete the replica
- volumeMap.add(bpid, replicaToKeep);
- if (replicaToDelete != null) {
- deleteReplica(replicaToDelete);
- }
- return replicaToKeep;
- }
-
- @VisibleForTesting
- static ReplicaInfo selectReplicaToDelete(final ReplicaInfo replica1,
- final ReplicaInfo replica2) {
- ReplicaInfo replicaToKeep;
- ReplicaInfo replicaToDelete;
-
- // it's the same block so don't ever delete it, even if GS or size
- // differs. caller should keep the one it just discovered on disk
- if (replica1.getBlockURI().equals(replica2.getBlockURI())) {
- return null;
- }
- if (replica1.getGenerationStamp() != replica2.getGenerationStamp()) {
- replicaToKeep = replica1.getGenerationStamp() > replica2.getGenerationStamp()
- ? replica1 : replica2;
- } else if (replica1.getNumBytes() != replica2.getNumBytes()) {
- replicaToKeep = replica1.getNumBytes() > replica2.getNumBytes() ?
- replica1 : replica2;
- } else if (replica1.getVolume().isTransientStorage() &&
- !replica2.getVolume().isTransientStorage()) {
- replicaToKeep = replica2;
- } else {
- replicaToKeep = replica1;
- }
-
- replicaToDelete = (replicaToKeep.equals(replica1)) ? replica2 : replica1;
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("resolveDuplicateReplicas decide to keep {}. Will try to delete {}", replicaToKeep, replicaToDelete);
- }
- return replicaToDelete;
- }
-
- private void deleteReplica(final ReplicaInfo replicaToDelete) {
- // Delete the files on disk. Failure here is okay.
- if (!replicaToDelete.deleteBlockData()) {
- LOG.warn("Failed to delete block file for replica {}", replicaToDelete);
- }
- if (!replicaToDelete.deleteMetadata()) {
- LOG.warn("Failed to delete meta file for replica {}", replicaToDelete);
- }
- }
-
- /**
- * Find out the number of bytes in the block that match its crc.
- *
- * This algorithm assumes that data corruption caused by unexpected
- * datanode shutdown occurs only in the last crc chunk. So it checks
- * only the last chunk.
- *
- * @param blockFile the block file
- * @param genStamp generation stamp of the block
- * @return the number of valid bytes
- */
- private long validateIntegrityAndSetLength(File blockFile, long genStamp) {
- try {
- final File metaFile = FsDatasetUtil.getMetaFile(blockFile, genStamp);
- long blockFileLen = blockFile.length();
- long metaFileLen = metaFile.length();
- int crcHeaderLen = DataChecksum.getChecksumHeaderSize();
- if (!blockFile.exists() || blockFileLen == 0 ||
- !metaFile.exists() || metaFileLen < crcHeaderLen) {
- return 0;
- }
- try (DataInputStream checksumIn = new DataInputStream(
- new BufferedInputStream(
- fileIoProvider.getFileInputStream(volume, metaFile),
- ioFileBufferSize))) {
- // read and handle the common header here. For now just a version
- final DataChecksum checksum = BlockMetadataHeader.readDataChecksum(
- checksumIn, metaFile);
- if (Type.NULL.equals(checksum.getChecksumType())) {
- // in case of NULL checksum type consider full file as valid
- return blockFileLen;
- }
- int bytesPerChecksum = checksum.getBytesPerChecksum();
- int checksumSize = checksum.getChecksumSize();
- long numChunks = Math.min(
- (blockFileLen + bytesPerChecksum - 1) / bytesPerChecksum,
- (metaFileLen - crcHeaderLen) / checksumSize);
- if (numChunks == 0) {
- return 0;
- }
- try (InputStream blockIn = fileIoProvider.getFileInputStream(
- volume, blockFile);
- ReplicaInputStreams ris = new ReplicaInputStreams(blockIn,
- checksumIn, volume.obtainReference(), fileIoProvider)) {
- ris.skipChecksumFully((numChunks - 1) * checksumSize);
- long lastChunkStartPos = (numChunks - 1) * bytesPerChecksum;
- ris.skipDataFully(lastChunkStartPos);
- int lastChunkSize = (int) Math.min(
- bytesPerChecksum, blockFileLen - lastChunkStartPos);
- byte[] buf = new byte[lastChunkSize + checksumSize];
- ris.readChecksumFully(buf, lastChunkSize, checksumSize);
- ris.readDataFully(buf, 0, lastChunkSize);
- checksum.update(buf, 0, lastChunkSize);
- long validFileLength;
- if (checksum.compare(buf, lastChunkSize)) { // last chunk matches crc
- validFileLength = lastChunkStartPos + lastChunkSize;
- } else { // last chunk is corrupt
- validFileLength = lastChunkStartPos;
- }
- // truncate if extra bytes are present without CRC
- if (blockFile.length() > validFileLength) {
- try (RandomAccessFile blockRAF =
- fileIoProvider.getRandomAccessFile(
- volume, blockFile, "rw")) {
- // truncate blockFile
- blockRAF.setLength(validFileLength);
- }
- }
- return validFileLength;
- }
- }
- } catch (IOException e) {
- FsDatasetImpl.LOG.warn("Getting exception while validating integrity " +
- "and setting length for blockFile", e);
- return 0;
- }
- }
-
- @Override
- public String toString() {
- return currentDir.getAbsolutePath();
- }
-
- void shutdown(BlockListAsLongs blocksListToPersist) {
- saveReplicas(blocksListToPersist);
-
- // Remove the shutdown hook to avoid any memory leak
- if (shutdownHook != null) {
- ShutdownHookManager.get().removeShutdownHook(shutdownHook);
- }
- }
-
- private boolean readReplicasFromCache(ReplicaMap volumeMap,
- final RamDiskReplicaTracker lazyWriteReplicaMap) {
- ReplicaMap tmpReplicaMap = new ReplicaMap(new DataSetLockManager());
- File replicaFile = new File(replicaCacheDir, REPLICA_CACHE_FILE);
- // Check whether the file exists or not.
- if (!replicaFile.exists()) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Replica Cache file: {} doesn't exist", replicaFile.getPath());
- }
- return false;
- }
- long fileLastModifiedTime = replicaFile.lastModified();
- if (System.currentTimeMillis() > fileLastModifiedTime + replicaCacheExpiry) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Replica Cache file: {} has gone stale", replicaFile.getPath());
- }
- // Just to make findbugs happy
- if (!replicaFile.delete()) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Replica Cache file: {} cannot be deleted", replicaFile.getPath());
- }
- }
- return false;
- }
- FileInputStream inputStream = null;
- try {
- inputStream = fileIoProvider.getFileInputStream(volume, replicaFile);
- BlockListAsLongs blocksList =
- BlockListAsLongs.readFrom(inputStream, maxDataLength);
- if (blocksList == null) {
- return false;
- }
-
- for (BlockReportReplica replica : blocksList) {
- switch (replica.getState()) {
- case FINALIZED:
- addReplicaToReplicasMap(replica, tmpReplicaMap, lazyWriteReplicaMap, true);
- break;
- case RUR:
- case RBW:
- case RWR:
- addReplicaToReplicasMap(replica, tmpReplicaMap, lazyWriteReplicaMap, false);
- break;
- default:
- break;
- }
- }
- // Now it is safe to add the replica into volumeMap
- // In case of any exception during parsing this cache file, fall back
- // to scan all the files on disk.
- for (Iterator<ReplicaInfo> iter =
- tmpReplicaMap.replicas(bpid).iterator(); iter.hasNext(); ) {
- ReplicaInfo info = iter.next();
- // We use a lightweight GSet to store replicaInfo, we need to remove
- // it from one GSet before adding to another.
- iter.remove();
- volumeMap.add(bpid, info);
- }
- if (LOG.isInfoEnabled()) {
- LOG.info("Successfully read replica from cache file : {}", replicaFile.getPath());
- }
- return true;
- } catch (Exception e) {
- // Any exception we need to revert back to read from disk
- // Log the error and return false
- if (LOG.isInfoEnabled()) {
- LOG.info("Exception occurred while reading the replicas cache file: {}", replicaFile.getPath(), e);
- }
- return false;
- }
- finally {
- // close the inputStream
- IOUtils.closeStream(inputStream);
-
- if (!fileIoProvider.delete(volume, replicaFile)) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Failed to delete replica cache file: {}", replicaFile.getPath());
- }
- }
- }
- }
-
- private void saveReplicas(BlockListAsLongs blocksListToPersist) {
- if (blocksListToPersist == null ||
- blocksListToPersist.getNumberOfBlocks()== 0) {
- return;
- }
- final File tmpFile = new File(replicaCacheDir, REPLICA_CACHE_FILE + ".tmp");
- final File replicaCacheFile = new File(replicaCacheDir, REPLICA_CACHE_FILE);
- if (!fileIoProvider.deleteWithExistsCheck(volume, tmpFile) ||
- !fileIoProvider.deleteWithExistsCheck(volume, replicaCacheFile)) {
- return;
- }
-
- FileOutputStream out = null;
- try {
- out = fileIoProvider.getFileOutputStream(volume, tmpFile);
- blocksListToPersist.writeTo(out);
- out.close();
- // Renaming the tmp file to replicas
- fileIoProvider.moveFile(volume, tmpFile, replicaCacheFile);
- } catch (Exception e) {
- // If write failed, the volume might be bad. Since the cache file is
- // not critical, log the error, delete both the files (tmp and cache)
- // and continue.
- LOG.warn("Failed to write replicas to cache ", e);
- fileIoProvider.deleteWithExistsCheck(volume, replicaCacheFile);
- } finally {
- IOUtils.closeStream(out);
- fileIoProvider.deleteWithExistsCheck(volume, tmpFile);
- }
- }
-
- void incrNumBlocks() {
- numOfBlocks.incrementAndGet();
- }
-
- void decrNumBlocks() {
- numOfBlocks.decrementAndGet();
- }
-
- public long getNumOfBlocks() {
- return numOfBlocks.get();
- }
-
- /**
- * Recursive action for add replica in map.
- */
- class AddReplicaProcessor extends RecursiveAction {
-
- private ReplicaMap volumeMap;
- private File dir;
- private RamDiskReplicaTracker lazyWriteReplicaMap;
- private boolean isFinalized;
- private List<IOException> exceptions;
- private Queue<RecursiveAction> subTaskQueue;
-
- /**
- * @param volumeMap
- * the replicas map
- * @param dir
- * an input directory
- * @param lazyWriteReplicaMap
- * Map of replicas on transient storage.
- * @param isFinalized
- * true if the directory has finalized replicas; false if the
- * directory has rbw replicas
- * @param exceptions
- * List of exception which need to return to parent thread.
- * @param subTaskQueue
- * queue of sub tasks
- */
- AddReplicaProcessor(ReplicaMap volumeMap, File dir,
- RamDiskReplicaTracker lazyWriteReplicaMap, boolean isFinalized,
- List<IOException> exceptions, Queue<RecursiveAction> subTaskQueue) {
- this.volumeMap = volumeMap;
- this.dir = dir;
- this.lazyWriteReplicaMap = lazyWriteReplicaMap;
- this.isFinalized = isFinalized;
- this.exceptions = exceptions;
- this.subTaskQueue = subTaskQueue;
- }
-
- @Override
- protected void compute() {
- try {
- addToReplicasMap(volumeMap, dir, lazyWriteReplicaMap, isFinalized,
- exceptions, subTaskQueue);
- } catch (IOException e) {
- LOG.warn("Caught exception while adding replicas from {} in subtask. Will throw later.", volume, e);
- exceptions.add(e);
- }
- }
- }
-
- /**
- * Return the size of fork pool used for adding replica in map.
- */
- @VisibleForTesting
- public static int getAddReplicaForkPoolSize() {
- return addReplicaThreadPool.getPoolSize();
- }
-
- @VisibleForTesting
- public ForkJoinPool getAddReplicaThreadPool() {
- return addReplicaThreadPool;
- }
-
- @VisibleForTesting
- public static void reInitializeAddReplicaThreadPool() {
- addReplicaThreadPool.shutdown();
- addReplicaThreadPool = null;
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/solr/modules/hdfs/src/test/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java
deleted file mode 100644
index f911d4801fb..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.namenode;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-
-import java.io.IOException;
-
-/**
- *
- * NameNodeResourceChecker provides a method -
- * <code>hasAvailableDiskSpace</code> - which will return true if and only if
- * the NameNode has disk space available on all required volumes, and any volume
- * which is configured to be redundant. Volumes containing file system edits dirs
- * are added by default, and arbitrary extra volumes may be configured as well.
- */
-public class NameNodeResourceChecker {
- public static final Object SOLR_HACK_FOR_CLASS_VERIFICATION = new Object();
-
- /**
- * Create a NameNodeResourceChecker, which will check the edits dirs and any
- * additional dirs to check set in <code>conf</code>.
- */
- public NameNodeResourceChecker(Configuration conf) throws IOException {
- }
-
-
- /**
- * Return true if disk space is available on at least one of the configured
- * redundant volumes, and all of the configured required volumes.
- *
- * @return True if the configured amount of disk space is available on at
- * least one redundant volume and all of the required volumes, false
- * otherwise.
- */
- public boolean hasAvailableDiskSpace() {
- return true;
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/package-info.java b/solr/modules/hdfs/src/test/org/apache/hadoop/package-info.java
deleted file mode 100644
index 28203e51c33..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/hadoop/package-info.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * The classes under this package were copied from Apache Hadoop and modified
- * to avoid certain issues during tests. The copied classes override the
- * Apache Hadoop dependency versions during tests.
- *
- * The classes BlockPoolSlice (HDFS-14251), DiskChecker, FileUtil, HardLink,
- * NameNodeResourceChecker, and RawLocalFileSystem were copied to avoid
- * issues with running Hadoop integration tests under the Java security
- * manager. Many of these classes use org.apache.hadoop.util.Shell
- * which shells out to try to do common filesystem checks.
- *
- * Overtime these classes should be removed as upstream fixes to Apache
- * Hadoop are made. When the Apache Hadoop dependency is upgraded in
- * Solr, the classes should be compared against that version.
- */
-package org.apache.hadoop;
-
diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/util/DiskChecker.java b/solr/modules/hdfs/src/test/org/apache/hadoop/util/DiskChecker.java
deleted file mode 100644
index 113e73d805b..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/hadoop/util/DiskChecker.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.util;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * Class that provides utility functions for checking disk problem
- */
-public class DiskChecker {
- public static final Object SOLR_HACK_FOR_CLASS_VERIFICATION = new Object();
-
- public static final Logger LOG = LoggerFactory.getLogger(DiskChecker.class); // nowarn_valid_logger
-
- public static class DiskErrorException extends IOException {
- public DiskErrorException(String msg) {
- super(msg);
- }
-
- public DiskErrorException(String msg, Throwable cause) {
- super(msg, cause);
- }
- }
-
- public static class DiskOutOfSpaceException extends IOException {
- public DiskOutOfSpaceException(String msg) {
- super(msg);
- }
- }
-
- // Provider that abstracts some FileOutputStream operations for
- // testability.
- private static AtomicReference<FileIoProvider> fileIoProvider =
- new AtomicReference<>(new DefaultFileIoProvider());
-
- /**
- * Create the directory if it doesn't exist and check that dir is readable,
- * writable and executable
- *
- * @param dir dir to check
- * @throws DiskErrorException exception checking dir
- */
- public static void checkDir(File dir) throws DiskErrorException {
- checkDirInternal(dir);
- }
-
- /**
- * Create the directory if it doesn't exist and check that dir is
- * readable, writable and executable. Perform some disk IO to
- * ensure that the disk is usable for writes.
- *
- * @param dir dir to check
- * @throws DiskErrorException exception checking dir
- */
- public static void checkDirWithDiskIo(File dir)
- throws DiskErrorException {
- checkDirInternal(dir);
- doDiskIo(dir);
- }
-
- private static void checkDirInternal(File dir)
- throws DiskErrorException {
- if (!mkdirsWithExistsCheck(dir)) {
- throw new DiskErrorException("Cannot create directory: "
- + dir.toString());
- }
- checkAccessByFileMethods(dir);
- }
-
- /**
- * Create the local directory if necessary, check permissions and also ensure
- * it can be read from and written into.
- *
- * @param localFS local filesystem
- * @param dir directory
- * @param expected permission
- * @throws DiskErrorException exception checking dir
- * @throws IOException exception checking dir
- */
- public static void checkDir(LocalFileSystem localFS, Path dir,
- FsPermission expected)
- throws DiskErrorException, IOException {
- checkDirInternal(localFS, dir, expected);
- }
-
-
- /**
- * Create the local directory if necessary, also ensure permissions
- * allow it to be read from and written into. Perform some diskIO
- * to ensure that the disk is usable for writes.
- *
- * @param localFS local filesystem
- * @param dir directory
- * @param expected permission
- * @throws DiskErrorException exception checking dir
- * @throws IOException exception checking dir
- */
- public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir,
- FsPermission expected)
- throws DiskErrorException, IOException {
- checkDirInternal(localFS, dir, expected);
- doDiskIo(localFS.pathToFile(dir));
- }
-
- private static void checkDirInternal(LocalFileSystem localFS, Path dir,
- FsPermission expected)
- throws DiskErrorException, IOException {
- mkdirsWithExistsAndPermissionCheck(localFS, dir, expected);
- checkAccessByFileMethods(localFS.pathToFile(dir));
- }
-
- /**
- * Checks that the current running process can read, write, and execute the
- * given directory by using methods of the File object.
- *
- * @param dir File to check
- * @throws DiskErrorException if dir is not readable, not writable, or not
- * executable
- */
- private static void checkAccessByFileMethods(File dir)
- throws DiskErrorException {
- if (!dir.isDirectory()) {
- throw new DiskErrorException("Not a directory: "
- + dir.toString());
- }
-
- if (!FileUtil.canRead(dir)) {
- throw new DiskErrorException("Directory is not readable: "
- + dir.toString());
- }
-
- if (!FileUtil.canWrite(dir)) {
- throw new DiskErrorException("Directory is not writable: "
- + dir.toString());
- }
- }
-
- /**
- * The semantics of mkdirsWithExistsCheck method is different from the mkdirs
- * method provided in the Sun's java.io.File class in the following way:
- * While creating the non-existent parent directories, this method checks for
- * the existence of those directories if the mkdir fails at any point (since
- * that directory might have just been created by some other process).
- * If both mkdir() and the exists() check fails for any seemingly
- * non-existent directory, then we signal an error; Sun's mkdir would signal
- * an error (return false) if a directory it is attempting to create already
- * exists or the mkdir fails.
- * @param dir dir to create and check
- * @return true on success, false on failure
- */
- private static boolean mkdirsWithExistsCheck(File dir) {
- if (dir.mkdir() || dir.exists()) {
- return true;
- }
- File canonDir;
- try {
- canonDir = dir.getCanonicalFile();
- } catch (IOException e) {
- return false;
- }
- String parent = canonDir.getParent();
- return (parent != null) &&
- (mkdirsWithExistsCheck(new File(parent)) &&
- (canonDir.mkdir() || canonDir.exists()));
- }
-
- /**
- * Create the directory or check permissions if it already exists.
- *
- * The semantics of mkdirsWithExistsAndPermissionCheck method is different
- * from the mkdirs method provided in the Sun's java.io.File class in the
- * following way:
- * While creating the non-existent parent directories, this method checks for
- * the existence of those directories if the mkdir fails at any point (since
- * that directory might have just been created by some other process).
- * If both mkdir() and the exists() check fails for any seemingly
- * non-existent directory, then we signal an error; Sun's mkdir would signal
- * an error (return false) if a directory it is attempting to create already
- * exists or the mkdir fails.
- *
- * @param localFS local filesystem
- * @param dir directory to be created or checked
- * @param expected expected permission
- * @throws IOException exception making dir and checking
- */
- static void mkdirsWithExistsAndPermissionCheck(
- LocalFileSystem localFS, Path dir, FsPermission expected)
- throws IOException {
- File directory = localFS.pathToFile(dir);
- boolean created = false;
-
- if (!directory.exists())
- created = mkdirsWithExistsCheck(directory);
-
- if (created || !localFS.getFileStatus(dir).getPermission().equals(expected))
- localFS.setPermission(dir, expected);
- }
-
- // State related to running disk IO checks.
- private static final String DISK_IO_FILE_PREFIX =
- "DiskChecker.OK_TO_DELETE_.";
-
- @VisibleForTesting
- static final int DISK_IO_MAX_ITERATIONS = 3;
-
- /**
- * Performs some disk IO by writing to a new file in the given directory
- * and sync'ing file contents to disk.
- *
- * This increases the likelihood of catching catastrophic disk/controller
- * failures sooner.
- *
- * @param dir directory to be checked.
- * @throws DiskErrorException if we hit an error while trying to perform
- * disk IO against the file.
- */
- private static void doDiskIo(File dir) throws DiskErrorException {
- try {
- IOException ioe = null;
-
- for (int i = 0; i < DISK_IO_MAX_ITERATIONS; ++i) {
- final File file = getFileNameForDiskIoCheck(dir, i+1);
- try {
- diskIoCheckWithoutNativeIo(file);
- return;
- } catch (IOException e) {
- // Let's retry a few times before we really give up and
- // declare the disk as bad.
- ioe = e;
- }
- }
- throw ioe; // Just rethrow the last exception to signal failure.
- } catch(IOException e) {
- throw new DiskErrorException("Error checking directory " + dir, e);
- }
- }
-
- /**
- * Try to perform some disk IO by writing to the given file
- * without using Native IO.
- *
- * @param file file to check
- * @throws IOException if there was a non-retriable error.
- */
- private static void diskIoCheckWithoutNativeIo(File file)
- throws IOException {
- FileOutputStream fos = null;
-
- try {
- final FileIoProvider provider = fileIoProvider.get();
- fos = provider.get(file);
- provider.write(fos, new byte[1]);
- fos.getFD().sync();
- fos.close();
- fos = null;
- if (!file.delete() && file.exists()) {
- throw new IOException("Failed to delete " + file);
- }
- file = null;
- } finally {
- IOUtils.cleanupWithLogger(LOG, fos);
- FileUtils.deleteQuietly(file);
- }
- }
-
- /**
- * Generate a path name for a test file under the given directory.
- *
- * @return file object.
- */
- @VisibleForTesting
- static File getFileNameForDiskIoCheck(File dir, int iterationCount) {
- if (iterationCount < DISK_IO_MAX_ITERATIONS) {
- // Use file names of the format prefix.001 by default.
- return new File(dir,
- DISK_IO_FILE_PREFIX + String.format("%03d", iterationCount));
- } else {
- // If the first few checks then fail, try using a randomly generated
- // file name.
- return new File(dir, DISK_IO_FILE_PREFIX + UUID.randomUUID());
- }
- }
-
- /**
- * An interface that abstracts operations on {@link FileOutputStream}
- * objects for testability.
- */
- interface FileIoProvider {
- FileOutputStream get(File f) throws FileNotFoundException;
- void write(FileOutputStream fos, byte[] data) throws IOException;
- }
-
- /**
- * The default implementation of {@link FileIoProvider}.
- */
- private static class DefaultFileIoProvider implements FileIoProvider {
- /**
- * See {@link FileOutputStream#FileOutputStream(File)}.
- */
- @Override
- public FileOutputStream get(File f) throws FileNotFoundException {
- return new FileOutputStream(f);
- }
-
- /**
- * See {@link FileOutputStream#write(byte[])}.
- */
- @Override
- public void write(FileOutputStream fos, byte[] data) throws IOException {
- fos.write(data);
- }
- }
-
- /**
- * Replace the {@link FileIoProvider} for tests.
- * This method MUST NOT be used outside of unit tests.
- *
- * @param newFosProvider FileIoProvider
- * @return the old FileIoProvider.
- */
- @VisibleForTesting
- static FileIoProvider replaceFileOutputStreamProvider(
- FileIoProvider newFosProvider) {
- return fileIoProvider.getAndSet(newFosProvider);
- }
-
- /**
- * Retrieve the current {@link FileIoProvider}.
- * This method MUST NOT be used outside of unit tests.
- *
- * @return the current FileIoProvider.
- */
- @VisibleForTesting
- static FileIoProvider getFileOutputStreamProvider() {
- return fileIoProvider.get();
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/util/PlatformName.java b/solr/modules/hdfs/src/test/org/apache/hadoop/util/PlatformName.java
deleted file mode 100644
index d8a8fb39ab9..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/hadoop/util/PlatformName.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.util;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.solr.client.solrj.util.Constants;
-import org.apache.solr.common.util.EnvUtils;
-
-/**
- * A helper class for getting build-info of the java-vm.
- *
- */
-public class PlatformName {
- /**
- * The complete platform 'name' to identify the platform as
- * per the java-vm.
- */
- public static final String PLATFORM_NAME =
- (EnvUtils.getProperty("os.name").startsWith("Windows")
- ? System.getenv("os") : EnvUtils.getProperty("os.name"))
- + "-" + EnvUtils.getProperty("os.arch")
- + "-" + EnvUtils.getProperty("sun.arch.data.model");
-
- /**
- * The java vendor name used in this platform.
- */
- public static final String JAVA_VENDOR_NAME = EnvUtils.getProperty("java.vendor");
-
- /**
- * A public static variable to indicate the current java vendor is
- * IBM java or not.
- */
- public static final boolean IBM_JAVA = Constants.IS_IBM_JAVA;
-
- public static void main(String[] args) {
- System.out.println(PLATFORM_NAME);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/HdfsDirectoryFactoryTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/HdfsDirectoryFactoryTest.java
deleted file mode 100644
index cf37d6a8736..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/HdfsDirectoryFactoryTest.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Random;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.NoLockFactory;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.cloud.api.collections.SplitShardCmd;
-import org.apache.solr.common.util.EnvUtils;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.DirectoryFactory.DirContext;
-import org.apache.solr.handler.SnapShooter;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.store.HdfsLocalityReporter;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.apache.solr.metrics.MetricsMap;
-import org.apache.solr.metrics.SolrMetricManager;
-import org.apache.solr.metrics.SolrMetricsContext;
-import org.apache.solr.util.MockCoreContainer.MockCoreDescriptor;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@ThreadLeakFilters(
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- System.clearProperty(SplitShardCmd.SHARDSPLIT_CHECKDISKSPACE_ENABLED);
- System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
- System.clearProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY);
- System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
- System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
- System.clearProperty(HdfsDirectoryFactory.LOCALITYMETRICS_ENABLED);
- }
- }
-
- @Test
- @SuppressWarnings({"try"})
- public void testInitArgsOrSysPropConfig() throws Exception {
- try (HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
- // test sys prop config
- System.setProperty(
- HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
- hdfsFactory.init(new NamedList<>());
- String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
-
- assertTrue(dataHome.endsWith("/solr1/mock/data"));
- assertFalse(
- Boolean.parseBoolean(
- EnvUtils.getProperty(SplitShardCmd.SHARDSPLIT_CHECKDISKSPACE_ENABLED)));
-
- System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
-
- // test init args config
- NamedList<Object> nl = new NamedList<>();
- nl.add(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr2");
- hdfsFactory.init(nl);
- dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
-
- assertTrue(dataHome.endsWith("/solr2/mock/data"));
-
- // test sys prop and init args config - init args wins
- System.setProperty(
- HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
- hdfsFactory.init(nl);
- dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
-
- assertTrue(dataHome.endsWith("/solr2/mock/data"));
-
- System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
-
- // set conf dir by sys prop
- Path confDir = createTempDir();
-
- System.setProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY, confDir.toString());
-
- try (Directory dir =
- hdfsFactory.create(
- HdfsTestUtil.getURI(dfsCluster) + "/solr",
- NoLockFactory.INSTANCE,
- DirContext.DEFAULT)) {
- assertEquals(confDir.toString(), hdfsFactory.getConfDir());
- }
-
- // check bool and int getConf impls
- nl = new NamedList<>();
- nl.add(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 4);
- System.setProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, "3");
- nl.add(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, true);
- System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
-
- hdfsFactory.init(nl);
-
- assertEquals(
- 4, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
- assertTrue(hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
-
- nl = new NamedList<>();
- hdfsFactory.init(nl);
- System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "true");
-
- assertEquals(
- 3, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
- assertTrue(hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
-
- System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
- System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
-
- assertEquals(
- 0, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
- assertFalse(hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
- }
- }
-
- @Test
- public void testCleanupOldIndexDirectories() throws Exception {
- try (HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
- System.setProperty(
- HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
- hdfsFactory.init(new NamedList<>());
- String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
- assertTrue(dataHome.endsWith("/solr1/mock/data"));
- System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
-
- try (FileSystem hdfs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) {
- org.apache.hadoop.fs.Path dataHomePath = new org.apache.hadoop.fs.Path(dataHome);
- org.apache.hadoop.fs.Path currentIndexDirPath =
- new org.apache.hadoop.fs.Path(dataHomePath, "index");
- assertFalse(checkHdfsDirectory(hdfs, currentIndexDirPath));
- hdfs.mkdirs(currentIndexDirPath);
- assertTrue(checkHdfsDirectory(hdfs, currentIndexDirPath));
-
- String timestamp1 =
- new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date());
- org.apache.hadoop.fs.Path oldIndexDirPath =
- new org.apache.hadoop.fs.Path(dataHomePath, "index." + timestamp1);
- assertFalse(checkHdfsDirectory(hdfs, oldIndexDirPath));
- hdfs.mkdirs(oldIndexDirPath);
- assertTrue(checkHdfsDirectory(hdfs, oldIndexDirPath));
-
- hdfsFactory.cleanupOldIndexDirectories(
- dataHomePath.toString(), currentIndexDirPath.toString(), false);
-
- assertTrue(checkHdfsDirectory(hdfs, currentIndexDirPath));
- assertFalse(checkHdfsDirectory(hdfs, oldIndexDirPath));
- }
- }
- }
-
- private boolean checkHdfsDirectory(FileSystem hdfs, org.apache.hadoop.fs.Path path)
- throws IOException {
- try {
- return hdfs.getFileStatus(path).isDirectory();
- } catch (FileNotFoundException e) {
- return false;
- }
- }
-
- @Test
- public void testLocalityReporter() throws Exception {
- Random r = random();
- try (HdfsDirectoryFactory factory = new HdfsDirectoryFactory()) {
- SolrMetricManager metricManager = new SolrMetricManager();
- String registry = TestUtil.randomSimpleString(r, 2, 10);
- String scope = TestUtil.randomSimpleString(r, 2, 10);
- Map<String, String> props = new HashMap<>();
- props.put(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr");
- props.put(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
- props.put(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_ENABLE, "false");
- props.put(HdfsDirectoryFactory.LOCALITYMETRICS_ENABLED, "true");
- factory.init(new NamedList<>(props));
- factory.initializeMetrics(new SolrMetricsContext(metricManager, registry, "foo"), scope);
-
- // get the metrics map for the locality bean
- MetricsMap metrics =
- (MetricsMap)
- ((SolrMetricManager.GaugeWrapper)
- metricManager
- .registry(registry)
- .getMetrics()
- .get("OTHER." + scope + ".hdfsLocality"))
- .getGauge();
- // We haven't done anything, so there should be no data
- Map<String, Object> statistics = metrics.getValue();
- assertEquals(
- "Saw bytes that were not written: "
- + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL),
- 0L,
- statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL));
- assertEquals(
- "Counted bytes as local when none written: "
- + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO),
- 0,
- statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO));
-
- // create a directory and a file
- String path = HdfsTestUtil.getURI(dfsCluster) + "/solr3/";
- try (Directory dir = factory.create(path, NoLockFactory.INSTANCE, DirContext.DEFAULT)) {
- try (IndexOutput writer = dir.createOutput("output", null)) {
- writer.writeLong(42L);
- }
-
- final long long_bytes = Long.SIZE / Byte.SIZE;
-
- // no locality because hostname not set
- factory.setHost("bogus");
- statistics = metrics.getValue();
- assertEquals(
- "Wrong number of total bytes counted: "
- + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL),
- long_bytes,
- statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL));
- assertEquals(
- "Wrong number of total blocks counted: "
- + statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_TOTAL),
- 1,
- statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_TOTAL));
- assertEquals(
- "Counted block as local when bad hostname set: "
- + statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_LOCAL),
- 0,
- statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_LOCAL));
-
- // set hostname and check again
- factory.setHost("127.0.0.1");
- statistics = metrics.getValue();
- assertEquals(
- "Did not count block as local after setting hostname: "
- + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_LOCAL),
- long_bytes,
- statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_LOCAL));
- }
- }
- }
-
- @Test
- public void testIsAbsolute() throws Exception {
- try (HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
- String pathToRepeat =
- RandomStrings.randomAsciiAlphanumOfLength(random(), random().nextInt(10) + 1) + '/';
- String relativePath = pathToRepeat.repeat(random().nextInt(5) + 1);
- assertFalse(hdfsFactory.isAbsolute(relativePath));
- assertFalse(hdfsFactory.isAbsolute("/" + relativePath));
-
- for (String rootPrefix : Arrays.asList("file://", "hdfs://", "s3a://", "foo://")) {
- assertTrue(hdfsFactory.isAbsolute(rootPrefix + relativePath));
- }
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/backup/repository/HdfsBackupRepositoryIntegrationTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/backup/repository/HdfsBackupRepositoryIntegrationTest.java
deleted file mode 100644
index 525bc7c00f7..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/backup/repository/HdfsBackupRepositoryIntegrationTest.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.hdfs.backup.repository;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.solr.cloud.api.collections.AbstractBackupRepositoryTest;
-import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.backup.repository.BackupRepository;
-import org.apache.solr.hdfs.HdfsDirectoryFactory;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
- "SimpleText"
-}) // Backups do checksum validation against a footer value not present in 'SimpleText'
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsBackupRepositoryIntegrationTest extends AbstractBackupRepositoryTest {
- private static MiniDFSCluster dfsCluster;
- private static String hdfsUri;
- private static FileSystem fs;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- hdfsUri = HdfsTestUtil.getURI(dfsCluster);
- try {
- URI uri = new URI(hdfsUri);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- fs = FileSystem.get(uri, conf);
-
- if (fs instanceof DistributedFileSystem) {
- // Make sure dfs is not in safe mode
- while (((DistributedFileSystem) fs)
- .setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET, true)) {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- Thread.interrupted();
- // continue
- }
- }
- }
-
- fs.mkdirs(new org.apache.hadoop.fs.Path("/backup"));
- } catch (IOException | URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- System.setProperty("solr.hdfs.default.backup.path", "/backup");
- System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
- useFactory("solr.StandardDirectoryFactory");
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- IOUtils.closeQuietly(fs);
- fs = null;
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
- }
- }
-
- @Override
- protected Class<? extends BackupRepository> getRepositoryClass() {
- return HdfsBackupRepository.class;
- }
-
- @Override
- protected BackupRepository getRepository() {
- HdfsBackupRepository repository = new HdfsBackupRepository();
- repository.init(getBaseBackupRepositoryConfiguration());
- return repository;
- }
-
- @Override
- protected URI getBaseUri() throws URISyntaxException {
- return new URI(hdfsUri + "/solr/tmp");
- }
-
- @Override
- protected NamedList<Object> getBaseBackupRepositoryConfiguration() {
- NamedList<Object> config = new NamedList<>();
- config.add(HdfsDirectoryFactory.HDFS_HOME, hdfsUri + "/solr");
- return config;
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/backup/repository/HdfsBackupRepositoryTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/backup/repository/HdfsBackupRepositoryTest.java
deleted file mode 100644
index 635f2b77b2c..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/backup/repository/HdfsBackupRepositoryTest.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.hdfs.backup.repository;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.solr.SolrTestCase;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.hdfs.HdfsDirectoryFactory;
-import org.apache.solr.hdfs.store.HdfsDirectory;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.Test;
-
- "SimpleText"
-}) // Backups do checksum validation against a footer value not present in 'SimpleText'
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsBackupRepositoryTest extends SolrTestCase {
-
- @Test(expected = NullPointerException.class)
- public void testHdfsHomePropertyMissing() throws IOException {
- try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
- NamedList<Object> namedList = new SimpleOrderedMap<>();
- hdfsBackupRepository.init(namedList);
- }
- }
-
- @Test
- public void testHdfsHomePropertySet() throws IOException {
- try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
- NamedList<Object> namedList = new SimpleOrderedMap<>();
- namedList.add(HdfsDirectoryFactory.HDFS_HOME, "hdfs://localhost");
- hdfsBackupRepository.init(namedList);
- }
- }
-
- @Test(expected = ClassCastException.class)
- public void testCopyBufferSizeNonNumeric() throws IOException {
- try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
- NamedList<Object> namedList = new SimpleOrderedMap<>();
- namedList.add("solr.hdfs.buffer.size", "xyz");
- hdfsBackupRepository.init(namedList);
- }
- }
-
- @Test(expected = ClassCastException.class)
- public void testCopyBufferSizeWrongType() throws IOException {
- try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
- NamedList<Object> namedList = new SimpleOrderedMap<>();
- namedList.add("solr.hdfs.buffer.size", "8192");
- hdfsBackupRepository.init(namedList);
- }
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testCopyBufferSizeNegative() throws IOException {
- try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
- NamedList<Object> namedList = new SimpleOrderedMap<>();
- namedList.add("solr.hdfs.buffer.size", -1);
- hdfsBackupRepository.init(namedList);
- }
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testCopyBufferSizeZero() throws IOException {
- try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
- NamedList<Object> namedList = new SimpleOrderedMap<>();
- namedList.add("solr.hdfs.buffer.size", 0);
- hdfsBackupRepository.init(namedList);
- }
- }
-
- @Test
- public void testCopyBufferSet() throws IOException {
- try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
- NamedList<Object> namedList = new SimpleOrderedMap<>();
- namedList.add(HdfsDirectoryFactory.HDFS_HOME, "hdfs://localhost");
- namedList.add("solr.hdfs.buffer.size", 32768);
- hdfsBackupRepository.init(namedList);
- assertEquals(hdfsBackupRepository.copyBufferSize, 32768);
- }
- }
-
- @Test
- public void testCopyBufferDefaultSize() throws IOException {
- try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
- NamedList<Object> namedList = new SimpleOrderedMap<>();
- namedList.add(HdfsDirectoryFactory.HDFS_HOME, "hdfs://localhost");
- hdfsBackupRepository.init(namedList);
- assertEquals(hdfsBackupRepository.copyBufferSize, HdfsDirectory.DEFAULT_BUFFER_SIZE);
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/DisableKeyStoresFactory.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/DisableKeyStoresFactory.java
deleted file mode 100644
index 7e88a83b066..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/DisableKeyStoresFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.TrustManager;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.ssl.KeyStoresFactory;
-import org.apache.hadoop.security.ssl.SSLFactory;
-
-public class DisableKeyStoresFactory implements KeyStoresFactory {
- @Override
- public void init(SSLFactory.Mode mode) throws IOException, GeneralSecurityException {}
-
- @Override
- public void destroy() {}
-
- @Override
- public KeyManager[] getKeyManagers() {
- return new KeyManager[0];
- }
-
- @Override
- public TrustManager[] getTrustManagers() {
- return new TrustManager[0];
- }
-
- @Override
- public void setConf(Configuration conf) {}
-
- @Override
- public Configuration getConf() {
- return null;
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsBasicDistributedZk2Test.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsBasicDistributedZk2Test.java
deleted file mode 100644
index 921569ee651..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsBasicDistributedZk2Test.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractBasicDistributedZk2TestBase;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsBasicDistributedZk2Test extends AbstractBasicDistributedZk2TestBase {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- protected String getDataDir(String dataDir) throws IOException {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsBasicDistributedZkTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsBasicDistributedZkTest.java
deleted file mode 100644
index 6b4d8daafa8..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsBasicDistributedZkTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractBasicDistributedZkTestBase;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsBasicDistributedZkTest extends AbstractBasicDistributedZkTestBase {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @Override
- protected boolean useTlogReplicas() {
- return false;
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- protected String getDataDir(String dataDir) throws IOException {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-
- @Test
- @Override
- @ShardsFixed(num = 4)
- public void test() throws Exception {
- super.test();
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsChaosMonkeyNothingIsSafeTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsChaosMonkeyNothingIsSafeTest.java
deleted file mode 100644
index 0d9744f0034..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsChaosMonkeyNothingIsSafeTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractChaosMonkeyNothingIsSafeTestBase;
-import org.apache.solr.hdfs.HdfsDirectoryFactory;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsChaosMonkeyNothingIsSafeTest extends AbstractChaosMonkeyNothingIsSafeTestBase {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- System.setProperty(
- "solr.hdfs.blockcache.global",
- "true"); // always use global cache, this test can create a lot of directories
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.global");
- }
- }
-
- @Override
- public void distribSetUp() throws Exception {
- super.distribSetUp();
-
- // super class may hard code directory
- useFactory(HdfsDirectoryFactory.class.getCanonicalName());
- }
-
- @Override
- protected String getDataDir(String dataDir) throws IOException {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsChaosMonkeySafeLeaderTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsChaosMonkeySafeLeaderTest.java
deleted file mode 100644
index e7dbee6b96f..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsChaosMonkeySafeLeaderTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractChaosMonkeySafeLeaderTestBase;
-import org.apache.solr.hdfs.HdfsDirectoryFactory;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsChaosMonkeySafeLeaderTest extends AbstractChaosMonkeySafeLeaderTestBase {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- System.setProperty(
- "solr.hdfs.blockcache.global",
- "true"); // always use global cache, this test can create a lot of directories
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.global");
- }
- }
-
- @Override
- public void distribSetUp() throws Exception {
- super.distribSetUp();
-
- // super class may hard code directory
- useFactory(HdfsDirectoryFactory.class.getCanonicalName());
- }
-
- @Override
- protected String getDataDir(String dataDir) throws IOException {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-
- @Override
- protected String getDirectoryFactory() {
- return HdfsDirectoryFactory.class.getCanonicalName();
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsCollectionsApiTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsCollectionsApiTest.java
deleted file mode 100644
index 0b69ee4c8f8..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsCollectionsApiTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shut down
-public class HdfsCollectionsApiTest extends SolrCloudTestCase {
-
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- configureCluster(2).configure();
-
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
-
- cluster.uploadConfigSet(configset("cloud-hdfs"), "conf1");
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- shutdownCluster(); // need to close before the MiniDFSCluster
- cluster = null;
- } finally {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
- }
-
- public void testDataDirIsNotReused() throws Exception {
- JettySolrRunner jettySolrRunner = cluster.getJettySolrRunner(0);
- String collection = "test";
- CollectionAdminRequest.createCollection(collection, "conf1", 1, 1)
- .setCreateNodeSet(jettySolrRunner.getNodeName())
- .process(cluster.getSolrClient());
- waitForState("", collection, clusterShape(1, 1));
- try (CloudSolrClient solrClient =
- cluster.basicSolrClientBuilder().withDefaultCollection(collection).build()) {
- solrClient.add(new SolrInputDocument("id", "1"));
- solrClient.add(new SolrInputDocument("id", "2"));
- solrClient.commit();
- solrClient.add(new SolrInputDocument("id", "3"));
-
- jettySolrRunner.stop();
- waitForState(
- "Waiting for replica to be marked down",
- collection,
- collectionState -> {
- Replica replica = collectionState.getSlice("shard1").getReplicas().iterator().next();
- return replica.getState() == Replica.State.DOWN;
- });
- CollectionAdminRequest.deleteCollection(collection).process(cluster.getSolrClient());
-
- jettySolrRunner.start();
-
- CollectionAdminRequest.createCollection(collection, "conf1", 1, 1)
- .setCreateNodeSet(cluster.getJettySolrRunner(1).getNodeName())
- .process(cluster.getSolrClient());
- waitForState("", collection, clusterShape(1, 1));
- QueryResponse response = cluster.getSolrClient().query(collection, new SolrQuery("*:*"));
- assertEquals(0L, response.getResults().getNumFound());
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsFakeGroupMapping.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsFakeGroupMapping.java
deleted file mode 100644
index 39a755f8505..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsFakeGroupMapping.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import java.util.Collections;
-import java.util.List;
-import org.apache.hadoop.security.GroupMappingServiceProvider;
-
-/** Fake mapping for Hadoop to prevent falling back to Shell group provider */
-public class HdfsFakeGroupMapping implements GroupMappingServiceProvider {
- @Override
- public List<String> getGroups(String user) {
- return Collections.singletonList("supergroup");
- }
-
- @Override
- public void cacheGroupsRefresh() {}
-
- @Override
- public void cacheGroupsAdd(List<String> groups) {}
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsNameNodeFailoverTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsNameNodeFailoverTest.java
deleted file mode 100644
index c77fe3c1d52..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsNameNodeFailoverTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractBasicDistributedZkTestBase;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsNameNodeFailoverTest extends AbstractBasicDistributedZkTestBase {
- private static final String COLLECTION = "collection";
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false, true);
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- protected String getDataDir(String dataDir) throws IOException {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-
- public HdfsNameNodeFailoverTest() {
- super();
- sliceCount = 1;
- fixShardCount(TEST_NIGHTLY ? 7 : random().nextInt(2) + 1);
- }
-
- @Override
- protected String getSolrXml() {
- return "solr.xml";
- }
-
- @Test
- @Override
- public void test() throws Exception {
- createCollection(COLLECTION, "conf1", 1, 1);
-
- waitForRecoveriesToFinish(COLLECTION, false);
-
- // TODO: SOLR-7360 Enable HDFS NameNode failover testing.
- // dfsCluster.transitionToStandby(0);
- // dfsCluster.transitionToActive(1);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsRecoverLeaseTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsRecoverLeaseTest.java
deleted file mode 100644
index e0d2691b22c..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsRecoverLeaseTest.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import java.net.URI;
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.apache.solr.hdfs.util.HdfsRecoverLeaseFileSystemUtils;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
-
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- }
-
- @Override
- @After
- public void tearDown() throws Exception {
- super.tearDown();
- }
-
- @Test
- public void testBasic() throws IOException {
- long startRecoverLeaseSuccessCount =
- HdfsRecoverLeaseFileSystemUtils.RECOVER_LEASE_SUCCESS_COUNT.get();
-
- URI uri = dfsCluster.getURI();
- Path path = new Path(uri);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- FileSystem fs1 = FileSystem.get(path.toUri(), conf);
- Path testFile = new Path(uri + "/testfile");
- FSDataOutputStream out = fs1.create(testFile);
-
- out.write(5);
- out.hflush();
- out.close();
-
- HdfsRecoverLeaseFileSystemUtils.recoverFileLease(fs1, testFile, conf, () -> false);
- assertEquals(
- 0,
- HdfsRecoverLeaseFileSystemUtils.RECOVER_LEASE_SUCCESS_COUNT.get()
- - startRecoverLeaseSuccessCount);
-
- fs1.close();
-
- FileSystem fs2 = FileSystem.get(path.toUri(), conf);
- Path testFile2 = new Path(uri + "/testfile2");
- FSDataOutputStream out2 = fs2.create(testFile2);
-
- if (random().nextBoolean()) {
- int cnt = random().nextInt(100);
- for (int i = 0; i < cnt; i++) {
- out2.write(random().nextInt(20000));
- }
- out2.hflush();
- }
-
- // closing the fs will close the file it seems
- // fs2.close();
-
- FileSystem fs3 = FileSystem.get(path.toUri(), conf);
-
- HdfsRecoverLeaseFileSystemUtils.recoverFileLease(fs3, testFile2, conf, () -> false);
- assertEquals(
- 1,
- HdfsRecoverLeaseFileSystemUtils.RECOVER_LEASE_SUCCESS_COUNT.get()
- - startRecoverLeaseSuccessCount);
-
- fs3.close();
- fs2.close();
- }
-
- @Test
- @SuppressWarnings("DoNotCall")
- public void testMultiThreaded() throws Exception {
- long startRecoverLeaseSuccessCount =
- HdfsRecoverLeaseFileSystemUtils.RECOVER_LEASE_SUCCESS_COUNT.get();
-
- final URI uri = dfsCluster.getURI();
- final Path path = new Path(uri);
- final Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
-
- // n threads create files
- class WriterThread extends Thread {
- private final FileSystem fs;
- private final int id;
-
- public WriterThread(int id) {
- this.id = id;
- try {
- fs = FileSystem.get(path.toUri(), conf);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void run() {
- Path testFile = new Path(uri + "/file-" + id);
- FSDataOutputStream out;
- try {
- out = fs.create(testFile);
-
- if (random().nextBoolean()) {
- int cnt = random().nextInt(100);
- for (int i = 0; i < cnt; i++) {
- out.write(random().nextInt(20000));
- }
- out.hflush();
- }
- } catch (IOException e) {
- throw new RuntimeException();
- }
- }
-
- public void close() throws IOException {
- fs.close();
- }
-
- public int getFileId() {
- return id;
- }
- }
-
- class RecoverThread extends Thread {
- private final FileSystem fs;
- private final int id;
-
- public RecoverThread(int id) {
- this.id = id;
- try {
- fs = FileSystem.get(path.toUri(), conf);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void run() {
- Path testFile = new Path(uri + "/file-" + id);
- try {
- HdfsRecoverLeaseFileSystemUtils.recoverFileLease(fs, testFile, conf, () -> false);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public void close() throws IOException {
- fs.close();
- }
- }
-
- Set<WriterThread> writerThreads = new HashSet<>();
- Set<RecoverThread> recoverThreads = new HashSet<>();
-
- int threadCount = 3;
- for (int i = 0; i < threadCount; i++) {
- WriterThread wt = new WriterThread(i);
- writerThreads.add(wt);
- // should be wt.start();
- wt.run();
- }
-
- for (WriterThread wt : writerThreads) {
- wt.join();
- }
-
- Thread.sleep(2000);
-
- for (WriterThread wt : writerThreads) {
- RecoverThread rt = new RecoverThread(wt.getFileId());
- recoverThreads.add(rt);
- // should be rt.start();
- rt.run();
- }
-
- for (WriterThread wt : writerThreads) {
- wt.close();
- }
-
- for (RecoverThread rt : recoverThreads) {
- rt.close();
- }
-
- assertEquals(
- threadCount,
- HdfsRecoverLeaseFileSystemUtils.RECOVER_LEASE_SUCCESS_COUNT.get()
- - startRecoverLeaseSuccessCount);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsRecoveryZkTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsRecoveryZkTest.java
deleted file mode 100644
index 3dbf810a075..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsRecoveryZkTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractRecoveryZkTestBase;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsRecoveryZkTest extends AbstractRecoveryZkTestBase {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
-
- cluster.uploadConfigSet(configset("cloud-hdfs"), "conf");
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- shutdownCluster(); // need to close before the MiniDFSCluster
- } finally {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsRestartWhileUpdatingTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsRestartWhileUpdatingTest.java
deleted file mode 100644
index 32a973dd841..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsRestartWhileUpdatingTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractRestartWhileUpdatingTestBase;
-import org.apache.solr.hdfs.HdfsDirectoryFactory;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsRestartWhileUpdatingTest extends AbstractRestartWhileUpdatingTestBase {
- private static MiniDFSCluster dfsCluster;
-
- public HdfsRestartWhileUpdatingTest() throws Exception {
- super();
-
- // The constructor runs after setupClass so reset the DirectoryFactory
- useFactory(HdfsDirectoryFactory.class.getCanonicalName());
- }
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- protected String getDataDir(String dataDir) {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsSyncSliceTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsSyncSliceTest.java
deleted file mode 100644
index b2dcd1d95c0..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsSyncSliceTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractSyncSliceTestBase;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsSyncSliceTest extends AbstractSyncSliceTestBase {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- protected String getDataDir(String dataDir) throws IOException {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsTestUtil.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsTestUtil.java
deleted file mode 100644
index 98eea035e1b..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsTestUtil.java
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import java.io.File;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.ForkJoinWorkerThread;
-import java.util.regex.Pattern;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.HardLink;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.MiniDFSNNTopology;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker;
-import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.util.DiskChecker;
-import org.apache.lucene.util.Constants;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.util.EnvUtils;
-import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.hdfs.HdfsDirectoryFactory;
-import org.apache.solr.hdfs.util.HdfsUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HdfsTestUtil {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final String SOLR_HACK_FOR_CLASS_VERIFICATION_FIELD =
- "SOLR_HACK_FOR_CLASS_VERIFICATION";
-
- private static final String LOGICAL_HOSTNAME = "ha-nn-uri-%d";
-
- private static final boolean HA_TESTING_ENABLED = false; // SOLR-XXX
-
- private static Map<MiniDFSCluster, Timer> timers = new HashMap<>();
- private static final Object TIMERS_LOCK = new Object();
-
- private static FSDataOutputStream badTlogOutStream;
-
- private static FileSystem badTlogOutStreamFs;
-
- public static MiniDFSCluster setupClass(String dir) throws Exception {
- return setupClass(dir, true, true);
- }
-
- public static MiniDFSCluster setupClass(String dir, boolean haTesting) throws Exception {
- return setupClass(dir, haTesting, true);
- }
-
- public static void checkAssumptions() {
- ensureHadoopHomeNotSet();
- checkHadoopWindows();
- checkOverriddenHadoopClasses();
- checkFastDateFormat();
- checkGeneratedIdMatches();
- }
-
- /**
- * If Hadoop home is set via environment variable HADOOP_HOME or Java system property
- * hadoop.home.dir, the behavior of test is undefined. Ensure that these are not set before
- * starting. It is not possible to easily unset environment variables so better to bail out early
- * instead of trying to test.
- */
- private static void ensureHadoopHomeNotSet() {
- if (System.getenv("HADOOP_HOME") != null) {
- SolrTestCaseJ4.fail("Ensure that HADOOP_HOME environment variable is not set.");
- }
- if (EnvUtils.getProperty("hadoop.home.dir") != null) {
- SolrTestCaseJ4.fail("Ensure that \"hadoop.home.dir\" Java property is not set.");
- }
- }
-
- /** Hadoop integration tests fail on Windows without Hadoop NativeIO */
- private static void checkHadoopWindows() {
- SolrTestCaseJ4.assumeTrue(
- "Hadoop does not work on Windows without Hadoop NativeIO",
- !Constants.WINDOWS || NativeIO.isAvailable());
- }
-
- /** Ensure that the tests are picking up the modified Hadoop classes */
- private static void checkOverriddenHadoopClasses() {
- List<Class<?>> modifiedHadoopClasses =
- new ArrayList<>(
- Arrays.asList(
- DiskChecker.class,
- FileUtil.class,
- HardLink.class,
- NameNodeResourceChecker.class,
- RawLocalFileSystem.class));
- // Dodge weird scope errors from the compiler (SOLR-14417)
- try {
- modifiedHadoopClasses.add(
- Class.forName("org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.BlockPoolSlice"));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- for (Class<?> clazz : modifiedHadoopClasses) {
- try {
- SolrTestCaseJ4.assertNotNull(
- "Field on " + clazz.getCanonicalName() + " should not have been null",
- clazz.getField(SOLR_HACK_FOR_CLASS_VERIFICATION_FIELD));
- } catch (NoSuchFieldException e) {
- SolrTestCaseJ4.fail(
- "Expected to load Solr modified Hadoop class "
- + clazz.getCanonicalName()
- + " , but it was not found.");
- }
- }
- }
-
- /** Checks that commons-lang3 FastDateFormat works with configured locale */
- @SuppressForbidden(reason = "Call FastDateFormat.format same way Hadoop calls it")
- private static void checkFastDateFormat() {
- try {
- org.apache.commons.lang3.time.FastDateFormat.getInstance().format(System.currentTimeMillis());
- } catch (ArrayIndexOutOfBoundsException e) {
- SolrTestCaseJ4.assumeNoException(
- "commons-lang3 FastDateFormat doesn't work with " + Locale.getDefault().toLanguageTag(),
- e);
- }
- }
-
- /** Hadoop fails to generate locale agnostic ids - Checks that generated string matches */
- private static void checkGeneratedIdMatches() {
- // This is basically how Namenode generates fsimage ids and checks that the fsimage filename
- // matches
- SolrTestCaseJ4.assumeTrue(
- "Check that generated id matches regex",
- Pattern.matches("(\\d+)", String.format(Locale.getDefault(), "%019d", 0)));
- }
-
- public static MiniDFSCluster setupClass(String dir, boolean safeModeTesting, boolean haTesting)
- throws Exception {
- checkAssumptions();
-
- if (!HA_TESTING_ENABLED) haTesting = false;
-
- DefaultMetricsSystem.setInstance(new FakeMetricsSystem());
-
- Configuration conf = getBasicConfiguration(new Configuration());
- conf.set("hdfs.minidfs.basedir", dir + File.separator + "hdfsBaseDir");
- conf.set("dfs.namenode.name.dir", dir + File.separator + "nameNodeNameDir");
- // Disable metrics logging for HDFS
- conf.setInt("dfs.namenode.metrics.logger.period.seconds", 0);
- conf.setInt("dfs.datanode.metrics.logger.period.seconds", 0);
- // Disable GcTimeMonitor - HDFS-15176 introduced in Hadoop 3.3.0 - causes thead leaks
- conf.setBoolean("dfs.namenode.gc.time.monitor.enable", false);
-
- System.setProperty("test.build.data", dir + File.separator + "hdfs" + File.separator + "build");
- System.setProperty("test.cache.data", dir + File.separator + "hdfs" + File.separator + "cache");
- System.setProperty("solr.lock.type", HdfsDirectoryFactory.LOCK_TYPE_HDFS);
- System.setProperty(
- SolrTestCaseJ4.UPDATELOG_SYSPROP, HdfsDirectoryFactory.HDFS_UPDATE_LOG_CLASS_NAME);
-
- // test-files/solr/solr.xml sets this to be 15000. This isn't long enough for HDFS in some
- // cases.
- System.setProperty("socketTimeout", "90000");
-
- String blockcacheGlobal =
- EnvUtils.getProperty(
- "solr.hdfs.blockcache.global", Boolean.toString(SolrTestCaseJ4.random().nextBoolean()));
- System.setProperty("solr.hdfs.blockcache.global", blockcacheGlobal);
- // Limit memory usage for HDFS tests
- if (Boolean.parseBoolean(blockcacheGlobal)) {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "4096");
- } else {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
- System.setProperty("tests.hdfs.numdatanodes", "1");
- }
-
- int dataNodes = Integer.getInteger("tests.hdfs.numdatanodes", 1);
- final MiniDFSCluster.Builder dfsClusterBuilder =
- new MiniDFSCluster.Builder(conf).numDataNodes(dataNodes).format(true);
- if (haTesting) {
- dfsClusterBuilder.nnTopology(MiniDFSNNTopology.simpleHATopology());
- }
-
- MiniDFSCluster dfsCluster = dfsClusterBuilder.build();
- HdfsUtil.TEST_CONF = getClientConfiguration(dfsCluster);
- System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
-
- dfsCluster.waitActive();
-
- if (haTesting) dfsCluster.transitionToActive(0);
-
- int rndMode = SolrTestCaseJ4.random().nextInt(3);
- if (safeModeTesting && rndMode == 1) {
- NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
-
- int rnd = SolrTestCaseJ4.random().nextInt(10000);
- Timer timer = new Timer();
- synchronized (TIMERS_LOCK) {
- if (timers == null) {
- timers = new HashMap<>();
- }
- timers.put(dfsCluster, timer);
- }
- timer.schedule(
- new TimerTask() {
-
- @Override
- public void run() {
- NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
- }
- },
- rnd);
- } else if (haTesting && rndMode == 2) {
- int rnd = SolrTestCaseJ4.random().nextInt(30000);
- Timer timer = new Timer();
- synchronized (TIMERS_LOCK) {
- if (timers == null) {
- timers = new HashMap<>();
- }
- timers.put(dfsCluster, timer);
- }
- timer.schedule(
- new TimerTask() {
-
- @Override
- public void run() {
- // TODO: randomly transition to standby
- // try {
- // dfsCluster.transitionToStandby(0);
- // dfsCluster.transitionToActive(1);
- // } catch (IOException e) {
- // throw new RuntimeException();
- // }
-
- }
- },
- rnd);
- } else {
- // TODO: we could do much better at testing this
- // force a lease recovery by creating a tlog file and not closing it
- URI uri = dfsCluster.getURI();
- Path hdfsDirPath =
- new Path(
- uri.toString() + "/solr/collection1/core_node1/data/tlog/tlog.0000000000000000000");
- // tran log already being created testing
- badTlogOutStreamFs = FileSystem.get(hdfsDirPath.toUri(), getClientConfiguration(dfsCluster));
- badTlogOutStream = badTlogOutStreamFs.create(hdfsDirPath);
- }
-
- SolrTestCaseJ4.useFactory(HdfsDirectoryFactory.class.getCanonicalName());
-
- return dfsCluster;
- }
-
- private static Configuration getBasicConfiguration(Configuration conf) {
- conf.setBoolean("dfs.block.access.token.enable", false);
- conf.setBoolean("dfs.permissions.enabled", false);
- conf.set("hadoop.security.authentication", "simple");
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
- return conf;
- }
-
- public static Configuration getClientConfiguration(MiniDFSCluster dfsCluster) {
- Configuration conf = getBasicConfiguration(dfsCluster.getConfiguration(0));
- if (dfsCluster.getNumNameNodes() > 1) {
- HATestUtil.setFailoverConfigurations(dfsCluster, conf);
- }
- return conf;
- }
-
- public static void teardownClass(MiniDFSCluster dfsCluster) throws Exception {
- HdfsUtil.TEST_CONF = null;
-
- if (badTlogOutStream != null) {
- IOUtils.closeQuietly(badTlogOutStream);
- badTlogOutStream = null;
- }
-
- if (badTlogOutStreamFs != null) {
- IOUtils.closeQuietly(badTlogOutStreamFs);
- badTlogOutStreamFs = null;
- }
-
- try {
- try {
- SolrTestCaseJ4.resetFactory();
- } catch (Exception e) {
- log.error("Exception trying to reset solr.directoryFactory", e);
- }
- if (dfsCluster != null) {
- synchronized (TIMERS_LOCK) {
- if (timers != null) {
- Timer timer = timers.remove(dfsCluster);
- if (timer != null) {
- timer.cancel();
- }
- if (timers.isEmpty()) {
- timers = null;
- }
- }
- }
- try {
- dfsCluster.shutdown(true);
- } catch (Error e) {
- // Added in SOLR-7134
- // Rarely, this can fail to either a NullPointerException
- // or a class not found exception. The later may fixable
- // by adding test dependencies.
- log.error("Exception shutting down dfsCluster", e);
- }
- }
- } finally {
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
-
- System.clearProperty("socketTimeout");
-
- System.clearProperty("tests.hdfs.numdatanodes");
-
- System.clearProperty("solr.lock.type");
-
- System.clearProperty(SolrTestCaseJ4.UPDATELOG_SYSPROP);
-
- // Clear "solr.hdfs." system properties
- EnvUtils.getProperties().keySet().stream()
- .filter(key -> key.startsWith("solr.hdfs."))
- .forEach(System::clearProperty);
- }
- }
-
- public static String getDataDir(MiniDFSCluster dfsCluster, String dataDir) {
- if (dataDir == null) {
- return null;
- }
- String dir =
- "/" + new File(dataDir).toString().replace(":", "_").replace("/", "_").replace(" ", "_");
-
- return getURI(dfsCluster) + dir;
- }
-
- public static String getURI(MiniDFSCluster dfsCluster) {
- if (dfsCluster.getNameNodeInfos().length > 1) {
- String logicalName =
- String.format(
- Locale.ENGLISH,
- LOGICAL_HOSTNAME,
- dfsCluster.getInstanceId()); // NOTE: hdfs uses default locale
- return "hdfs://" + logicalName;
- } else {
- URI uri = dfsCluster.getURI(0);
- return uri.toString();
- }
- }
-
- /**
- * By default in JDK9+, the ForkJoinWorkerThreadFactory does not give SecurityManager permissions
- * to threads that are created. This works around that with a custom thread factory. See SOLR-9515
- * and HDFS-14251 Used in org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.BlockPoolSlice
- */
- public static class HDFSForkJoinThreadFactory
- implements ForkJoinPool.ForkJoinWorkerThreadFactory {
- @Override
- public ForkJoinWorkerThread newThread(ForkJoinPool pool) {
- ForkJoinWorkerThread worker = new SecurityManagerWorkerThread(pool);
- worker.setName("solr-hdfs-threadpool-" + worker.getPoolIndex());
- return worker;
- }
- }
-
- private static class SecurityManagerWorkerThread extends ForkJoinWorkerThread {
- SecurityManagerWorkerThread(ForkJoinPool pool) {
- super(pool);
- }
- }
-
- /** Ensures that we don't try to initialize metrics and read files outside the source tree. */
- public static class FakeMetricsSystem extends MetricsSystemImpl {
- @Override
- public synchronized MetricsSystem init(String prefix) {
- return this;
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsThreadLeakTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsThreadLeakTest.java
deleted file mode 100644
index c8221060235..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsThreadLeakTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsThreadLeakTest extends SolrTestCaseJ4 {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false, false);
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Test
- public void testBasic() throws IOException {
- String uri = HdfsTestUtil.getURI(dfsCluster);
- Path path = new Path(uri);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- try (FileSystem fs = FileSystem.get(path.toUri(), conf)) {
- Path testFile = new Path(uri + "/testfile");
- try (FSDataOutputStream out = fs.create(testFile)) {
- out.write(5);
- out.hflush();
- }
-
- ((DistributedFileSystem) fs).recoverLease(testFile);
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsTlogReplayBufferedWhileIndexingTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsTlogReplayBufferedWhileIndexingTest.java
deleted file mode 100644
index 8bd94fcd708..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsTlogReplayBufferedWhileIndexingTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractTlogReplayBufferedWhileIndexingTestBase;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsTlogReplayBufferedWhileIndexingTest
- extends AbstractTlogReplayBufferedWhileIndexingTestBase {
- private static MiniDFSCluster dfsCluster;
-
- public HdfsTlogReplayBufferedWhileIndexingTest() throws Exception {
- super();
- }
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- protected String getDataDir(String dataDir) {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsUnloadDistributedZkTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsUnloadDistributedZkTest.java
deleted file mode 100644
index 6cea5b61c0b..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsUnloadDistributedZkTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractUnloadDistributedZkTestBase;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsUnloadDistributedZkTest extends AbstractUnloadDistributedZkTestBase {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- protected String getDataDir(String dataDir) {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsWriteToMultipleCollectionsTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsWriteToMultipleCollectionsTest.java
deleted file mode 100644
index 05626a298f0..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/HdfsWriteToMultipleCollectionsTest.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.NRTCachingDirectory;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.lucene.util.IOUtils;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.cloud.AbstractBasicDistributedZkTestBase;
-import org.apache.solr.cloud.StoppableIndexingThread;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.DirectoryFactory;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.hdfs.HdfsDirectoryFactory;
-import org.apache.solr.hdfs.store.blockcache.BlockCache;
-import org.apache.solr.hdfs.store.blockcache.BlockDirectory;
-import org.apache.solr.hdfs.store.blockcache.BlockDirectoryCache;
-import org.apache.solr.hdfs.store.blockcache.Cache;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.apache.solr.util.RefCounted;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsWriteToMultipleCollectionsTest extends AbstractBasicDistributedZkTestBase {
- private static final String ACOLLECTION = "acollection";
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- schemaString = "schema15.xml"; // we need a string id
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- schemaString = null;
- }
- }
-
- @Override
- protected String getDataDir(String dataDir) throws IOException {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-
- public HdfsWriteToMultipleCollectionsTest() {
- super();
- sliceCount = 1;
- fixShardCount(3);
- }
-
- @Override
- protected String getSolrXml() {
- return "solr.xml";
- }
-
- @Test
- @Override
- public void test() throws Exception {
- int docCount = random().nextInt(1313) + 1;
- int cnt = random().nextInt(4) + 1;
- for (int i = 0; i < cnt; i++) {
- createCollection(ACOLLECTION + i, "conf1", 2, 2);
- }
- for (int i = 0; i < cnt; i++) {
- waitForRecoveriesToFinish(ACOLLECTION + i, false);
- }
- List<CloudSolrClient> cloudClients = new ArrayList<>();
- List<StoppableIndexingThread> threads = new ArrayList<>();
- for (int i = 0; i < cnt; i++) {
- CloudSolrClient client =
- new RandomizingCloudSolrClientBuilder(
- Collections.singletonList(zkServer.getZkAddress()), Optional.empty())
- .withDefaultCollection(ACOLLECTION + i)
- .build();
- cloudClients.add(client);
- StoppableIndexingThread indexThread =
- new StoppableIndexingThread(null, client, "1", true, docCount, 1, true);
- threads.add(indexThread);
- indexThread.start();
- }
-
- int addCnt = 0;
- for (StoppableIndexingThread thread : threads) {
- thread.join();
- addCnt += thread.getNumAdds() - thread.getNumDeletes();
- }
-
- long collectionsCount = 0;
- for (CloudSolrClient client : cloudClients) {
- client.commit();
- collectionsCount += client.query(new SolrQuery("*:*")).getResults().getNumFound();
- }
-
- IOUtils.close(cloudClients);
-
- assertEquals(addCnt, collectionsCount);
-
- BlockCache lastBlockCache = null;
- // assert that we are using the block directory and that write and read caching are being used
- for (JettySolrRunner jetty : jettys) {
- CoreContainer cores = jetty.getCoreContainer();
- Collection<SolrCore> solrCores = cores.getCores();
- for (SolrCore core : solrCores) {
- if (core.getCoreDescriptor()
- .getCloudDescriptor()
- .getCollectionName()
- .startsWith(ACOLLECTION)) {
- DirectoryFactory factory = core.getDirectoryFactory();
- assertTrue(
- "Found: " + core.getDirectoryFactory().getClass().getName(),
- factory instanceof HdfsDirectoryFactory);
- Directory dir = factory.get(core.getDataDir(), null, null);
- try {
- long dataDirSize = factory.size(dir);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- FileSystem fileSystem =
- FileSystem.newInstance(new Path(core.getDataDir()).toUri(), conf);
- long size = fileSystem.getContentSummary(new Path(core.getDataDir())).getLength();
- assertEquals(size, dataDirSize);
- } finally {
- core.getDirectoryFactory().release(dir);
- }
-
- RefCounted<IndexWriter> iwRef =
- core.getUpdateHandler().getSolrCoreState().getIndexWriter(core);
- try {
- IndexWriter iw = iwRef.get();
- NRTCachingDirectory directory = (NRTCachingDirectory) iw.getDirectory();
- BlockDirectory blockDirectory = (BlockDirectory) directory.getDelegate();
- assertTrue(blockDirectory.isBlockCacheReadEnabled());
- // see SOLR-6424
- assertFalse(blockDirectory.isBlockCacheWriteEnabled());
- Cache cache = blockDirectory.getCache();
- // we know it's a BlockDirectoryCache, but future proof
- assertTrue(cache instanceof BlockDirectoryCache);
- BlockCache blockCache = ((BlockDirectoryCache) cache).getBlockCache();
- if (lastBlockCache != null) {
- if (Boolean.getBoolean("solr.hdfs.blockcache.global")) {
- assertEquals(lastBlockCache, blockCache);
- } else {
- assertNotSame(lastBlockCache, blockCache);
- }
- }
- lastBlockCache = blockCache;
- } finally {
- iwRef.decref();
- }
- }
- }
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/MoveReplicaHdfsFailoverTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/MoveReplicaHdfsFailoverTest.java
deleted file mode 100644
index f80f3242130..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/MoveReplicaHdfsFailoverTest.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import java.io.IOException;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ClusterStateUtil;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@Nightly // test is too long for non nightly
-public class MoveReplicaHdfsFailoverTest extends SolrCloudTestCase {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- configureCluster(2)
- .addConfig(
- "conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
- .configure();
-
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
-
- cluster.uploadConfigSet(configset("cloud-hdfs"), "conf1");
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- shutdownCluster();
- } finally {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
- }
-
- @Test
- public void testDataDirAndUlogAreMaintained() throws Exception {
- String coll = "movereplicatest_coll2";
- CollectionAdminRequest.createCollection(coll, "conf1", 1, 1)
- .setCreateNodeSet("")
- .process(cluster.getSolrClient());
- String hdfsUri = HdfsTestUtil.getURI(dfsCluster);
- String dataDir = hdfsUri + "/dummyFolder/dataDir";
- String ulogDir = hdfsUri + "/dummyFolder2/ulogDir";
- CollectionAdminResponse res =
- CollectionAdminRequest.addReplicaToShard(coll, "shard1")
- .setDataDir(dataDir)
- .setUlogDir(ulogDir)
- .setNode(cluster.getJettySolrRunner(0).getNodeName())
- .process(cluster.getSolrClient());
-
- ZkStateReader zkStateReader = cluster.getZkStateReader();
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
-
- DocCollection docCollection = zkStateReader.getClusterState().getCollection(coll);
- Replica replica = docCollection.getReplicas().iterator().next();
- assertTrue(
- replica.getStr("ulogDir"),
- replica.getStr("ulogDir").equals(ulogDir)
- || replica.getStr("ulogDir").equals(ulogDir + '/'));
- assertTrue(
- replica.getStr("dataDir"),
- replica.getStr("dataDir").equals(dataDir)
- || replica.getStr("dataDir").equals(dataDir + '/'));
-
- new CollectionAdminRequest.MoveReplica(
- coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
- .process(cluster.getSolrClient());
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
- docCollection = zkStateReader.getClusterState().getCollection(coll);
- assertEquals(1, docCollection.getSlice("shard1").getReplicas().size());
- Replica newReplica = docCollection.getReplicas().iterator().next();
- assertEquals(newReplica.getNodeName(), cluster.getJettySolrRunner(1).getNodeName());
- assertTrue(
- newReplica.getStr("ulogDir"),
- newReplica.getStr("ulogDir").equals(ulogDir)
- || newReplica.getStr("ulogDir").equals(ulogDir + '/'));
- assertTrue(
- newReplica.getStr("dataDir"),
- newReplica.getStr("dataDir").equals(dataDir)
- || newReplica.getStr("dataDir").equals(dataDir + '/'));
-
- assertEquals(replica.getName(), newReplica.getName());
- assertEquals(replica.getCoreName(), newReplica.getCoreName());
- assertNotEquals(replica.getNodeName(), newReplica.getNodeName());
- final int numDocs = 100;
- addDocs(coll, numDocs); // indexed but not committed
-
- cluster.getJettySolrRunner(1).stop();
- Thread.sleep(5000);
- new CollectionAdminRequest.MoveReplica(
- coll, newReplica.getName(), cluster.getJettySolrRunner(0).getNodeName())
- .process(cluster.getSolrClient());
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
-
- // assert that the old core will be removed on startup
- cluster.getJettySolrRunner(1).start();
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
- docCollection = zkStateReader.getClusterState().getCollection(coll);
- assertEquals(1, docCollection.getReplicas().size());
- newReplica = docCollection.getReplicas().iterator().next();
- assertEquals(newReplica.getNodeName(), cluster.getJettySolrRunner(0).getNodeName());
- assertTrue(
- newReplica.getStr("ulogDir"),
- newReplica.getStr("ulogDir").equals(ulogDir)
- || newReplica.getStr("ulogDir").equals(ulogDir + '/'));
- assertTrue(
- newReplica.getStr("dataDir"),
- newReplica.getStr("dataDir").equals(dataDir)
- || newReplica.getStr("dataDir").equals(dataDir + '/'));
-
- assertEquals(0, cluster.getJettySolrRunner(1).getCoreContainer().getCores().size());
-
- cluster.getSolrClient().commit(coll);
- assertEquals(
- numDocs,
- cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
- CollectionAdminRequest.deleteCollection(coll).process(cluster.getSolrClient());
- }
-
- @Test
- public void testOldReplicaIsDeleted() throws Exception {
- String coll = "movereplicatest_coll3";
- CollectionAdminRequest.createCollection(coll, "conf1", 1, 1)
- .setCreateNodeSet(cluster.getJettySolrRunner(0).getNodeName())
- .process(cluster.getSolrClient());
- addDocs(coll, 2);
- Replica replica = getCollectionState(coll).getReplicas().iterator().next();
-
- cluster.getJettySolrRunners().get(0).stop();
- assertTrue(ClusterStateUtil.waitForAllReplicasNotLive(cluster.getZkStateReader(), 20000));
-
- // move replica from node0 -> node1
- new CollectionAdminRequest.MoveReplica(
- coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
- .process(cluster.getSolrClient());
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getZkStateReader(), 20000));
-
- cluster.getJettySolrRunners().get(1).stop();
- assertTrue(ClusterStateUtil.waitForAllReplicasNotLive(cluster.getZkStateReader(), 20000));
-
- // node0 will delete it replica because of CloudUtil.checkSharedFSFailoverReplaced()
- cluster.getJettySolrRunners().get(0).start();
- Thread.sleep(5000);
- assertTrue(ClusterStateUtil.waitForAllReplicasNotLive(cluster.getZkStateReader(), 20000));
-
- cluster.getJettySolrRunners().get(1).start();
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getZkStateReader(), 20000));
-
- assertEquals(1, getCollectionState(coll).getReplicas().size());
- assertEquals(
- 2, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
- CollectionAdminRequest.deleteCollection(coll).process(cluster.getSolrClient());
- }
-
- @Test
- public void testOldReplicaIsDeletedInRaceCondition() throws Exception {
- String coll = "movereplicatest_coll4";
- CollectionAdminRequest.createCollection(coll, "conf1", 1, 1)
- .setCreateNodeSet(cluster.getJettySolrRunner(0).getNodeName())
- .process(cluster.getSolrClient());
- addDocs(coll, 100);
- Replica replica = getCollectionState(coll).getReplicas().iterator().next();
-
- cluster.getJettySolrRunners().get(0).stop();
- assertTrue(ClusterStateUtil.waitForAllReplicasNotLive(cluster.getZkStateReader(), 20000));
-
- // move replica from node0 -> node1
- new CollectionAdminRequest.MoveReplica(
- coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
- .process(cluster.getSolrClient());
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getZkStateReader(), 20000));
-
- cluster.getJettySolrRunners().get(1).stop();
- assertTrue(ClusterStateUtil.waitForAllReplicasNotLive(cluster.getZkStateReader(), 20000));
-
- cluster.getJettySolrRunners().get(1).start();
- // node0 will delete it replica because of CloudUtil.checkSharedFSFailoverReplaced()
- cluster.getJettySolrRunners().get(0).start();
- Thread.sleep(5000);
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getZkStateReader(), 20000));
-
- assertEquals(1, getCollectionState(coll).getReplicas().size());
- assertEquals(
- 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
- CollectionAdminRequest.deleteCollection(coll).process(cluster.getSolrClient());
- }
-
- private void addDocs(String collection, int numDocs) throws SolrServerException, IOException {
- SolrClient solrClient = cluster.getSolrClient();
- for (int docId = 1; docId <= numDocs; docId++) {
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField("id", docId);
- solrClient.add(collection, doc);
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/MoveReplicaHdfsTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/MoveReplicaHdfsTest.java
deleted file mode 100644
index 3d504762802..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/MoveReplicaHdfsTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.lucene.tests.util.TimeUnits;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.AbstractMoveReplicaTestBase;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@TimeoutSuite(millis = TimeUnits.HOUR)
-@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13924")
-public class MoveReplicaHdfsTest extends AbstractMoveReplicaTestBase {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(LuceneTestCase.createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- protected String getConfigSet() {
- return "cloud-hdfs";
- }
-
- @Test
- public void testNormalMove() throws Exception {
- inPlaceMove = false;
- super.test();
- }
-
- @Test
- // 2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
- // 21-May-2018
- // commented 9-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
- // added 20-Jul-2018
- // commented 23-AUG-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
- // added 17-Aug-2018
- // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
- // added 23-Aug-2018
- // commented 20-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
- // 6-Sep-2018
- // Commented 14-Oct-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
- // added 20-Sep-2018
- @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
- public void testNormalFailedMove() throws Exception {
- inPlaceMove = false;
- super.testFailedMove();
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java
deleted file mode 100644
index 68fd8715181..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java
+++ /dev/null
@@ -1,436 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest.Create;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
-import org.apache.solr.cloud.ChaosMonkey;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ClusterStateUtil;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.ExecutorUtil;
-import org.apache.solr.common.util.SolrNamedThreadFactory;
-import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.apache.solr.util.LogLevel;
-import org.apache.solr.util.TestInjection;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Nightly
-@SuppressSSL
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@LogLevel("org.apache.solr.cloud.*=DEBUG")
- bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
-public class SharedFileSystemAutoReplicaFailoverTest extends AbstractFullDistribZkTestBase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final boolean DEBUG = true;
- private static MiniDFSCluster dfsCluster;
-
- CompletionService<Object> completionService;
- Set<Future<Object>> pending;
- private final Map<String, String> collectionUlogDirMap = new HashMap<>();
-
- @BeforeClass
- public static void hdfsFailoverBeforeClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- System.setProperty(
- "solr.hdfs.blockcache.global",
- "true"); // always use global cache, this test can create a lot of directories
- schemaString = "schema15.xml";
- }
-
- @AfterClass
- public static void hdfsFailoverAfterClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
- dfsCluster = null;
- }
- }
-
- @Before
- @Override
- public void setUp() throws Exception {
- super.setUp();
- collectionUlogDirMap.clear();
- }
-
- @Override
- public void distribSetUp() throws Exception {
- super.distribSetUp();
- useJettyDataDir = false;
- }
-
- @Override
- protected String getSolrXml() {
- return "solr.xml";
- }
-
- public SharedFileSystemAutoReplicaFailoverTest() {
- executor =
- new ExecutorUtil.MDCAwareThreadPoolExecutor(
- 0,
- Integer.MAX_VALUE,
- 5,
- TimeUnit.SECONDS,
- new SynchronousQueue<>(),
- new SolrNamedThreadFactory("testExecutor"));
- completionService = new ExecutorCompletionService<>(executor);
- pending = new HashSet<>();
- }
-
- @Test
- @ShardsFixed(num = 4)
- // 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
- // commented 23-AUG-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
- // added 20-Jul-2018
- public void test() throws Exception {
- try {
- // to keep uncommitted docs during failover
- TestInjection.skipIndexWriterCommitOnClose = true;
- testBasics();
- } finally {
- TestInjection.reset();
- if (DEBUG) {
- super.printLayout();
- }
- }
- }
-
- // very slow tests, especially since jetty is started and stopped
- // serially
- private void testBasics() throws Exception {
- String collection1 = "solrj_collection";
- Create createCollectionRequest =
- CollectionAdminRequest.createCollection(collection1, "conf1", 2, 2)
- .setRouterField("myOwnField");
- CollectionAdminResponse response = createCollectionRequest.process(cloudClient);
-
- assertEquals(0, response.getStatus());
- assertTrue(response.isSuccess());
- waitForRecoveriesToFinish(collection1, false);
-
- String collection2 = "solrj_collection2";
- createCollectionRequest =
- CollectionAdminRequest.createCollection(collection2, "conf1", 2, 2)
- .setRouterField("myOwnField");
- CollectionAdminResponse response2 = createCollectionRequest.process(getCommonCloudSolrClient());
-
- assertEquals(0, response2.getStatus());
- assertTrue(response2.isSuccess());
-
- waitForRecoveriesToFinish(collection2, false);
-
- String collection3 = "solrj_collection3";
- createCollectionRequest =
- CollectionAdminRequest.createCollection(collection3, "conf1", 5, 1)
- .setRouterField("myOwnField");
- CollectionAdminResponse response3 = createCollectionRequest.process(getCommonCloudSolrClient());
-
- assertEquals(0, response3.getStatus());
- assertTrue(response3.isSuccess());
-
- waitForRecoveriesToFinish(collection3, false);
-
- // a collection has only 1 replica per a shard
- String collection4 = "solrj_collection4";
- createCollectionRequest =
- CollectionAdminRequest.createCollection(collection4, "conf1", 5, 1).setRouterField("text");
- CollectionAdminResponse response4 = createCollectionRequest.process(getCommonCloudSolrClient());
-
- assertEquals(0, response4.getStatus());
- assertTrue(response4.isSuccess());
-
- waitForRecoveriesToFinish(collection4, false);
-
- // all collections
- String[] collections = {collection1, collection2, collection3, collection4};
-
- // add some documents to collection4
- final int numDocs = 100;
- addDocs(collection4, numDocs, false); // indexed but not committed
-
- // no result because not committed yet
- queryAndAssertResultSize(collection4, 0, 10000);
-
- assertUlogDir(collections);
-
- jettys.get(1).stop();
- jettys.get(2).stop();
-
- Thread.sleep(5000);
-
- assertSliceAndReplicaCount(collection1, 2, 2, 120000);
-
- assertEquals(
- 4,
- ClusterStateUtil.getLiveAndActiveReplicaCount(
- ZkStateReader.from(cloudClient), collection1));
- assertTrue(
- ClusterStateUtil.getLiveAndActiveReplicaCount(ZkStateReader.from(cloudClient), collection2)
- < 4);
-
- // collection3 has maxShardsPerNode=1, there are 4 standard jetties and one control jetty and 2
- // nodes stopped
- ClusterStateUtil.waitForLiveAndActiveReplicaCount(
- ZkStateReader.from(cloudClient), collection3, 3, 30000);
-
- // collection4 has maxShardsPerNode=5 and setMaxShardsPerNode=5
- ClusterStateUtil.waitForLiveAndActiveReplicaCount(
- ZkStateReader.from(cloudClient), collection4, 5, 30000);
-
- // all docs should be queried after failover
- cloudClient.commit(); // to query all docs
- assertSliceAndReplicaCount(collection4, 5, 1, 120000);
- queryAndAssertResultSize(collection4, numDocs, 10000);
-
- // collection1 should still be at 4
- assertEquals(
- 4,
- ClusterStateUtil.getLiveAndActiveReplicaCount(
- ZkStateReader.from(cloudClient), collection1));
- // and collection2 less than 4
- assertTrue(
- ClusterStateUtil.getLiveAndActiveReplicaCount(ZkStateReader.from(cloudClient), collection2)
- < 4);
-
- assertUlogDir(collections);
-
- boolean allowOverseerRestart = random().nextBoolean();
- List<JettySolrRunner> stoppedJetties =
- allowOverseerRestart
- ? jettys.stream()
- .filter(jettySolrRunner -> random().nextBoolean())
- .collect(Collectors.toList())
- : notOverseerJetties();
- ChaosMonkey.stop(stoppedJetties);
- controlJetty.stop();
-
- assertTrue(
- "Timeout waiting for all not live",
- waitingForReplicasNotLive(ZkStateReader.from(cloudClient), 45000, stoppedJetties));
-
- ChaosMonkey.start(stoppedJetties);
- controlJetty.start();
-
- assertSliceAndReplicaCount(collection1, 2, 2, 120000);
- assertSliceAndReplicaCount(collection3, 5, 1, 120000);
-
- // all docs should be queried
- assertSliceAndReplicaCount(collection4, 5, 1, 120000);
- queryAndAssertResultSize(collection4, numDocs, 10000);
-
- assertUlogDir(collections);
-
- int jettyIndex = random().nextInt(jettys.size());
- jettys.get(jettyIndex).stop();
- jettys.get(jettyIndex).start();
-
- assertSliceAndReplicaCount(collection1, 2, 2, 120000);
-
- assertUlogDir(collections);
-
- assertSliceAndReplicaCount(collection3, 5, 1, 120000);
- assertSliceAndReplicaCount(collection4, 5, 1, 120000);
- }
-
- private void queryAndAssertResultSize(String collection, int expectedResultSize, int timeoutMS)
- throws SolrServerException, IOException, InterruptedException {
- long startTimestamp = System.nanoTime();
-
- long actualResultSize = 0;
- while (true) {
- if (System.nanoTime() - startTimestamp > TimeUnit.MILLISECONDS.toNanos(timeoutMS)
- || actualResultSize > expectedResultSize) {
- fail("expected: " + expectedResultSize + ", actual: " + actualResultSize);
- }
- SolrParams queryAll = new SolrQuery("*:*");
- CloudSolrClient solrClient = this.getSolrClient(collection);
- try {
- QueryResponse queryResponse = solrClient.query(queryAll);
- actualResultSize = queryResponse.getResults().getNumFound();
- if (expectedResultSize == actualResultSize) {
- return;
- }
- } catch (SolrServerException | IOException e) {
- log.warn(
- "Querying solr threw an exception. This can be expected to happen during restarts.", e);
- }
-
- Thread.sleep(1000);
- }
- }
-
- private void addDocs(String collection, int numDocs, boolean commit)
- throws SolrServerException, IOException {
- CloudSolrClient solrClient = this.getSolrClient(collection);
- for (int docId = 1; docId <= numDocs; docId++) {
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField("id", docId);
- doc.addField("text", "shard" + docId % 5);
-
- solrClient.add(doc);
- }
- if (commit) {
- solrClient.commit();
- }
- }
-
- /** After failover, ulogDir should not be changed. */
- private void assertUlogDir(String... collections) throws IOException {
- for (String collection : collections) {
- Collection<Slice> slices =
- cloudClient.getClusterState().getCollection(collection).getSlices();
- for (Slice slice : slices) {
- for (Replica replica : slice.getReplicas()) {
- Map<String, Object> properties = replica.getProperties();
- String coreName = replica.getCoreName();
- String curUlogDir = (String) properties.get(CoreDescriptor.CORE_ULOGDIR);
- String prevUlogDir = collectionUlogDirMap.get(coreName);
- if (curUlogDir != null) {
- if (prevUlogDir == null) {
- collectionUlogDirMap.put(coreName, curUlogDir);
- } else {
- assertEquals(prevUlogDir, curUlogDir);
- }
- }
- }
- }
- }
- }
-
- // Overseer failover is not currently guaranteed with MoveReplica or Policy Framework
- private List<JettySolrRunner> notOverseerJetties() throws IOException, SolrServerException {
- CollectionAdminResponse response =
- CollectionAdminRequest.getOverseerStatus().process(cloudClient);
- String overseerNode = (String) response.getResponse().get("leader");
- return jettys.stream()
- .filter(
- jetty ->
- !(jetty.getCoreContainer() != null && overseerNode.equals(jetty.getNodeName())))
- .collect(Collectors.toList());
- }
-
- private boolean waitingForReplicasNotLive(
- ZkStateReader zkStateReader, int timeoutInMs, List<JettySolrRunner> jetties) {
- Set<String> nodeNames =
- jetties.stream()
- .filter(jetty -> jetty.getCoreContainer() != null)
- .map(JettySolrRunner::getNodeName)
- .collect(Collectors.toSet());
- return ClusterStateUtil.waitFor(
- zkStateReader,
- null,
- timeoutInMs,
- TimeUnit.MILLISECONDS,
- (liveNodes, state) ->
- ClusterStateUtil.replicasOfActiveSlicesStream(state)
- .noneMatch(
- replica ->
- nodeNames.contains(replica.getNodeName())
- && liveNodes.contains(replica.getNodeName())));
- }
-
- private void assertSliceAndReplicaCount(
- String collection, int numSlices, int numReplicas, int timeOutInMs)
- throws InterruptedException {
-
- try {
- ZkStateReader.from(cloudClient)
- .waitForState(
- collection,
- timeOutInMs,
- TimeUnit.MILLISECONDS,
- (liveNodes, c) -> {
- Collection<Slice> slices = c.getActiveSlices();
- if (slices.size() == numSlices) {
- for (Slice slice : slices) {
- int count = 0;
- for (Replica replica : slice.getReplicas()) {
- if (replica.getState() == Replica.State.ACTIVE
- && liveNodes.contains(replica.getNodeName())) {
- count++;
- }
- }
- if (count < numReplicas) {
- return false;
- }
- }
- return true;
- }
- return false;
- });
- } catch (TimeoutException e) {
- fail(
- "Expected numSlices="
- + numSlices
- + " numReplicas="
- + numReplicas
- + " but found "
- + cloudClient.getClusterState().getCollection(collection)
- + " with /live_nodes: "
- + cloudClient.getClusterState().getLiveNodes());
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.java
deleted file mode 100644
index 326daaddc03..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.cloud.AbstractBasicDistributedZkTestBase;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CollectionParams.CollectionAction;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {
- private static final String DELETE_DATA_DIR_COLLECTION = "delete_data_dir";
- private static MiniDFSCluster dfsCluster;
-
- private boolean testRestartIntoSafeMode;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- protected String getDataDir(String dataDir) throws IOException {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-
- public StressHdfsTest() {
- super();
- sliceCount = 1;
- fixShardCount(TEST_NIGHTLY ? 7 : random().nextInt(2) + 1);
- testRestartIntoSafeMode = random().nextBoolean();
- }
-
- @Override
- protected String getSolrXml() {
- return "solr.xml";
- }
-
- @Test
- @Override
- public void test() throws Exception {
- randomlyEnableAutoSoftCommit();
-
- int cnt = random().nextInt(2) + 1;
- for (int i = 0; i < cnt; i++) {
- createAndDeleteCollection();
- }
-
- if (testRestartIntoSafeMode) {
- Timer timer = new Timer();
-
- try {
- createCollection(DELETE_DATA_DIR_COLLECTION, "conf1", 1, 1);
-
- waitForRecoveriesToFinish(DELETE_DATA_DIR_COLLECTION, false);
-
- jettys.get(0).stop();
-
- // enter safe mode and restart a node
- NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
-
- int rnd = random().nextInt(10000);
-
- timer.schedule(
- new TimerTask() {
-
- @Override
- public void run() {
- NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
- }
- },
- rnd);
-
- jettys.get(0).start();
-
- waitForRecoveriesToFinish(DELETE_DATA_DIR_COLLECTION, false);
- } finally {
- timer.cancel();
- }
- }
- }
-
- private void createAndDeleteCollection() throws Exception {
- boolean overshard = random().nextBoolean();
- int rep;
- int nShards;
- if (overshard) {
- nShards = getShardCount() * 2;
- rep = 1;
- } else {
- nShards = getShardCount() / 2;
- rep = 2;
- if (nShards == 0) nShards = 1;
- }
-
- createCollection(DELETE_DATA_DIR_COLLECTION, "conf1", nShards, rep);
-
- waitForRecoveriesToFinish(DELETE_DATA_DIR_COLLECTION, false);
-
- // data dirs should be in zk, SOLR-8913
- ClusterState clusterState = cloudClient.getClusterState();
- final DocCollection docCollection =
- clusterState.getCollectionOrNull(DELETE_DATA_DIR_COLLECTION);
- assertNotNull("Could not find :" + DELETE_DATA_DIR_COLLECTION, docCollection);
- Slice slice = docCollection.getSlice("shard1");
- assertNotNull(docCollection.getSlices().toString(), slice);
- Collection<Replica> replicas = slice.getReplicas();
- for (Replica replica : replicas) {
- assertNotNull(replica.getProperties().toString(), replica.get("dataDir"));
- assertNotNull(replica.getProperties().toString(), replica.get("ulogDir"));
- }
-
- CloudSolrClient solrClient = this.getSolrClient(DELETE_DATA_DIR_COLLECTION);
-
- ZkStateReader.from(solrClient).forceUpdateCollection(DELETE_DATA_DIR_COLLECTION);
- for (int i = 1; i < nShards + 1; i++) {
- ZkStateReader.from(solrClient).getLeaderRetry(DELETE_DATA_DIR_COLLECTION, "shard" + i, 30000);
- }
-
- // collect the data dirs
- List<String> dataDirs = new ArrayList<>();
-
- int i = 0;
- for (SolrClient client : clients) {
- try (SolrClient c =
- new HttpSolrClient.Builder(getBaseUrl(client))
- .withDefaultCollection(DELETE_DATA_DIR_COLLECTION)
- .withConnectionTimeout(30000, TimeUnit.MILLISECONDS)
- .build()) {
- int docCnt = random().nextInt(1000) + 1;
- for (int j = 0; j < docCnt; j++) {
- c.add(getDoc("id", i++, "txt_t", "just some random text for a doc"));
- }
-
- if (random().nextBoolean()) {
- c.commit();
- } else {
- c.commit(true, true, true);
- }
-
- NamedList<Object> response =
- c.query(new SolrQuery().setRequestHandler("/admin/system")).getResponse();
- @SuppressWarnings({"unchecked"})
- NamedList<Object> coreInfo = (NamedList<Object>) response.get("core");
- @SuppressWarnings({"unchecked"})
- String dataDir = (String) ((NamedList<Object>) coreInfo.get("directory")).get("data");
- dataDirs.add(dataDir);
- }
- }
-
- if (random().nextBoolean()) {
- solrClient.deleteByQuery("*:*");
- solrClient.commit();
-
- assertEquals(0, solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());
- }
-
- solrClient.commit();
- solrClient.query(new SolrQuery("*:*"));
-
- // delete collection
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", CollectionAction.DELETE.toString());
- params.set("name", DELETE_DATA_DIR_COLLECTION);
- QueryRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
- solrClient.request(request);
-
- waitForCollectionToDisappear(
- DELETE_DATA_DIR_COLLECTION, ZkStateReader.from(cloudClient), true, 10);
-
- // check that all dirs are gone
- for (String dataDir : dataDirs) {
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- try (FileSystem fs = FileSystem.get(new URI(HdfsTestUtil.getURI(dfsCluster)), conf)) {
- assertFalse(
- "Data directory exists after collection removal : " + dataDir,
- fs.exists(new Path(dataDir)));
- }
- }
- }
-
- protected String getBaseUrl(SolrClient client) {
- return ((HttpSolrClient) client).getBaseURL();
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/api/collections/HdfsCloudIncrementalBackupTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/api/collections/HdfsCloudIncrementalBackupTest.java
deleted file mode 100644
index 508e3c42ee4..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/api/collections/HdfsCloudIncrementalBackupTest.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.hdfs.cloud.api.collections;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest;
-import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
- "SimpleText"
-}) // Backups do checksum validation against a footer value not present in 'SimpleText'
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsCloudIncrementalBackupTest extends AbstractIncrementalBackupTest {
- public static final String SOLR_XML =
- "<solr>\n"
- + "\n"
- + " <str name=\"shareSchema\">${shareSchema:false}</str>\n"
- + " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n"
- + " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n"
- + "\n"
- + " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n"
- + " <str name=\"urlScheme\">${urlScheme:}</str>\n"
- + " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n"
- + " <int name=\"connTimeout\">${connTimeout:15000}</int>\n"
- + " </shardHandlerFactory>\n"
- + "\n"
- + " <solrcloud>\n"
- + " <str name=\"host\">127.0.0.1</str>\n"
- + " <int name=\"hostPort\">${hostPort:8983}</int>\n"
- + " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n"
- + " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n"
- + " <int name=\"leaderVoteWait\">10000</int>\n"
- + " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n"
- + " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n"
- + " </solrcloud>\n"
- + " \n"
- + " <backup>\n"
- + " <repository name=\"trackingBackupRepository\" default=\"true\" class=\"org.apache.solr.core.TrackingBackupRepository\"> \n"
- + " <str name=\"delegateRepoName\">hdfs</str>\n"
- + " </repository>\n"
- + " <repository name=\"hdfs\" class=\"org.apache.solr.hdfs.backup.repository.HdfsBackupRepository\"> \n"
- + " <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n"
- + " <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n"
- + " <str name=\"solr.hdfs.confdir\">${solr.hdfs.confdir:}</str>\n"
- + " </repository>\n"
- + " </backup>\n"
- + " \n"
- + "</solr>\n";
-
- private static MiniDFSCluster dfsCluster;
- private static String hdfsUri;
- private static FileSystem fs;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- hdfsUri = HdfsTestUtil.getURI(dfsCluster);
- try {
- URI uri = new URI(hdfsUri);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- fs = FileSystem.get(uri, conf);
-
- if (fs instanceof DistributedFileSystem) {
- // Make sure dfs is not in safe mode
- while (((DistributedFileSystem) fs)
- .setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET, true)) {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- Thread.interrupted();
- // continue
- }
- }
- }
-
- fs.mkdirs(new org.apache.hadoop.fs.Path("/backup"));
- } catch (IOException | URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- System.setProperty("solr.hdfs.default.backup.path", "/backup");
- System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
- useFactory("solr.StandardDirectoryFactory");
-
- configureCluster(NUM_NODES) // nodes
- .addConfig(
- "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
- .withSolrXml(SOLR_XML)
- .configure();
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- IOUtils.closeQuietly(fs);
- fs = null;
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
- }
- }
-
- @Override
- public String getCollectionNamePrefix() {
- return "hdfsbackuprestore";
- }
-
- @Override
- public String getBackupLocation() {
- return null;
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/api/collections/HdfsCollectionsApiDistributedZkTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/api/collections/HdfsCollectionsApiDistributedZkTest.java
deleted file mode 100644
index 2015e4934e9..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/api/collections/HdfsCollectionsApiDistributedZkTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud.api.collections;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.lucene.tests.util.TimeUnits;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.cloud.api.collections.AbstractCollectionsAPIDistributedZkTestBase;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@Nightly
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@TimeoutSuite(millis = TimeUnits.HOUR)
-public class HdfsCollectionsApiDistributedZkTest
- extends AbstractCollectionsAPIDistributedZkTestBase {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
-
- configureCluster(4)
- .addConfig("conf", configset("cloud-hdfs"))
- .addConfig("conf2", configset("cloud-hdfs"))
- .withSolrXml(TEST_PATH().resolve("solr.xml"))
- .configure();
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/api/collections/TestHdfsCloudBackupRestore.java
deleted file mode 100644
index 9cb6c9ec25a..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/api/collections/TestHdfsCloudBackupRestore.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.cloud.api.collections;
-
-import static org.apache.solr.core.backup.BackupManager.CONFIG_STATE_DIR;
-import static org.apache.solr.core.backup.BackupManager.TRADITIONAL_BACKUP_PROPS_FILE;
-import static org.apache.solr.core.backup.BackupManager.ZK_STATE_DIR;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.ZkConfigSetService;
-import org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.params.CollectionAdminParams;
-import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.backup.BackupManager;
-import org.apache.solr.core.backup.BackupProperties;
-import org.apache.solr.hdfs.backup.repository.HdfsBackupRepository;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** This class implements the tests for HDFS integration for Solr backup/restore capability. */
- "SimpleText"
-}) // Backups do checksum validation against a footer value not present in 'SimpleText'
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCase {
- public static final String SOLR_XML =
- "<solr>\n"
- + "\n"
- + " <str name=\"shareSchema\">${shareSchema:false}</str>\n"
- + " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n"
- + " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n"
- + "\n"
- + " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n"
- + " <str name=\"urlScheme\">${urlScheme:}</str>\n"
- + " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n"
- + " <int name=\"connTimeout\">${connTimeout:15000}</int>\n"
- + " </shardHandlerFactory>\n"
- + "\n"
- + " <solrcloud>\n"
- + " <str name=\"host\">127.0.0.1</str>\n"
- + " <int name=\"hostPort\">${hostPort:8983}</int>\n"
- + " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n"
- + " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n"
- + " <int name=\"leaderVoteWait\">10000</int>\n"
- + " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n"
- + " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n"
- + " </solrcloud>\n"
- + " \n"
- + " <backup>\n"
- + " <repository name=\"hdfs\" class=\"org.apache.solr.hdfs.backup.repository.HdfsBackupRepository\"> \n"
- + " <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n"
- + " <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n"
- + " <str name=\"solr.hdfs.confdir\">${solr.hdfs.confdir:}</str>\n"
- + " </repository>\n"
- + " <repository name=\"poisioned\" default=\"true\" "
- + "class=\"org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository\"> \n"
- + " </repository>\n"
- + " </backup>\n"
- + " \n"
- + "</solr>\n";
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static MiniDFSCluster dfsCluster;
- private static String hdfsUri;
- private static FileSystem fs;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- hdfsUri = HdfsTestUtil.getURI(dfsCluster);
- try {
- URI uri = new URI(hdfsUri);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- fs = FileSystem.get(uri, conf);
-
- if (fs instanceof DistributedFileSystem) {
- // Make sure dfs is not in safe mode
- while (((DistributedFileSystem) fs).setSafeMode(SafeModeAction.SAFEMODE_GET, true)) {
- log.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again.");
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- Thread.interrupted();
- // continue
- }
- }
- }
-
- fs.mkdirs(new org.apache.hadoop.fs.Path("/backup"));
- } catch (IOException | URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- System.setProperty("solr.hdfs.default.backup.path", "/backup");
- System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
- useFactory("solr.StandardDirectoryFactory");
-
- configureCluster(NUM_SHARDS) // nodes
- .addConfig(
- "conf1", TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf"))
- .addConfig(
- "confFaulty",
- TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf"))
- .withSolrXml(SOLR_XML)
- .configure();
- cluster
- .getZkClient()
- .delete(
- ZkConfigSetService.CONFIGS_ZKNODE
- + Path.SEPARATOR
- + "confFaulty"
- + Path.SEPARATOR
- + "solrconfig.xml",
- -1,
- true);
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- IOUtils.closeQuietly(fs);
- fs = null;
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
- }
- }
-
- @Override
- public String getCollectionNamePrefix() {
- return "hdfsbackuprestore";
- }
-
- @Override
- public String getBackupRepoName() {
- return "hdfs";
- }
-
- @Override
- public String getBackupLocation() {
- return null;
- }
-
- @Override
- protected void testConfigBackupOnly(String configName, String collectionName) throws Exception {
- String backupName = "configonlybackup";
- CloudSolrClient solrClient = cluster.getSolrClient();
-
- CollectionAdminRequest.Backup backup =
- CollectionAdminRequest.backupCollection(collectionName, backupName)
- .setRepositoryName(getBackupRepoName())
- .setIncremental(false)
- .setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY);
- backup.process(solrClient);
-
- Map<String, String> params = new HashMap<>();
- params.put("location", "/backup");
- params.put("solr.hdfs.home", hdfsUri + "/solr");
-
- HdfsBackupRepository repo = new HdfsBackupRepository();
- repo.init(new NamedList<>(params));
-
- URI baseLoc = repo.createDirectoryURI("/backup");
-
- BackupManager mgr =
- BackupManager.forRestore(
- repo, cluster.getZkStateReader(), repo.resolve(baseLoc, backupName));
- BackupProperties props = mgr.readBackupProperties();
- assertNotNull(props);
- assertEquals(collectionName, props.getCollection());
- assertEquals(backupName, props.getBackupName());
- assertEquals(configName, props.getConfigName());
-
- DocCollection collectionState = mgr.readCollectionState(collectionName);
- assertNotNull(collectionState);
- assertEquals(collectionName, collectionState.getName());
-
- URI configDirLoc =
- repo.resolve(baseLoc, backupName, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
- assertTrue(repo.exists(configDirLoc));
-
- Collection<String> expected = Arrays.asList(TRADITIONAL_BACKUP_PROPS_FILE, ZK_STATE_DIR);
- URI backupLoc = repo.resolve(baseLoc, backupName);
- String[] dirs = repo.listAll(backupLoc);
- for (String d : dirs) {
- assertTrue(expected.contains(d));
- }
- }
-
- @Override
- @Test
- public void test() throws Exception {
- super.test();
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/handler/TestHdfsBackupRestoreCore.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/handler/TestHdfsBackupRestoreCore.java
deleted file mode 100644
index 29674f74886..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/handler/TestHdfsBackupRestoreCore.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.handler;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.params.CoreAdminParams;
-import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
-import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.core.backup.BackupId;
-import org.apache.solr.core.backup.ShardBackupId;
-import org.apache.solr.handler.BackupRestoreUtils;
-import org.apache.solr.handler.BackupStatusChecker;
-import org.apache.solr.handler.ReplicationHandler;
-import org.apache.solr.handler.TestRestoreCoreUtil;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
[email protected] // Currently unknown why SSL does not work with this test
- "SimpleText"
-}) // Backups do checksum validation against a footer value not present in 'SimpleText'
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
- public static final String HDFS_REPO_SOLR_XML =
- "<solr>\n"
- + "\n"
- + " <str name=\"shareSchema\">${shareSchema:false}</str>\n"
- + " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n"
- + " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n"
- + "\n"
- + " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n"
- + " <str name=\"urlScheme\">${urlScheme:}</str>\n"
- + " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n"
- + " <int name=\"connTimeout\">${connTimeout:15000}</int>\n"
- + " </shardHandlerFactory>\n"
- + "\n"
- + " <solrcloud>\n"
- + " <str name=\"host\">127.0.0.1</str>\n"
- + " <int name=\"hostPort\">${hostPort:8983}</int>\n"
- + " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n"
- + " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n"
- + " <int name=\"leaderVoteWait\">10000</int>\n"
- + " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n"
- + " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n"
- + " </solrcloud>\n"
- + " \n"
- + " <backup>\n"
- + " <repository name=\"hdfs\" class=\"org.apache.solr.hdfs.backup.repository.HdfsBackupRepository\"> \n"
- + " <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n"
- + " <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n"
- + " <str name=\"solr.hdfs.confdir\">${solr.hdfs.confdir:}</str>\n"
- + " <str name=\"solr.hdfs.permissions.umask-mode\">${solr.hdfs.permissions.umask-mode:000}</str>\n"
- + " </repository>\n"
- + " </backup>\n"
- + " \n"
- + "</solr>\n";
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static MiniDFSCluster dfsCluster;
- private static String hdfsUri;
- private static FileSystem fs;
- private static long docsSeed; // see indexDocs()
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- hdfsUri = HdfsTestUtil.getURI(dfsCluster);
- try {
- URI uri = new URI(hdfsUri);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- fs = FileSystem.get(uri, conf);
-
- if (fs instanceof DistributedFileSystem) {
- // Make sure dfs is not in safe mode
- while (((DistributedFileSystem) fs).setSafeMode(SafeModeAction.SAFEMODE_GET, true)) {
- log.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again.");
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- Thread.interrupted();
- // continue
- }
- }
- }
-
- fs.mkdirs(new org.apache.hadoop.fs.Path("/backup"));
- } catch (IOException | URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- System.setProperty("solr.hdfs.default.backup.path", "/backup");
- System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
- useFactory("solr.StandardDirectoryFactory");
-
- configureCluster(1) // nodes
- .addConfig(
- "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
- .withSolrXml(HDFS_REPO_SOLR_XML)
- .configure();
-
- docsSeed = random().nextLong();
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- IOUtils.closeQuietly(fs);
- fs = null;
- try {
- SolrTestCaseJ4.resetFactory();
- } finally {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
- }
- }
- }
-
- @Test
- public void test() throws Exception {
- CloudSolrClient solrClient = cluster.getSolrClient();
- String collectionName = "HdfsBackupRestore";
- CollectionAdminRequest.Create create =
- CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1);
- create.process(solrClient);
-
- int nDocs = BackupRestoreUtils.indexDocs(solrClient, collectionName, docsSeed);
-
- DocCollection collectionState = solrClient.getClusterState().getCollection(collectionName);
- assertEquals(1, collectionState.getActiveSlices().size());
- Slice shard = collectionState.getActiveSlices().iterator().next();
- assertEquals(1, shard.getReplicas().size());
- Replica replica = shard.getReplicas().iterator().next();
-
- String replicaBaseUrl = replica.getBaseUrl();
- String coreName = replica.getCoreName();
- String backupName = TestUtil.randomSimpleString(random(), 1, 5);
-
- boolean testViaReplicationHandler = random().nextBoolean();
- String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString();
- final String shardBackupId = new ShardBackupId("standalone", BackupId.zero()).getIdAsString();
-
- try (SolrClient leaderClient = getHttpSolrClient(replicaBaseUrl)) {
- // Create a backup.
- if (testViaReplicationHandler) {
- log.info("Running Backup via replication handler");
- BackupRestoreUtils.runReplicationHandlerCommand(
- baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName);
- final BackupStatusChecker backupStatus =
- new BackupStatusChecker(leaderClient, "/" + coreName + "/replication");
- backupStatus.waitForBackupSuccess(backupName, 30);
- } else {
- log.info("Running Backup via core admin api");
- Map<String, String> params = new HashMap<>();
- params.put("name", backupName);
- params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs");
- params.put(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId);
- BackupRestoreUtils.runCoreAdminCommand(
- replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params);
- }
-
- int numRestoreTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1;
- for (int attempts = 0; attempts < numRestoreTests; attempts++) {
- // Modify existing index before we call restore.
- if (nDocs > 0) {
- // Delete a few docs
- int numDeletes = TestUtil.nextInt(random(), 1, nDocs);
- for (int i = 0; i < numDeletes; i++) {
- leaderClient.deleteByQuery(collectionName, "id:" + i);
- }
- leaderClient.commit(collectionName);
-
- // Add a few more
- int moreAdds = TestUtil.nextInt(random(), 1, 100);
- for (int i = 0; i < moreAdds; i++) {
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField("id", i + nDocs);
- doc.addField("name", "name = " + (i + nDocs));
- leaderClient.add(collectionName, doc);
- }
- // Purposely not calling commit once in a while. There can be some docs which are not
- // committed
- if (usually()) {
- leaderClient.commit(collectionName);
- }
- }
- // Snapshooter prefixes "snapshot." to the backup name.
- if (testViaReplicationHandler) {
- log.info("Running Restore via replication handler");
- // Snapshooter prefixes "snapshot." to the backup name.
- BackupRestoreUtils.runReplicationHandlerCommand(
- baseUrl, coreName, ReplicationHandler.CMD_RESTORE, "hdfs", backupName);
- while (!TestRestoreCoreUtil.fetchRestoreStatus(baseUrl, coreName)) {
- Thread.sleep(1000);
- }
- } else {
- log.info("Running Restore via core admin api");
- Map<String, String> params = new HashMap<>();
- params.put("name", "snapshot." + backupName);
- params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs");
- params.put(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId);
- BackupRestoreUtils.runCoreAdminCommand(
- replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params);
- }
- // See if restore was successful by checking if all the docs are present again
- BackupRestoreUtils.verifyDocs(nDocs, leaderClient, coreName);
-
- // Verify the permissions on the backup folder.
- final String backupPath =
- (testViaReplicationHandler)
- ? "/backup/snapshot." + backupName
- : "/backup/shard_backup_metadata";
- final FsAction expectedPerms =
- (testViaReplicationHandler) ? FsAction.ALL : FsAction.READ_EXECUTE;
-
- FileStatus status = fs.getFileStatus(new org.apache.hadoop.fs.Path(backupPath));
- FsPermission perm = status.getPermission();
- assertEquals(FsAction.ALL, perm.getUserAction());
- assertEquals(expectedPerms, perm.getGroupAction());
- assertEquals(expectedPerms, perm.getOtherAction());
- }
- }
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/index/CheckHdfsIndexTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/index/CheckHdfsIndexTest.java
deleted file mode 100644
index 5294fd35fab..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/index/CheckHdfsIndexTest.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.index;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.index.BaseTestCheckIndex;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.store.HdfsDirectory;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
- private static MiniDFSCluster dfsCluster;
- private static Path path;
-
- private BaseTestCheckIndex testCheckIndex;
- private Directory directory;
-
- public CheckHdfsIndexTest() {
- super();
- sliceCount = 1;
- fixShardCount(1);
-
- testCheckIndex = new BaseTestCheckIndex();
- }
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- path = new Path(HdfsTestUtil.getURI(dfsCluster) + "/solr/");
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- path = null;
- }
- }
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
-
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- directory = new HdfsDirectory(path, conf);
- }
-
- @Override
- @After
- public void tearDown() throws Exception {
- try {
- if (null != directory) {
- directory.close();
- }
- } finally {
- try (FileSystem fs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) {
- fs.delete(path, true);
- } finally {
- super.tearDown();
- }
- }
- }
-
- @Override
- protected String getDataDir(String dataDir) throws IOException {
- return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
- }
-
- @Test
- public void doTest() throws Exception {
- waitForRecoveriesToFinish(false);
-
- indexr(id, 1);
- commit();
-
- waitForRecoveriesToFinish(false);
-
- String[] args;
- {
- SolrClient client = clients.get(0);
- NamedList<Object> response =
- client.query(new SolrQuery().setRequestHandler("/admin/system")).getResponse();
- @SuppressWarnings({"unchecked"})
- NamedList<Object> coreInfo = (NamedList<Object>) response.get("core");
- @SuppressWarnings({"unchecked"})
- String indexDir = ((NamedList<Object>) coreInfo.get("directory")).get("data") + "/index";
-
- args = new String[] {indexDir};
- }
-
- assertEquals("CheckHdfsIndex return status", 0, CheckHdfsIndex.doMain(args));
- }
-
- @Test
- public void testDeletedDocs() throws IOException {
- testCheckIndex.testDeletedDocs(directory);
- }
-
- @Test
- public void testChecksumsOnly() throws IOException {
- testCheckIndex.testChecksumsOnly(directory);
- }
-
- @Test
- public void testChecksumsOnlyVerbose() throws IOException {
- testCheckIndex.testChecksumsOnlyVerbose(directory);
- }
-
- @Test
- @Ignore("We explicitly use a NoLockFactory, so this test doesn't make sense.")
- public void testObtainsLock() throws IOException {
- testCheckIndex.testObtainsLock(directory);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/search/TestRecoveryHdfs.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/search/TestRecoveryHdfs.java
deleted file mode 100644
index 3a382b13fed..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/search/TestRecoveryHdfs.java
+++ /dev/null
@@ -1,1156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.search;
-
-import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayDeque;
-import java.util.Arrays;
-import java.util.Deque;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.Future;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.update.HdfsUpdateLog;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.update.UpdateHandler;
-import org.apache.solr.update.UpdateLog;
-import org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase;
-import org.apache.solr.util.TestInjection;
-import org.apache.solr.util.TimeOut;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-// TODO: longer term this should be combined with TestRecovery somehow ??
-public class TestRecoveryHdfs extends SolrTestCaseJ4 {
- // means that we've seen the leader and have version info (i.e. we are a non-leader replica)
- private static final String FROM_LEADER = DistribPhase.FROMLEADER.toString();
-
- // acquire timeout in seconds. change this to a huge number when debugging to prevent threads
- // from advancing.
- private static final int TIMEOUT = 60;
-
- private static MiniDFSCluster dfsCluster;
- private static String hdfsUri;
- private static FileSystem fs;
-
- @After
- public void afterTest() {
- TestInjection.reset(); // do after every test, don't wait for AfterClass
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- hdfsUri = HdfsTestUtil.getURI(dfsCluster);
-
- try {
- URI uri = new URI(hdfsUri);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- fs = FileSystem.get(uri, conf);
- } catch (IOException | URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- System.setProperty("solr.ulog.dir", hdfsUri + "/solr/shard1");
-
- initCore("solrconfig-tlog.xml", "schema15.xml");
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- IOUtils.closeQuietly(fs);
- fs = null;
- try {
- // Make sure to clean up test resources before shutting down HDFS
- SolrTestCaseJ4.teardownTestCases();
- } finally {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- hdfsUri = null;
- System.clearProperty("solr.ulog.dir");
- }
- }
- }
-
- @Test
- public void testReplicationFactor() throws Exception {
- clearIndex();
-
- HdfsUpdateLog ulog = (HdfsUpdateLog) h.getCore().getUpdateHandler().getUpdateLog();
-
- assertU(commit());
- addAndGetVersion(sdoc("id", "REP1"), null);
- assertU(commit());
-
- String[] logList = ulog.getLogList(new Path(ulog.getTlogDir()));
- boolean foundRep2 = false;
- for (String tl : logList) {
- FileStatus status = fs.getFileStatus(new Path(ulog.getTlogDir(), tl));
- if (status.getReplication() == 2) {
- foundRep2 = true;
- break;
- }
- }
-
- assertTrue("Expected to find tlogs with a replication factor of 2", foundRep2);
- }
-
- @Test
- public void testLogReplay() throws Exception {
- try {
- TestInjection.skipIndexWriterCommitOnClose = true;
- final Semaphore logReplay = new Semaphore(0);
- final Semaphore logReplayFinish = new Semaphore(0);
-
- UpdateLog.testing_logReplayHook =
- () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
-
- clearIndex();
- assertU(commit());
-
- Deque<Long> versions = new ArrayDeque<>();
- versions.addFirst(addAndGetVersion(sdoc("id", "A1"), null));
- versions.addFirst(addAndGetVersion(sdoc("id", "A11"), null));
- versions.addFirst(addAndGetVersion(sdoc("id", "A12"), null));
- versions.addFirst(deleteByQueryAndGetVersion("id:A11", null));
- versions.addFirst(addAndGetVersion(sdoc("id", "A13"), null));
-
- assertJQ(req("q", "*:*"), "/response/numFound==0");
-
- assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);
-
- h.close();
- createCore();
- // Solr should kick this off now
- // h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();
-
- // verify that previous close didn't do a commit
- // recovery should be blocked by our hook
- assertJQ(req("q", "*:*"), "/response/numFound==0");
-
- // make sure we can still access versions after a restart
- assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);
-
- // unblock recovery
- logReplay.release(1000);
-
- // make sure we can still access versions during recovery
- assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);
-
- // wait until recovery has finished
- assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
-
- assertJQ(req("q", "*:*"), "/response/numFound==3");
-
- // make sure we can still access versions after recovery
- assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);
-
- assertU(adoc("id", "A2"));
- assertU(adoc("id", "A3"));
- assertU(delI("A2"));
- assertU(adoc("id", "A4"));
-
- assertJQ(req("q", "*:*"), "/response/numFound==3");
-
- h.close();
- createCore();
- // Solr should kick this off now
- // h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();
-
- // wait until recovery has finished
- assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- assertJQ(req("q", "*:*"), "/response/numFound==5");
- assertJQ(req("q", "id:A2"), "/response/numFound==0");
-
- // no updates, so insure that recovery does not run
- h.close();
- int permits = logReplay.availablePermits();
- createCore();
- // Solr should kick this off now
- // h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();
-
- assertJQ(req("q", "*:*"), "/response/numFound==5");
- Thread.sleep(100);
- assertEquals(
- permits, logReplay.availablePermits()); // no updates, so insure that recovery didn't run
-
- assertEquals(
- UpdateLog.State.ACTIVE, h.getCore().getUpdateHandler().getUpdateLog().getState());
-
- } finally {
- UpdateLog.testing_logReplayHook = null;
- UpdateLog.testing_logReplayFinishHook = null;
- }
- }
-
- @Test
- public void testBuffering() throws Exception {
- TestInjection.skipIndexWriterCommitOnClose = true;
- final Semaphore logReplay = new Semaphore(0);
- final Semaphore logReplayFinish = new Semaphore(0);
-
- UpdateLog.testing_logReplayHook =
- () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
-
- SolrQueryRequest req = req();
- UpdateHandler uhandler = req.getCore().getUpdateHandler();
- UpdateLog ulog = uhandler.getUpdateLog();
-
- try {
- clearIndex();
- assertU(commit());
-
- assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
- ulog.bufferUpdates();
- assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
- Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
- assertNull(rinfoFuture);
- assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
-
- ulog.bufferUpdates();
- assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
-
- // simulate updates from a leader
- updateJ(
- jsonAdd(sdoc("id", "B1", "_version_", "1010")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "B11", "_version_", "1015")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonDelQ("id:B1 id:B11 id:B2 id:B3"),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-1017"));
- updateJ(
- jsonAdd(sdoc("id", "B2", "_version_", "1020")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "B3", "_version_", "1030")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- deleteAndGetVersion("B1", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2010"));
-
- assertJQ(
- req("qt", "/get", "getVersions", "6"),
- "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
-
- assertU(commit());
-
- assertJQ(
- req("qt", "/get", "getVersions", "6"),
- "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
-
- // updates should be buffered, so we should not see any results yet.
- assertJQ(req("q", "*:*"), "/response/numFound==0");
-
- // real-time get should also not show anything (this could change in the future,
- // but it's currently used for validating version numbers too, so it would
- // be bad for updates to be visible if we're just buffering.
- assertJQ(req("qt", "/get", "id", "B3"), "=={'doc':null}");
-
- rinfoFuture = ulog.applyBufferedUpdates();
- assertNotNull(rinfoFuture);
-
- assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
-
- logReplay.release(1000);
-
- UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
- assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
-
- assertJQ(
- req("qt", "/get", "getVersions", "6"),
- "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
-
- assertJQ(req("q", "*:*"), "/response/numFound==2");
-
- // move back to recovering
- ulog.bufferUpdates();
- assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
-
- Long ver = getVer(req("qt", "/get", "id", "B3"));
- assertEquals(1030L, ver.longValue());
-
- // add a reordered doc that shouldn't overwrite one in the index
- updateJ(
- jsonAdd(sdoc("id", "B3", "_version_", "3")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- // reorder two buffered updates
- updateJ(
- jsonAdd(sdoc("id", "B4", "_version_", "1040")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- deleteAndGetVersion(
- "B4",
- params(
- DISTRIB_UPDATE_PARAM,
- FROM_LEADER,
- "_version_",
- "-940")); // this update should not take affect
- updateJ(
- jsonAdd(sdoc("id", "B6", "_version_", "1060")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "B5", "_version_", "1050")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "B8", "_version_", "1080")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- // test that delete by query is at least buffered along with everything else so it will delete
- // the
- // currently buffered id:8 (even if it doesn't currently support versioning)
- updateJ(
- "{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}",
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-3000"));
-
- assertJQ(
- req("qt", "/get", "getVersions", "13"),
- "=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}" // the
- // "3"
- // appears because versions aren't checked while buffering
- );
-
- logReplay.drainPermits();
- rinfoFuture = ulog.applyBufferedUpdates();
- assertNotNull(rinfoFuture);
- assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
-
- // apply a single update
- logReplay.release(1);
-
- // now add another update
- updateJ(
- jsonAdd(sdoc("id", "B7", "_version_", "1070")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- // a reordered update that should be dropped
- deleteAndGetVersion("B5", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-950"));
-
- deleteAndGetVersion("B6", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2060"));
-
- logReplay.release(1000);
- UpdateLog.RecoveryInfo recInfo = rinfoFuture.get();
-
- assertJQ(
- req("q", "*:*", "sort", "id asc", "fl", "id,_version_"),
- "/response/docs==["
- + "{'id':'B3','_version_':1030}"
- + ",{'id':'B4','_version_':1040}"
- + ",{'id':'B5','_version_':1050}"
- + ",{'id':'B7','_version_':1070}"
- + "]");
-
- assertEquals(1, recInfo.deleteByQuery);
-
- assertEquals(
- UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
- } finally {
- UpdateLog.testing_logReplayHook = null;
- UpdateLog.testing_logReplayFinishHook = null;
-
- req().close();
- }
- }
-
- @Test
- @Ignore("HDFS-3107: no truncate support yet")
- public void testDropBuffered() throws Exception {
- TestInjection.skipIndexWriterCommitOnClose = true;
- final Semaphore logReplay = new Semaphore(0);
- final Semaphore logReplayFinish = new Semaphore(0);
-
- UpdateLog.testing_logReplayHook =
- () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
-
- SolrQueryRequest req = req();
- UpdateHandler uhandler = req.getCore().getUpdateHandler();
- UpdateLog ulog = uhandler.getUpdateLog();
-
- try {
- clearIndex();
- assertU(commit());
-
- assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
- ulog.bufferUpdates();
- assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
- Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
- assertNull(rinfoFuture);
- assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
-
- ulog.bufferUpdates();
- assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
-
- // simulate updates from a leader
- updateJ(
- jsonAdd(sdoc("id", "C1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "C2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "C3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- assertTrue(ulog.dropBufferedUpdates());
- ulog.bufferUpdates();
- updateJ(
- jsonAdd(sdoc("id", "C4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "C5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- logReplay.release(1000);
- rinfoFuture = ulog.applyBufferedUpdates();
- UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
- assertEquals(2, rinfo.adds);
-
- assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[105,104]}");
-
- // this time add some docs first before buffering starts (so tlog won't be at pos 0)
- updateJ(
- jsonAdd(sdoc("id", "C100", "_version_", "200")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "C101", "_version_", "201")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- ulog.bufferUpdates();
- updateJ(
- jsonAdd(sdoc("id", "C103", "_version_", "203")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "C104", "_version_", "204")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- assertTrue(ulog.dropBufferedUpdates());
- ulog.bufferUpdates();
- updateJ(
- jsonAdd(sdoc("id", "C105", "_version_", "205")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "C106", "_version_", "206")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- rinfoFuture = ulog.applyBufferedUpdates();
- rinfo = rinfoFuture.get();
- assertEquals(2, rinfo.adds);
-
- assertJQ(
- req("q", "*:*", "sort", "_version_ asc", "fl", "id,_version_"),
- "/response/docs==["
- + "{'id':'C4','_version_':104}"
- + ",{'id':'C5','_version_':105}"
- + ",{'id':'C100','_version_':200}"
- + ",{'id':'C101','_version_':201}"
- + ",{'id':'C105','_version_':205}"
- + ",{'id':'C106','_version_':206}"
- + "]");
-
- assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[206,205,201,200,105,104]}");
-
- ulog.bufferUpdates();
- assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
- updateJ(
- jsonAdd(sdoc("id", "C301", "_version_", "998")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "C302", "_version_", "999")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- assertTrue(ulog.dropBufferedUpdates());
-
- // make sure we can overwrite with a lower version
- // TODO: is this functionality needed?
- updateJ(
- jsonAdd(sdoc("id", "C301", "_version_", "301")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "C302", "_version_", "302")),
- params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- assertU(commit());
-
- assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[302,301]}");
-
- assertJQ(
- req("q", "*:*", "sort", "_version_ desc", "fl", "id,_version_", "rows", "2"),
- "/response/docs==["
- + "{'id':'C302','_version_':302}"
- + ",{'id':'C301','_version_':301}"
- + "]");
-
- updateJ(
- jsonAdd(sdoc("id", "C2", "_version_", "302")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- assertEquals(
- UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
- } finally {
- UpdateLog.testing_logReplayHook = null;
- UpdateLog.testing_logReplayFinishHook = null;
-
- req().close();
- }
- }
-
- @Test
- public void testExistOldBufferLog() throws Exception {
- TestInjection.skipIndexWriterCommitOnClose = true;
-
- SolrQueryRequest req = req();
- UpdateHandler uhandler = req.getCore().getUpdateHandler();
- UpdateLog ulog = uhandler.getUpdateLog();
-
- try {
- clearIndex();
- assertU(commit());
-
- assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
- ulog.bufferUpdates();
-
- // simulate updates from a leader
- updateJ(
- jsonAdd(sdoc("id", "Q1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "Q2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "Q3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
-
- req.close();
- h.close();
- createCore();
-
- req = req();
- uhandler = req.getCore().getUpdateHandler();
- ulog = uhandler.getUpdateLog();
-
- // the core no longer replay updates from buffer tlog on startup
- assertTrue(
- ulog.existOldBufferLog()); // since we died while buffering, we should see this last
-
- // buffer tlog won't be removed on restart
- req.close();
- h.close();
- createCore();
-
- req = req();
- uhandler = req.getCore().getUpdateHandler();
- ulog = uhandler.getUpdateLog();
-
- assertTrue(ulog.existOldBufferLog());
-
- ulog.bufferUpdates();
- // simulate receiving no updates
- ulog.applyBufferedUpdates();
- updateJ(
- jsonAdd(sdoc("id", "Q7", "_version_", "117")),
- params(
- DISTRIB_UPDATE_PARAM,
- FROM_LEADER)); // do another add to make sure flags are back to normal
-
- req.close();
- h.close();
- createCore();
-
- req = req();
- uhandler = req.getCore().getUpdateHandler();
- ulog = uhandler.getUpdateLog();
-
- assertFalse(ulog.existOldBufferLog());
- // Timeout for Q7 get replayed, because it was added on tlog, therefore it will be replayed on
- // restart
- TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- timeout.waitFor(
- "Timeout waiting for finish replay updates",
- () -> h.getCore().getUpdateHandler().getUpdateLog().getState() == UpdateLog.State.ACTIVE);
- assertJQ(req("qt", "/get", "id", "Q7"), "/doc/id==Q7");
- } finally {
- UpdateLog.testing_logReplayHook = null;
- UpdateLog.testing_logReplayFinishHook = null;
-
- req().close();
- }
- }
-
- // make sure that on a restart, versions don't start too low
- @Test
- public void testVersionsOnRestart() throws Exception {
- clearIndex();
- assertU(commit());
-
- assertU(adoc("id", "D1", "val_i", "1"));
- assertU(adoc("id", "D2", "val_i", "1"));
- assertU(commit());
- long v1 = getVer(req("q", "id:D1"));
- long v1a = getVer(req("q", "id:D2"));
-
- h.close();
- createCore();
-
- assertU(adoc("id", "D1", "val_i", "2"));
- assertU(commit());
- long v2 = getVer(req("q", "id:D1"));
-
- assertTrue(v2 > v1);
-
- assertJQ(req("qt", "/get", "getVersions", "2"), "/versions==[" + v2 + "," + v1a + "]");
- }
-
- // make sure that log isn't needlessly replayed after a clean close
- @Test
- public void testCleanShutdown() throws Exception {
- final Semaphore logReplay = new Semaphore(0);
- final Semaphore logReplayFinish = new Semaphore(0);
-
- UpdateLog.testing_logReplayHook =
- () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
-
- SolrQueryRequest req = req();
- UpdateHandler uhandler = req.getCore().getUpdateHandler();
- UpdateLog ulog = uhandler.getUpdateLog();
-
- try {
- clearIndex();
- assertU(commit());
-
- assertU(adoc("id", "E1", "val_i", "1"));
- assertU(adoc("id", "E2", "val_i", "1"));
-
- // set to a high enough number so this test won't hang on a bug
- logReplay.release(10);
-
- h.close();
- createCore();
-
- // make sure the docs got committed
- assertJQ(req("q", "*:*"), "/response/numFound==2");
-
- // make sure no replay happened
- assertEquals(10, logReplay.availablePermits());
-
- } finally {
- UpdateLog.testing_logReplayHook = null;
- UpdateLog.testing_logReplayFinishHook = null;
-
- req().close();
- }
- }
-
- private void addDocs(int nDocs, int start, LinkedList<Long> versions) throws Exception {
- for (int i = 0; i < nDocs; i++) {
- versions.addFirst(addAndGetVersion(sdoc("id", Integer.toString(start + nDocs)), null));
- }
- }
-
- @Test
- @SuppressWarnings("JdkObsolete")
- public void testRemoveOldLogs() throws Exception {
- try {
- TestInjection.skipIndexWriterCommitOnClose = true;
- final Semaphore logReplay = new Semaphore(0);
- final Semaphore logReplayFinish = new Semaphore(0);
-
- UpdateLog.testing_logReplayHook =
- () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
-
- clearIndex();
- assertU(commit());
-
- String logDir = h.getCore().getUpdateHandler().getUpdateLog().getTlogDir();
-
- h.close();
-
- String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
- for (String file : files) {
- fs.delete(new Path(logDir, file), false);
- }
-
- assertEquals(0, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
-
- createCore();
-
- int start = 0;
- int maxReq = 50;
-
- LinkedList<Long> versions = new LinkedList<>();
- addDocs(10, start, versions);
- start += 10;
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
- assertU(commit());
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
-
- addDocs(10, start, versions);
- start += 10;
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
- assertU(commit());
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
-
- assertEquals(2, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
-
- addDocs(105, start, versions);
- start += 105;
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
- assertU(commit());
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
-
- // previous two logs should be gone now
- assertEquals(1, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
-
- addDocs(1, start, versions);
- start += 1;
- h.close();
- createCore(); // trigger recovery, make sure that tlog reference handling is correct
-
- // test we can get versions while replay is happening
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
-
- logReplay.release(1000);
- assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
-
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
-
- addDocs(105, start, versions);
- start += 105;
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
- assertU(commit());
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
-
- // previous logs should be gone now
- assertEquals(1, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
-
- //
- // test that a corrupt tlog file doesn't stop us from coming up, or seeing versions before
- // that tlog file.
- //
- addDocs(
- 1,
- start,
- new LinkedList<
- Long>()); // don't add this to the versions list because we are going to lose it...
- h.close();
- files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
- Arrays.sort(files);
-
- FSDataOutputStream dos =
- fs.create(new Path(new Path(logDir), files[files.length - 1]), (short) 1);
- dos.writeUTF("This is a trashed log file that really shouldn't work at all, but we'll see..");
- dos.close();
-
- ignoreException("Failure to open existing");
- createCore();
- // we should still be able to get the list of versions (not including the trashed log file)
- assertJQ(
- req("qt", "/get", "getVersions", "" + maxReq),
- "/versions==" + versions.subList(0, Math.min(maxReq, start)));
- resetExceptionIgnores();
-
- } finally {
- UpdateLog.testing_logReplayHook = null;
- UpdateLog.testing_logReplayFinishHook = null;
- }
- }
-
- //
- // test that a partially written last tlog entry (that will cause problems for both reverse
- // reading and for
- // log replay) doesn't stop us from coming up, and from recovering the documents that were not cut
- // off.
- //
- @Test
- public void testTruncatedLog() throws Exception {
- try {
- TestInjection.skipIndexWriterCommitOnClose = true;
- final Semaphore logReplay = new Semaphore(0);
- final Semaphore logReplayFinish = new Semaphore(0);
-
- UpdateLog.testing_logReplayHook =
- () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
-
- String logDir = h.getCore().getUpdateHandler().getUpdateLog().getTlogDir();
-
- clearIndex();
- assertU(commit());
-
- assertU(adoc("id", "F1"));
- assertU(adoc("id", "F2"));
- assertU(adoc("id", "F3"));
-
- h.close();
-
- String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
- Arrays.sort(files);
-
- FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length - 1]));
-
- dos.writeLong(0xffffffffffffffffL);
- dos.writeChars(
- "This should be appended to a good log file, representing a bad partially written record.");
- dos.close();
-
- logReplay.release(1000);
- logReplayFinish.drainPermits();
- ignoreException(
- "OutOfBoundsException"); // this is what the corrupted log currently produces... subject
- // to change.
- createCore();
- assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- resetExceptionIgnores();
- assertJQ(req("q", "*:*"), "/response/numFound==3");
-
- //
- // Now test that the bad log file doesn't mess up retrieving latest versions
- //
-
- updateJ(
- jsonAdd(sdoc("id", "F4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "F5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "F6", "_version_", "106")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- // This currently skips the bad log file and also returns the version of the clearIndex (del
- // *:*)
- // assertJQ(req("qt","/get", "getVersions","6"), "/versions==[106,105,104]");
- assertJQ(req("qt", "/get", "getVersions", "3"), "/versions==[106,105,104]");
-
- } finally {
- UpdateLog.testing_logReplayHook = null;
- UpdateLog.testing_logReplayFinishHook = null;
- }
- }
-
- //
- // test that a corrupt tlog doesn't stop us from coming up
- //
- @Test
- public void testCorruptLog() throws Exception {
- try {
- TestInjection.skipIndexWriterCommitOnClose = true;
-
- String logDir = h.getCore().getUpdateHandler().getUpdateLog().getTlogDir();
-
- clearIndex();
- assertU(commit());
-
- assertU(adoc("id", "G1"));
- assertU(adoc("id", "G2"));
- assertU(adoc("id", "G3"));
-
- h.close();
-
- String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
- Arrays.sort(files);
-
- FSDataOutputStream dos = fs.create(new Path(logDir, files[files.length - 1]), (short) 1);
- dos.write(new byte[800]); // zero out file
- dos.close();
-
- ignoreException(
- "Failure to open existing log file"); // this is what the corrupted log currently
- // produces... subject to change.
- createCore();
- resetExceptionIgnores();
-
- // just make sure it responds
- assertJQ(req("q", "*:*"), "/response/numFound==0");
-
- //
- // Now test that the bad log file doesn't mess up retrieving latest versions
- //
-
- updateJ(
- jsonAdd(sdoc("id", "G4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "G5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- updateJ(
- jsonAdd(sdoc("id", "G6", "_version_", "106")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
-
- // This currently skips the bad log file and also returns the version of the clearIndex (del
- // *:*)
- // assertJQ(req("qt","/get", "getVersions","6"), "/versions==[106,105,104]");
- assertJQ(req("qt", "/get", "getVersions", "3"), "/versions==[106,105,104]");
-
- assertU(commit());
-
- assertJQ(req("q", "*:*"), "/response/numFound==3");
-
- // This messes up some other tests (on windows) if we don't remove the bad log.
- // This *should* hopefully just be because the tests are too fragile and not because of real
- // bugs - but it should be investigated further.
- deleteLogs();
-
- } finally {
- UpdateLog.testing_logReplayHook = null;
- UpdateLog.testing_logReplayFinishHook = null;
- }
- }
-
- // in rare circumstances, two logs can be left uncapped (lacking a commit at the end signifying
- // that all the content in the log was committed)
- @Test
- public void testRecoveryMultipleLogs() throws Exception {
- try {
- TestInjection.skipIndexWriterCommitOnClose = true;
- final Semaphore logReplay = new Semaphore(0);
- final Semaphore logReplayFinish = new Semaphore(0);
-
- UpdateLog.testing_logReplayHook =
- () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
-
- String logDir = h.getCore().getUpdateHandler().getUpdateLog().getTlogDir();
-
- clearIndex();
- assertU(commit());
-
- assertU(adoc("id", "AAAAAA"));
- assertU(adoc("id", "BBBBBB"));
- assertU(adoc("id", "CCCCCC"));
-
- h.close();
- String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
- Arrays.sort(files);
- String fname = files[files.length - 1];
-
- try (FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length - 1]))) {
- dos.writeLong(0xffffffffffffffffL);
- dos.writeChars(
- "This should be appended to a good log file, representing a bad partially written record.");
- }
-
- try (FSDataInputStream dis = fs.open(new Path(logDir, files[files.length - 1]))) {
- byte[] content = new byte[dis.available()];
- dis.readFully(content);
-
- // Now make a newer log file with just the IDs changed. NOTE: this may not work if log
- // format changes too much!
- findReplace(
- "AAAAAA".getBytes(StandardCharsets.UTF_8),
- "aaaaaa".getBytes(StandardCharsets.UTF_8),
- content);
- findReplace(
- "BBBBBB".getBytes(StandardCharsets.UTF_8),
- "bbbbbb".getBytes(StandardCharsets.UTF_8),
- content);
- findReplace(
- "CCCCCC".getBytes(StandardCharsets.UTF_8),
- "cccccc".getBytes(StandardCharsets.UTF_8),
- content);
-
- // WARNING... assumes format of .00000n where n is less than 9
- long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf('.') + 1));
- String fname2 =
- String.format(
- Locale.ROOT, UpdateLog.LOG_FILENAME_PATTERN, UpdateLog.TLOG_NAME, logNumber + 1);
-
- try (FSDataOutputStream dos = fs.create(new Path(logDir, fname2), (short) 1)) {
- dos.write(content);
- }
- }
-
- logReplay.release(1000);
- logReplayFinish.drainPermits();
- ignoreException(
- "OutOfBoundsException"); // this is what the corrupted log currently produces... subject
- // to change.
- createCore();
- assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- resetExceptionIgnores();
- assertJQ(req("q", "*:*"), "/response/numFound==6");
-
- } finally {
- UpdateLog.testing_logReplayHook = null;
- UpdateLog.testing_logReplayFinishHook = null;
- }
- }
-
- // NOTE: replacement must currently be same size
- private static void findReplace(byte[] from, byte[] to, byte[] data) {
- int idx = -from.length;
- for (; ; ) {
- idx = indexOf(from, data, idx + from.length); // skip over previous match
- if (idx < 0) break;
- System.arraycopy(to, 0, data, idx, to.length);
- }
- }
-
- private static int indexOf(byte[] target, byte[] data, int start) {
- outer:
- for (int i = start; i < data.length - target.length; i++) {
- for (int j = 0; j < target.length; j++) {
- if (data[i + j] != target[j]) continue outer;
- }
- return i;
- }
- return -1;
- }
-
- // stops the core, removes the transaction logs, restarts the core.
- void deleteLogs() throws Exception {
- String logDir = h.getCore().getUpdateHandler().getUpdateLog().getTlogDir();
-
- h.close();
-
- try {
- String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
- for (String file : files) {
- // new File(logDir, file).delete();
- fs.delete(new Path(logDir, file), false);
- }
-
- assertEquals(0, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
- } finally {
- // make sure we create the core again, even if the assert fails so it won't mess
- // up the next test.
- createCore();
- assertJQ(req("q", "*:*"), "/response/numFound=="); // ensure it works
- }
- }
-
- private static Long getVer(SolrQueryRequest req) throws Exception {
- String response = JQ(req);
- @SuppressWarnings({"rawtypes"})
- Map rsp = (Map) Utils.fromJSONString(response);
- @SuppressWarnings({"rawtypes"})
- Map doc = null;
- if (rsp.containsKey("doc")) {
- doc = (Map) rsp.get("doc");
- } else if (rsp.containsKey("docs")) {
- @SuppressWarnings({"rawtypes"})
- List lst = (List) rsp.get("docs");
- if (lst.size() > 0) {
- doc = (Map) lst.get(0);
- }
- } else if (rsp.containsKey("response")) {
- @SuppressWarnings({"rawtypes"})
- Map responseMap = (Map) rsp.get("response");
- @SuppressWarnings({"rawtypes"})
- List lst = (List) responseMap.get("docs");
- if (lst.size() > 0) {
- doc = (Map) lst.get(0);
- }
- }
-
- if (doc == null) return null;
-
- return (Long) doc.get("_version_");
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/HdfsDirectoryTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/HdfsDirectoryTest.java
deleted file mode 100644
index 0a0b9aad759..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/HdfsDirectoryTest.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.store;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.FileAlreadyExistsException;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.store.ByteBuffersDirectory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@ThreadLeakLingering(
- linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsDirectoryTest extends SolrTestCaseJ4 {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final int MAX_NUMBER_OF_WRITES = 10000;
- private static final int MIN_FILE_SIZE = 100;
- private static final int MAX_FILE_SIZE = 100000;
- private static final int MIN_BUFFER_SIZE = 1;
- private static final int MAX_BUFFER_SIZE = 5000;
- private static final int MAX_NUMBER_OF_READS = 10000;
- private static MiniDFSCluster dfsCluster;
- private Configuration directoryConf;
- private Path directoryPath;
- private HdfsDirectory directory;
- private Random random;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
-
- directoryConf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- directoryConf.set("dfs.permissions.enabled", "false");
-
- directoryPath =
- new Path(
- dfsCluster.getURI().toString() + createTempDir().toFile().getAbsolutePath() + "/hdfs");
- directory = new HdfsDirectory(directoryPath, directoryConf);
-
- random = random();
- }
-
- @Override
- @After
- public void tearDown() throws Exception {
- super.tearDown();
- }
-
- @Test
- public void testWritingAndReadingAFile() throws IOException {
- String[] listAll = directory.listAll();
- for (String file : listAll) {
- directory.deleteFile(file);
- }
-
- IndexOutput output = directory.createOutput("testing.test", new IOContext());
- output.writeInt(12345);
- output.close();
-
- IndexInput input = directory.openInput("testing.test", new IOContext());
- assertEquals(12345, input.readInt());
- input.close();
-
- listAll = directory.listAll();
- assertEquals(1, listAll.length);
- assertEquals("testing.test", listAll[0]);
-
- assertEquals(4, directory.fileLength("testing.test"));
-
- IndexInput input1 = directory.openInput("testing.test", new IOContext());
-
- IndexInput input2 = input1.clone();
- assertEquals(12345, input2.readInt());
- input2.close();
-
- assertEquals(12345, input1.readInt());
- input1.close();
-
- assertFalse(slowFileExists(directory, "testing.test.other"));
- assertTrue(slowFileExists(directory, "testing.test"));
- directory.deleteFile("testing.test");
- assertFalse(slowFileExists(directory, "testing.test"));
- }
-
- public void testRename() throws IOException {
- String[] listAll = directory.listAll();
- for (String file : listAll) {
- directory.deleteFile(file);
- }
-
- IndexOutput output = directory.createOutput("testing.test", new IOContext());
- output.writeInt(12345);
- output.close();
- directory.rename("testing.test", "testing.test.renamed");
- assertFalse(slowFileExists(directory, "testing.test"));
- assertTrue(slowFileExists(directory, "testing.test.renamed"));
- IndexInput input = directory.openInput("testing.test.renamed", new IOContext());
- assertEquals(12345, input.readInt());
- assertEquals(input.getFilePointer(), input.length());
- input.close();
- directory.deleteFile("testing.test.renamed");
- assertFalse(slowFileExists(directory, "testing.test.renamed"));
- }
-
- @Test
- public void testEOF() throws IOException {
- Directory fsDir = new ByteBuffersDirectory();
- String name = "test.eof";
- createFile(name, fsDir, directory);
- long fsLength = fsDir.fileLength(name);
- long hdfsLength = directory.fileLength(name);
- assertEquals(fsLength, hdfsLength);
- testEof(name, fsDir, fsLength);
- testEof(name, directory, hdfsLength);
- }
-
- private void testEof(String name, Directory directory, long length) throws IOException {
- IndexInput input = directory.openInput(name, new IOContext());
- input.seek(length);
- expectThrows(Exception.class, input::readByte);
- }
-
- @Test
- public void testRandomAccessWrites() throws IOException {
- int i = 0;
- try {
- Set<String> names = new HashSet<>();
- for (; i < 10; i++) {
- Directory fsDir = new ByteBuffersDirectory();
- String name = getName();
- System.out.println("Working on pass [" + i + "] contains [" + names.contains(name) + "]");
- names.add(name);
- createFile(name, fsDir, directory);
- assertInputsEquals(name, fsDir, directory);
- fsDir.close();
- }
- } catch (Exception e) {
- log.error("Test failed on pass [{}]", i, e);
- fail("Test failed on pass [" + i + "]");
- }
- }
-
- private void assertInputsEquals(String name, Directory fsDir, HdfsDirectory hdfs)
- throws IOException {
- int reads = random.nextInt(MAX_NUMBER_OF_READS);
- IndexInput fsInput = fsDir.openInput(name, new IOContext());
- IndexInput hdfsInput = hdfs.openInput(name, new IOContext());
- assertEquals(fsInput.length(), hdfsInput.length());
- int fileLength = (int) fsInput.length();
- for (int i = 0; i < reads; i++) {
- int nextInt = Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength);
- byte[] fsBuf = new byte[random.nextInt(nextInt > 0 ? nextInt : 1) + MIN_BUFFER_SIZE];
- byte[] hdfsBuf = new byte[fsBuf.length];
- int offset = random.nextInt(fsBuf.length);
-
- nextInt = fsBuf.length - offset;
- int length = random.nextInt(nextInt > 0 ? nextInt : 1);
- nextInt = fileLength - length;
- int pos = random.nextInt(nextInt > 0 ? nextInt : 1);
- fsInput.seek(pos);
- fsInput.readBytes(fsBuf, offset, length);
- hdfsInput.seek(pos);
- hdfsInput.readBytes(hdfsBuf, offset, length);
- for (int f = offset; f < length; f++) {
- if (fsBuf[f] != hdfsBuf[f]) {
- fail();
- }
- }
- }
- fsInput.close();
- hdfsInput.close();
- }
-
- private void createFile(String name, Directory fsDir, HdfsDirectory hdfs) throws IOException {
- int writes = random.nextInt(MAX_NUMBER_OF_WRITES);
- int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE;
- IndexOutput fsOutput = fsDir.createOutput(name, new IOContext());
- IndexOutput hdfsOutput = hdfs.createOutput(name, new IOContext());
- for (int i = 0; i < writes; i++) {
- byte[] buf =
- new byte
- [random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength))
- + MIN_BUFFER_SIZE];
- random.nextBytes(buf);
- int offset = random.nextInt(buf.length);
- int length = random.nextInt(buf.length - offset);
- fsOutput.writeBytes(buf, offset, length);
- hdfsOutput.writeBytes(buf, offset, length);
- }
- fsOutput.close();
- hdfsOutput.close();
- }
-
- private String getName() {
- return Long.toUnsignedString(random.nextLong());
- }
-
- public void testCantOverrideFiles() throws IOException {
- try (IndexOutput out = directory.createOutput("foo", IOContext.DEFAULT)) {
- out.writeByte((byte) 42);
- }
- expectThrows(
- FileAlreadyExistsException.class, () -> directory.createOutput("foo", IOContext.DEFAULT));
- }
-
- public void testCreateTempFiles() throws IOException {
- String file1;
- try (Directory dir = new HdfsDirectory(directoryPath, directoryConf);
- IndexOutput out = dir.createTempOutput("foo", "bar", IOContext.DEFAULT)) {
- out.writeByte((byte) 42);
- file1 = out.getName();
- }
- assertTrue(file1.startsWith("foo_bar"));
- assertTrue(file1.endsWith(".tmp"));
- // Create the directory again to force the counter to be reset
- String file2;
- try (Directory dir = new HdfsDirectory(directoryPath, directoryConf);
- IndexOutput out = dir.createTempOutput("foo", "bar", IOContext.DEFAULT)) {
- out.writeByte((byte) 42);
- file2 = out.getName();
- }
- assertTrue(file2.startsWith("foo_bar"));
- assertTrue(file2.endsWith(".tmp"));
- assertNotEquals(file1, file2);
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/HdfsLockFactoryTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/HdfsLockFactoryTest.java
deleted file mode 100644
index cb7c6000aa1..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/store/HdfsLockFactoryTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.store;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-@Nightly
-public class HdfsLockFactoryTest extends SolrTestCaseJ4 {
-
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Test
- public void testBasic() throws IOException {
- String uri = HdfsTestUtil.getURI(dfsCluster);
- Path lockPath = new Path(uri, "/basedir/lock");
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- HdfsDirectory dir = new HdfsDirectory(lockPath, conf);
-
- try (Lock lock = dir.obtainLock("testlock")) {
- assertNotNull(lock);
- try (Lock lock2 = dir.obtainLock("testlock")) {
- assertNotNull(lock2);
- fail("Locking should fail");
- } catch (LockObtainFailedException lofe) {
- // pass
- }
- }
- // now repeat after close()
- try (Lock lock = dir.obtainLock("testlock")) {
- assertNotNull(lock);
- try (Lock lock2 = dir.obtainLock("testlock")) {
- assertNotNull(lock2);
- fail("Locking should fail");
- } catch (LockObtainFailedException lofe) {
- // pass
- }
- }
- dir.close();
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/update/TestHdfsUpdateLog.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/update/TestHdfsUpdateLog.java
deleted file mode 100644
index 793093d4e30..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/update/TestHdfsUpdateLog.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.update;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
-import org.apache.solr.SolrIgnoredThreadsFilter;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.hdfs.cloud.HdfsTestUtil;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.update.DirectUpdateHandler2;
-import org.apache.solr.update.UpdateHandler;
-import org.apache.solr.update.UpdateLog;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@ThreadLeakFilters(
- defaultFilters = true,
- filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
- })
-public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
- private static MiniDFSCluster dfsCluster;
- private static String hdfsUri;
- private static FileSystem fs;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- hdfsUri = HdfsTestUtil.getURI(dfsCluster);
-
- try {
- URI uri = new URI(hdfsUri);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- fs = FileSystem.get(uri, conf);
- } catch (IOException | URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- System.setProperty("solr.ulog.dir", hdfsUri + "/solr/shard1");
-
- initCore("solrconfig-tlog.xml", "schema15.xml");
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- IOUtils.closeQuietly(fs);
- fs = null;
- try {
- // Make sure to clean up test resources before shutting down HDFS
- SolrTestCaseJ4.teardownTestCases();
- } finally {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- hdfsUri = null;
- System.clearProperty("solr.ulog.dir");
- }
- }
- }
-
- @Test
- public void testFSThreadSafety() throws Exception {
- final SolrQueryRequest req = req();
- final UpdateHandler uhandler = req.getCore().getUpdateHandler();
- ((DirectUpdateHandler2) uhandler).getCommitTracker().setTimeUpperBound(100);
- ((DirectUpdateHandler2) uhandler).getCommitTracker().setOpenSearcher(false);
- final UpdateLog ulog = uhandler.getUpdateLog();
-
- clearIndex();
- assertU(commit());
-
- // we hammer on init in a background thread to make
- // sure we don't run into any filesystem already closed
- // problems (SOLR-7113)
-
- Thread thread =
- new Thread() {
- @Override
- public void run() {
- int cnt = 0;
- while (true) {
- ulog.init(uhandler, req.getCore());
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
-
- }
- if (cnt++ > 50) {
- break;
- }
- }
- }
- };
-
- Thread thread2 =
- new Thread() {
- @Override
- public void run() {
- int cnt = 0;
- while (true) {
- assertU(adoc("id", Integer.toString(cnt)));
- try {
- Thread.sleep(10);
- } catch (InterruptedException e) {
-
- }
- if (cnt++ > 500) {
- break;
- }
- }
- }
- };
-
- thread.start();
- thread2.start();
- thread.join();
- thread2.join();
- }
-}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/util/BadHdfsThreadsFilter.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/util/BadHdfsThreadsFilter.java
deleted file mode 100644
index d2fc7d790cd..00000000000
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/util/BadHdfsThreadsFilter.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.hdfs.util;
-
-import com.carrotsearch.randomizedtesting.ThreadFilter;
-
-public class BadHdfsThreadsFilter implements ThreadFilter {
-
- @Override
- public boolean reject(Thread t) {
- String name = t.getName();
- if (name.startsWith("IPC Parameter Sending Thread ")) { // SOLR-5007
- return true;
- }
- if (name.startsWith("IPC Client")) { // SOLR-5007
- return true;
- } else if (name.startsWith("org.apache.hadoop.hdfs.PeerCache")) { // SOLR-7288
- return true;
- } else if (name.endsWith("StatisticsDataReferenceCleaner")) {
- return true;
- } else if (name.startsWith("LeaseRenewer")) { // SOLR-7287
- return true;
- } else if (name.startsWith("org.apache.hadoop.fs.FileSystem$Statistics")) { // SOLR-11261
- return true;
- } else if (name.startsWith("ForkJoinPool.")) { // JVM built in pool
- return true;
- } else if (name.startsWith("solr-hdfs-threadpool-")) { // SOLR-9515 and HDFS-14251
- return true;
- } else if (name.startsWith("nioEventLoopGroup")) {
- return true; // netty threads waiting for a web server stop confirm that won't happen
- } else if (name.startsWith("Command processor")) { // SOLR-16043 and HDFS-14997
- return true;
- }
-
- return false;
- }
-}
diff --git a/solr/modules/jwt-auth/src/test-files/log4j2.xml b/solr/modules/jwt-auth/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/modules/jwt-auth/src/test-files/log4j2.xml
+++ b/solr/modules/jwt-auth/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/modules/langid/src/test-files/log4j2.xml b/solr/modules/langid/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/modules/langid/src/test-files/log4j2.xml
+++ b/solr/modules/langid/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/modules/llm/src/test-files/log4j2.xml b/solr/modules/llm/src/test-files/log4j2.xml
index 5e696935965..f668c06a1ed 100644
--- a/solr/modules/llm/src/test-files/log4j2.xml
+++ b/solr/modules/llm/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
@@ -40,4 +38,3 @@
</Root>
</Loggers>
</Configuration>
-
diff --git a/solr/modules/ltr/src/test-files/log4j2.xml b/solr/modules/ltr/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/modules/ltr/src/test-files/log4j2.xml
+++ b/solr/modules/ltr/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/modules/opentelemetry/src/test-files/log4j2.xml b/solr/modules/opentelemetry/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/modules/opentelemetry/src/test-files/log4j2.xml
+++ b/solr/modules/opentelemetry/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/modules/s3-repository/src/test-files/log4j2.xml b/solr/modules/s3-repository/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/modules/s3-repository/src/test-files/log4j2.xml
+++ b/solr/modules/s3-repository/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/modules/scripting/src/test-files/log4j2.xml b/solr/modules/scripting/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/modules/scripting/src/test-files/log4j2.xml
+++ b/solr/modules/scripting/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/prometheus-exporter/src/test-files/log4j2.xml b/solr/prometheus-exporter/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/prometheus-exporter/src/test-files/log4j2.xml
+++ b/solr/prometheus-exporter/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/server/etc/security.policy b/solr/server/etc/security.policy
index d9f8791486d..aec689deae6 100644
--- a/solr/server/etc/security.policy
+++ b/solr/server/etc/security.policy
@@ -104,16 +104,7 @@ grant {
permission java.lang.RuntimePermission "closeClassLoader";
// needed by HttpSolrClient
permission java.lang.RuntimePermission "getFileSystemAttributes";
- // needed by hadoop hdfs (TODO: there is a cleaner way to handle this)
- permission java.lang.RuntimePermission "loadLibrary.jaas";
- permission java.lang.RuntimePermission "loadLibrary.jaas_unix";
- permission java.lang.RuntimePermission "loadLibrary.jaas_nt";
- // needed by hadoop common RawLocalFileSystem for java nio getOwner
- permission java.lang.RuntimePermission "accessUserInformation";
- // needed by hadoop hdfs
- permission java.lang.RuntimePermission "readFileDescriptor";
- permission java.lang.RuntimePermission "writeFileDescriptor";
- // needed by hadoop http
+ // need to lookup remote JMX
permission java.lang.RuntimePermission "getProtectionDomain";
// needed by aws s3 sdk (Apache HTTP Client)
permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.reflect";
@@ -139,21 +130,10 @@ grant {
permission javax.management.MBeanServerPermission "releaseMBeanServer";
permission javax.management.MBeanTrustPermission "register";
- // needed by hadoop hdfs
- permission javax.security.auth.AuthPermission "getSubject";
- permission javax.security.auth.AuthPermission "modifyPrincipals";
- permission javax.security.auth.AuthPermission "doAs";
- permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
- permission javax.security.auth.AuthPermission "modifyPublicCredentials";
- permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
-
+
// needed by crossdc
permission javax.security.auth.AuthPermission "getLoginConfiguration";
permission javax.security.auth.AuthPermission "setLoginConfiguration";
-
- // needed by hadoop security
- permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
- permission java.security.SecurityPermission "insertProvider";
// Needed by JWT integration tests & S3 clients
permission java.lang.RuntimePermission "setFactory";
@@ -189,9 +169,6 @@ grant {
// additional permissions based on system properties set by /bin/solr
// NOTE: if the property is not set, the permission entry is ignored.
grant {
- permission java.io.FilePermission "${hadoop.security.credential.provider.path}", "read,write,delete,readlink";
- permission java.io.FilePermission "${hadoop.security.credential.provider.path}${/}-", "read,write,delete,readlink";
-
permission java.io.FilePermission "${solr.jetty.keystore}", "read,readlink";
permission java.io.FilePermission "${solr.jetty.truststore}", "read,readlink";
diff --git a/solr/server/resources/log4j2-console.xml b/solr/server/resources/log4j2-console.xml
index 2d5b8690bf6..b14295161c7 100644
--- a/solr/server/resources/log4j2-console.xml
+++ b/solr/server/resources/log4j2-console.xml
@@ -32,7 +32,6 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<AsyncLogger name="org.apache.zookeeper" level="ERROR"/>
- <AsyncLogger name="org.apache.hadoop" level="WARN"/>
<AsyncLogger name="org.eclipse.jetty.deploy" level="warn"/>
<AsyncLogger name="org.eclipse.jetty.webapp" level="warn"/>
<AsyncLogger name="org.eclipse.jetty.server.session" level="warn"/>
@@ -42,5 +41,3 @@
</AsyncRoot>
</Loggers>
</Configuration>
-
-
diff --git a/solr/server/resources/log4j2.xml b/solr/server/resources/log4j2.xml
index f5b373338b7..1c59712cc79 100644
--- a/solr/server/resources/log4j2.xml
+++ b/solr/server/resources/log4j2.xml
@@ -63,7 +63,6 @@
</Appenders>
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchronous logging respectively -->
- <AsyncLogger name="org.apache.hadoop" level="warn"/>
<AsyncLogger name="org.apache.solr.update.LoggingInfoStream" level="off"/>
<AsyncLogger name="org.apache.zookeeper" level="warn"/>
<!-- HttpSolrCall adds markers denoting the handler class to allow fine grained control, metrics are
@@ -84,4 +83,3 @@
</AsyncRoot>
</Loggers>
</Configuration>
-
diff --git a/solr/solr-ref-guide/build.gradle b/solr/solr-ref-guide/build.gradle
index 811b0748568..0dcdd3b1408 100644
--- a/solr/solr-ref-guide/build.gradle
+++ b/solr/solr-ref-guide/build.gradle
@@ -113,7 +113,6 @@ task buildLocalAntoraYaml {
props.putAll([
["dep_version_commons_codec", libs.versions.commons.codec],
["dep_version_dropwizard", libs.versions.dropwizard.metrics],
- ["dep_version_hadoop", libs.versions.apache.hadoop.asProvider()],
["dep_version_log4j", libs.versions.apache.log4j],
["dep_version_opennlp", libs.versions.apache.opennlp],
["dep_version_tika", libs.versions.apache.tika],
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/index-location-format.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/index-location-format.adoc
index 17355724cd9..f10b0cc3290 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-location-format.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-location-format.adoc
@@ -57,9 +57,3 @@ Use this DirectoryFactory to store your index in RAM.
----
<directoryFactory class="org.apache.solr.core.RAMDirectoryFactory"/>
----
-
-[NOTE]
-====
-If you are using Hadoop and would like to store your indexes in HDFS, you should use the {solr-javadocs}/modules/hdfs/org/apache/solr/hdfs/HdfsDirectoryFactory.html[`solr.HdfsDirectoryFactory`] instead of either of the above implementations.
-For more details, see the section xref:deployment-guide:solr-on-hdfs.adoc[].
-====
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc
index ca292a85a79..165fc021984 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc
@@ -304,9 +304,6 @@ WARNING: If multiple Solr instances in different JVMs modify an index, this type
+
See also the {lucene-javadocs}/core/org/apache/lucene/store/SingleInstanceLockFactory.html[SingleInstanceLockFactory javadocs].
-* `hdfs` uses `HdfsLockFactory` to support reading and writing index and transaction log files to a HDFS filesystem.
-See the section xref:deployment-guide:solr-on-hdfs.adoc[] for more details on using this feature.
-
[source,xml]
----
<lockType>native</lockType>
diff --git a/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc b/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc
index 2e2b631f2b5..47ba93002f8 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc
@@ -28,7 +28,6 @@
** xref:backup-restore.adoc[]
** xref:solr-in-docker.adoc[]
*** xref:docker-faq.adoc[]
-** xref:solr-on-hdfs.adoc[]
* Scaling Solr
** xref:cluster-types.adoc[]
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc
index 6b48bc179e1..cc5454cb2e2 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc
@@ -36,7 +36,7 @@ Likewise, committing changes using `openSearcher=false` may result in changes co
Support for backups in SolrCloud is provided with the xref:collection-management.adoc#backup[Collections API].
This allows the backups to be generated across multiple shards, and restored to the same number of shards and replicas as the original collection.
-NOTE: SolrCloud Backup/Restore requires a shared file system mounted at the same path on all nodes, or HDFS.
+NOTE: SolrCloud Backup/Restore requires a shared file system mounted at the same path on all nodes.
Four different API commands are supported:
@@ -382,8 +382,8 @@ If the status is anything other than "success", an error message will explain wh
== Backup/Restore Storage Repositories
Solr provides a repository abstraction to allow users to backup and restore their data to a variety of different storage systems.
-For example, a Solr cluster running on a local filesystem (e.g., EXT3) can store backup data on the same disk, on a remote network-mounted drive, in HDFS, or even in some popular "cloud storage" providers, depending on the 'repository' implementation chosen.
-Solr offers multiple different repository implementations out of the box (`LocalFileSystemRepository`, `HdfsBackupRepository`, `GCSBackupRepository` and `S3BackupRepository`), and allows users to create plugins for their own storage systems as needed. It is also possible to create a `DelegatingBackupRepository` that delegates to another `BackupRepository` and adds or modifies some behavior on top of it.
+For example, a Solr cluster running on a local filesystem (e.g., EXT3) can store backup data on the same disk, on a remote network-mounted drive, or in some popular "cloud storage" providers, depending on the 'repository' implementation chosen.
+Solr offers multiple different repository implementations out of the box (`LocalFileSystemRepository`, `GCSBackupRepository` and `S3BackupRepository`), and allows users to create plugins for their own storage systems as needed. It is also possible to create a `DelegatingBackupRepository` that delegates to another `BackupRepository` and adds or modifies some behavior on top of it.
Users can define any number of repositories in their `solr.xml` file.
The backup and restore APIs described above allow users to select which of these definitions they want to use at runtime via the `repository` parameter.
@@ -441,67 +441,6 @@ An example configuration using this property can be found below.
</backup>
----
-
-=== HdfsBackupRepository
-
-Stores and retrieves backup files from HDFS directories.
-
-This is provided via the `hdfs` xref:configuration-guide:solr-modules.adoc[Solr Module] that needs to be enabled before use.
-
-HdfsBackupRepository accepts the following configuration options:
-
-`solr.hdfs.buffer.size`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `4096` kilobytes
-|===
-+
-The size, in bytes, of the buffer used to transfer data to and from HDFS.
-Better throughput is often attainable with a larger buffer, where memory allows.
-
-`solr.hdfs.home`::
-+
-[%autowidth,frame=none]
-|===
-s|Required |Default: none
-|===
-+
-A HDFS URI in the format `hdfs://<host>:<port>/<hdfsBaseFilePath>` that points Solr to the HDFS cluster to store (or retrieve) backup files on.
-
-`solr.hdfs.permissions.umask-mode`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: none
-|===
-+
-A permission umask used when creating files in HDFS.
-
-`location`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: none
-|===
-+
-A valid directory path on the HDFS cluster to use for backup storage and retrieval.
-Used as a fallback when users don't provide a `location` parameter in their Backup or Restore API commands.
-
-An example configuration using these properties can be found below:
-
-[source,xml]
-----
-<backup>
- <repository name="hdfs" class="org.apache.solr.hdfs.backup.repository.HdfsBackupRepository" default="false">
- <str name="solr.hdfs.home">hdfs://some_hdfs_host:1234/solr/backup/data</str>
- <int name="solr.hdfs.buffer.size">8192</int>
- <str name="solr.hdfs.permissions.umask-mode">0022</str>
- <str name="location">/default/hdfs/backup/location</str>
- </repository>
-</backup>
-----
-
=== GCSBackupRepository
Stores and retrieves backup files in a Google Cloud Storage ("GCS") bucket.
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/enabling-ssl.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/enabling-ssl.adoc
index fdaebc2f975..5d49dd99fde 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/enabling-ssl.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/enabling-ssl.adoc
@@ -136,89 +136,6 @@ If you are using SolrCloud, you need to <<Configure ZooKeeper>> before starting
If you are running Solr in a user-managed cluster or single-node installation, you can skip to <<Start User-Managed Cluster or Single-Node Solr>>.
-
-
-=== Password Distribution via Hadoop Credential Store
-
-Solr supports reading keystore and truststore passwords from Hadoop credential store.
-This approach can be beneficial if password rotation and distribution is already handled by credential stores.
-
-If you are not using a Hadoop credential store, you can skip this step.
-
-Hadoop credential store can be used with Solr using the following two steps.
-
-==== Provide a Hadoop Credential Store
-Create a Hadoop credstore file and define the entries below with the actual keystore passwords.
-
-[source,text]
-----
-solr.jetty.keystore.password
-solr.jetty.truststore.password
-javax.net.ssl.keyStorePassword
-javax.net.ssl.trustStorePassword
-----
-
-Note that if the `javax.net.ssl.\*` configurations are not set, they will fallback to the corresponding `solr.jetty.*` configurations.
-
-==== Configure Solr to use Hadoop Credential Store
-
-Solr requires three parameters to be configured in order to use the credential store file for keystore passwords.
-
-`solr.ssl.credential.provider.chain`::
-+
-[%autowidth,frame=none]
-|===
-s|Required |Default: none
-|===
-+
-The credential provider chain.
-This should be set to `hadoop`.
-
-`SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH`::
-+
-[%autowidth,frame=none]
-|===
-s|Required |Default: none
-|===
-+
-The path to the credential store file.
-
-`HADOOP_CREDSTORE_PASSWORD`::
-+
-[%autowidth,frame=none]
-|===
-s|Required |Default: none
-|===
-+
-The password to the credential store.
-
-[tabs#credstore]
-======
-*nix::
-+
-====
-.solr.in.sh
-[source,bash]
-----
-SOLR_OPTS=" -Dsolr.ssl.credential.provider.chain=hadoop"
-SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
-HADOOP_CREDSTORE_PASSWORD="credStorePass123"
-----
-====
-
-Windows::
-+
-====
-.solr.in.cmd
-[source,powershell]
-----
-set SOLR_OPTS=" -Dsolr.ssl.credential.provider.chain=hadoop"
-set SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
-set HADOOP_CREDSTORE_PASSWORD="credStorePass123"
-----
-====
-======
-
=== Configure ZooKeeper
After creating the keystore described above and before you start any SolrCloud nodes, you must configure your Solr cluster properties in ZooKeeper so that Solr nodes know to communicate via SSL.
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/replica-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/replica-management.adoc
index c66d8492e23..73c19a9db47 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/replica-management.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/replica-management.adoc
@@ -422,7 +422,6 @@ This parameter is required unless `shard` and `sourceNode` are specified, howeve
+
The number of seconds to wait for the replica to be live in the new location before deleting the replica in the old location.
Deletion will not occur and creation will not be rolled back in the event of a timeout, potentially leaving an extra replica.
-Presently, this parameter is ignored if the replica is an hdfs replica.
`inPlaceMove`::
+
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/shard-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/shard-management.adoc
index c2d0c050965..71bcff98911 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/shard-management.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/shard-management.adoc
@@ -278,7 +278,6 @@ If the status is anything other than "success", an error message will explain wh
When splitting a shard, a free disk space check is performed on the local file system of the leader shard.
This can be disabled through the `solr.shardSplit.checkDiskSpace.enabled` system property (i.e. `-Dsolr.shardSplit.checkDiskSpace.enabled=false`).
-It is already disabled by default for xref:solr-on-hdfs.adoc[HDFS].
[[createshard]]
== CREATESHARD: Create a Shard
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-on-hdfs.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-on-hdfs.adoc
deleted file mode 100644
index f99cff169ae..00000000000
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-on-hdfs.adoc
+++ /dev/null
@@ -1,290 +0,0 @@
-= Solr on HDFS
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-
-The Solr HDFS Module has support for writing and reading Solr's index and transaction log files to the HDFS distributed filesystem.
-It does not use Hadoop MapReduce to process Solr data.
-
-To use HDFS rather than a local filesystem, you must be using Hadoop 2.x and you will need to instruct Solr to use the `HdfsDirectoryFactory`.
-There are also several additional parameters to define.
-These can be set in one of three ways:
-
-* Pass JVM arguments to the `bin/solr` script.
-These would need to be passed every time you start Solr with `bin/solr`.
-* Modify `solr.in.sh` (or `solr.in.cmd` on Windows) to pass the JVM arguments automatically when using `bin/solr` without having to set them manually.
-* Define the properties in `solrconfig.xml`.
-These configuration changes would need to be repeated for every collection, so is a good option if you only want some of your collections stored in HDFS.
-
-== Module
-
-This is provided via the `hdfs` xref:configuration-guide:solr-modules.adoc[Solr Module] that needs to be enabled before use.
-
-== Starting Solr on HDFS
-
-=== SolrCloud Installations
-
-In SolrCloud mode, it's best to leave the data and update log directories as the defaults Solr comes with and simply specify the `solr.hdfs.home`.
-All dynamically created collections will create the appropriate directories automatically under the `solr.hdfs.home` root directory.
-
-* Set `solr.hdfs.home` in the form `hdfs://host:port/path`
-* You should specify a lock factory type of `'hdfs'` or none.
-
-[source,bash]
-----
-bin/solr start -Dsolr.directoryFactory=HdfsDirectoryFactory
- -Dsolr.lock.type=hdfs
- -Dsolr.hdfs.home=hdfs://host:port/path
-----
-
-This command starts Solr using the defined JVM properties.
-
-=== User-Managed Clusters and Single-Node Installations
-
-For For user-managed clusters or single-node Solr installations, there are a few parameters you should modify before starting Solr.
-These can be set in `solrconfig.xml` (more on that <<HdfsDirectoryFactory Parameters,below>>), or passed to the `bin/solr` script at startup.
-
-* You need to use an `HdfsDirectoryFactory` and a data directory in the form `hdfs://host:port/path`
-* You need to specify an `updateLog` location in the form `hdfs://host:port/path`
-* You should specify a lock factory type of `'hdfs'` or none.
-
-If you do not modify `solrconfig.xml`, you can instead start Solr on HDFS with the following command:
-
-[source,bash]
-----
-bin/solr start -Dsolr.directoryFactory=HdfsDirectoryFactory
- -Dsolr.lock.type=hdfs
- -Dsolr.data.dir=hdfs://host:port/path
- -Dsolr.updatelog=hdfs://host:port/path
-----
-
-This example will start Solr using the defined JVM properties (explained in more detail <<HdfsDirectoryFactory Parameters,below>>).
-
-=== Modifying solr.in.sh (*nix) or solr.in.cmd (Windows)
-
-The examples above assume you will pass JVM arguments as part of the start command every time you use `bin/solr` to start Solr.
-However, `bin/solr` looks for an include file named `solr.in.sh` (`solr.in.cmd` on Windows) to set environment variables.
-By default, this file is found in the `bin` directory, and you can modify it to permanently add the `HdfsDirectoryFactory` settings and ensure they are used every time Solr is started.
-
-For example, to set JVM arguments to always use HDFS when running in SolrCloud mode (as shown above), you would add a section such as this:
-
-[source,bash]
-----
-# Set HDFS DirectoryFactory & Settings
--Dsolr.directoryFactory=HdfsDirectoryFactory \
--Dsolr.lock.type=hdfs \
--Dsolr.hdfs.home=hdfs://host:port/path \
-----
-
-== The Block Cache
-
-For performance, the `HdfsDirectoryFactory` uses a Directory that will cache HDFS blocks.
-This caching mechanism replaces the standard file system cache that Solr utilizes.
-By default, this cache is allocated off-heap.
-This cache will often need to be quite large and you may need to raise the off-heap memory limit for the specific JVM you are running Solr in.
-For the Oracle/OpenJDK JVMs, the following is an example command-line parameter that you can use to raise the limit when starting Solr:
-
-[source,bash]
-----
--XX:MaxDirectMemorySize=20g
-----
-
-== HdfsDirectoryFactory Parameters
-
-The `HdfsDirectoryFactory` has a number of settings defined as part of the `directoryFactory` configuration.
-
-=== Solr HDFS Settings
-
-`solr.hdfs.home`::
-+
-[%autowidth,frame=none]
-|===
-s|Required |Default: none
-|===
-+
-A root location in HDFS for Solr to write collection data to.
-Rather than specifying an HDFS location for the data directory or update log directory, use this to specify one root location and have everything automatically created within this HDFS location.
-The structure of this parameter is `hdfs://host:port/path/solr`.
-
-=== Block Cache Settings
-
-`solr.hdfs.blockcache.enabled`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `true`
-|===
-+
-Enable the blockcache.
-
-`solr.hdfs.blockcache.read.enabled`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `true`
-|===
-+
-Enable the read cache.
-
-`solr.hdfs.blockcache.direct.memory.allocation`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `true`
-|===
-+
-Enable direct memory allocation.
-If this is `false`, heap is used.
-
-`solr.hdfs.blockcache.slab.count`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `1`
-|===
-+
-Number of memory slabs to allocate.
-Each slab is 128 MB in size.
-
-`solr.hdfs.blockcache.global`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `true`
-|===
-+
-Enable/Disable using one global cache for all SolrCores.
-The settings used will be from the first HdfsDirectoryFactory created.
-
-=== NRTCachingDirectory Settings
-
-`solr.hdfs.nrtcachingdirectory.enable`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `true`
-|===
-+
-Enable the use of NRTCachingDirectory.
-
-`solr.hdfs.nrtcachingdirectory.maxmergesizemb`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `16`
-|===
-+
-NRTCachingDirectory max segment size for merges.
-
-`solr.hdfs.nrtcachingdirectory.maxcachedmb`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `192`
-|===
-+
-NRTCachingDirectory max cache size.
-
-=== HDFS Client Configuration Settings
-
-`solr.hdfs.confdir`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: none
-|===
-+
-Pass the location of HDFS client configuration files - needed for HDFS HA for example.
-
-=== Kerberos Authentication Settings
-
-Hadoop can be configured to use the Kerberos protocol to verify user identity when trying to access core services like HDFS.
-If your HDFS directories are protected using Kerberos, then you need to configure Solr's HdfsDirectoryFactory to authenticate using Kerberos in order to read and write to HDFS.
-To enable Kerberos authentication from Solr, you need to set the following parameters:
-
-`solr.hdfs.security.kerberos.enabled`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `false`
-|===
-+
-Set to `true` to enable Kerberos authentication.
-
-`solr.hdfs.security.kerberos.keytabfile`::
-+
-[%autowidth,frame=none]
-|===
-s|Required |Default: none
-|===
-+
-A keytab file contains pairs of Kerberos principals and encrypted keys which allows for password-less authentication when Solr attempts to authenticate with secure Hadoop.
-+
-This file will need to be present on all Solr servers at the same path provided in this parameter.
-
-`solr.hdfs.security.kerberos.principal`::
-+
-[%autowidth,frame=none]
-|===
-s|Required |Default: none
-|===
-+
-The Kerberos principal that Solr should use to authenticate to secure Hadoop; the format of a typical Kerberos V5 principal is: `primary/instance@realm`.
-
-== Update Log settings
-When using HDFS to store Solr indexes, it is recommended to also store the transaction logs on HDFS. This can be done by using the `solr.HdfsUpdateLog` update log handler class.
-The solrconfig.xml is often used to define an update log handler class name either using a variable reference or direct specification, for example:
-
-[source,xml]
-----
-<updateLog class="${solr.ulog:solr.UpdateLog}">
-----
-
-When specifying a class like this, it needs to be ensured that the correct class name is specified.
-When no class name is specified, Solr automatically picks the correct update log handler class `solr.HdfsUpdateLog` for collections which are configured to use the HdfsDirectory Factory.
-
-
-== Example solrconfig.xml for HDFS
-
-Here is a sample `solrconfig.xml` configuration for storing Solr indexes on HDFS:
-
-[source,xml]
-----
-<directoryFactory name="DirectoryFactory" class="solr.HdfsDirectoryFactory">
- <str name="solr.hdfs.home">hdfs://host:port/solr</str>
- <bool name="solr.hdfs.blockcache.enabled">true</bool>
- <int name="solr.hdfs.blockcache.slab.count">1</int>
- <bool name="solr.hdfs.blockcache.direct.memory.allocation">true</bool>
- <int name="solr.hdfs.blockcache.blocksperbank">16384</int>
- <bool name="solr.hdfs.blockcache.read.enabled">true</bool>
- <bool name="solr.hdfs.nrtcachingdirectory.enable">true</bool>
- <int name="solr.hdfs.nrtcachingdirectory.maxmergesizemb">16</int>
- <int name="solr.hdfs.nrtcachingdirectory.maxcachedmb">192</int>
-</directoryFactory>
-----
-
-If using Kerberos, you will need to add the three Kerberos related properties to the `<directoryFactory>` element in `solrconfig.xml`, such as:
-
-[source,xml]
-----
-<directoryFactory name="DirectoryFactory" class="solr.HdfsDirectoryFactory">
- ...
- <bool name="solr.hdfs.security.kerberos.enabled">true</bool>
- <str name="solr.hdfs.security.kerberos.keytabfile">/etc/krb5.keytab</str>
- <str name="solr.hdfs.security.kerberos.principal">solr/[email protected]</str>
-</directoryFactory>
-----
diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc
index 09beabda54a..650b89b110c 100644
--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc
+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc
@@ -99,3 +99,5 @@ Users that need to vary JAR accessibility on a per-core basis can use Solr's xre
Users who that don't need to vary JAR access on a per-core basis have several options, including the xref:configuration-guide:configuring-solr-xml.adoc[`<sharedLib/>` tag supported by solr.xml] or manipulation of Solr's classpath prior to JVM startup.
* Kerberos based authentication has been removed. This results in changes to SolrJ, the Solr Admin app, and the removal of the `hadoop-auth` module.
+
+* Storing indexes and snapshots in HDFS has been removed. This results in changes to solrconfig.xml and related configuration files and removal of the `hdfs` module.
diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
index 0da5f969ade..e84a88e9184 100644
--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
@@ -395,9 +395,9 @@ If you plan on using SolrJ as a JDBC driver, please refer to the xref:query-guid
* `SolrMetricProducer` / `SolrInfoBean` APIs have changed and third-party components that implement these APIs need to be updated.
* Use of blacklist/whitelist terminology has been completely removed. JWTAuthPlugin parameter `algWhitelist` is now `algAllowlist`. The old parameter will still work in 9.x. Environment variables `SOLR_IP_WHITELIST` and `SOLR_IP_BLACKLIST` are no longer supported, but replaced with `SOLR_IP_ALLOWLIST` and `SOLR_IP_DENYLIST`.
* Solr Backups - Async responses for backups now correctly aggregate and return information. For collection's snapshot backup request responses additional fields `indexVersion`, `indexFileCount`, etc. were added similar to incremental backup request responses.
-* If you are using the HDFS backup repository, you need to change the repository class to `org.apache.solr.hdfs.backup.repository.HdfsBackupRepository` - see the xref:deployment-guide:backup-restore.adoc#hdfsbackuprepository[HDFS Backup Repository] section.
+* If you are using the HDFS backup repository, you need to change the repository class to `org.apache.solr.hdfs.backup.repository.HdfsBackupRepository` - see the xref:deployment-guide:backup-restore.adoc[Backup and Restore] page for the "HDFS Backup Repository" section in a pre Solr 10 Ref Guide.
* HDFS storage support has been moved to a module. Existing Solr configurations do not need any HDFS-related
-changes, however the module needs to be installed - see the section xref:deployment-guide:solr-on-hdfs.adoc[].
+changes, however the module needs to be installed - see the section Solr on HDFS in a pre Solr 10 Ref Guide.
* The folder `$SOLR_HOME/userfiles`, used by the "cat" streaming expression, is no longer created automatically on startup. The user must create this folder.
* Solr no longer requires a `solr.xml` in `$SOLR_HOME`. If one is not found, Solr will instead use the default one from `$SOLR_TIP/server/solr/solr.xml`. You can revert to the pre-9.0 behaviour by setting environment variable `SOLR_SOLRXML_REQUIRED=true` or system property `-Dsolr.solrxml.required=true`. Solr also does not require a `zoo.cfg` in `$SOLR_HOME` if started with embedded zookeeper.
* `base_url` has been removed from stored cluster state. If you're able to upgrade SolrJ to 8.8.x for all of your client applications, then you can set `-Dsolr.storeBaseUrl=false` (introduced in Solr 8.8.1) to better align the stored state in Zookeeper with future versions of Solr; as of Solr 9.x, the `base_url` will no longer be persisted in stored state.
diff --git a/solr/solrj-streaming/src/test-files/log4j2.xml b/solr/solrj-streaming/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/solrj-streaming/src/test-files/log4j2.xml
+++ b/solr/solrj-streaming/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/solrj/src/resources/EnvToSyspropMappings.properties b/solr/solrj/src/resources/EnvToSyspropMappings.properties
index fb51bf34028..dcfa553961a 100644
--- a/solr/solrj/src/resources/EnvToSyspropMappings.properties
+++ b/solr/solrj/src/resources/EnvToSyspropMappings.properties
@@ -15,7 +15,6 @@ SOLR_DISABLE_REQUEST_ID=solr.disableRequestId
SOLR_ENABLE_PACKAGES=enable.packages
SOLR_ENABLE_REMOTE_STREAMING=solr.enableRemoteStreaming
SOLR_ENABLE_STREAM_BODY=solr.enableStreamBody
-SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=hadoop.security.credential.provider.path
SOLR_HIDDEN_SYS_PROPS=solr.hiddenSysProps
SOLR_HOME=solr.solr.home
SOLR_HOST=host
diff --git a/solr/solrj/src/test-files/log4j2.xml b/solr/solrj/src/test-files/log4j2.xml
index 96f69f1dc8b..528299e3e0b 100644
--- a/solr/solrj/src/test-files/log4j2.xml
+++ b/solr/solrj/src/test-files/log4j2.xml
@@ -30,9 +30,7 @@
<Loggers>
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
<Logger name="org.apache.zookeeper" level="WARN"/>
- <Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
- <Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java b/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java
index 505e276e170..d86de3bab64 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java
@@ -52,8 +52,7 @@ public abstract class SolrJettyTestBase extends SolrTestCaseJ4 {
Properties nodeProps = new Properties();
if (configFile != null) nodeProps.setProperty("solrconfig", configFile);
if (schemaFile != null) nodeProps.setProperty("schema", schemaFile);
- if (System.getProperty("solr.data.dir") == null
- && System.getProperty("solr.hdfs.home") == null) {
+ if (System.getProperty("solr.data.dir") == null) {
nodeProps.setProperty("solr.data.dir", createTempDir().toFile().getCanonicalPath());
}
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index e08afa4405e..d8a9ac37ed6 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -685,8 +685,7 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
}
/**
- * The directory used as the <code>dataDir</code> for the TestHarness unless {@link #hdfsDataDir}
- * is non null.
+ * The directory used as the <code>dataDir</code> for the TestHarness.
*
* <p>Will be set to null by {@link #deleteCore} and re-initialized as needed by {@link
* #createCore}. In the event of a test failure, the contents will be left on disk.
@@ -697,9 +696,6 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
*/
@Deprecated protected static volatile File initCoreDataDir;
- // hack due to File dataDir
- protected static String hdfsDataDir;
-
/**
* Initializes things your test might need
*
@@ -732,10 +728,7 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
solrConfig = TestHarness.createConfig(testSolrHome, coreName, getSolrConfigFile());
h =
new TestHarness(
- coreName,
- hdfsDataDir == null ? initAndGetDataDir().getAbsolutePath() : hdfsDataDir,
- solrConfig,
- getSchemaFile());
+ coreName, initAndGetDataDir().getAbsolutePath(), solrConfig, getSchemaFile());
lrf = h.getRequestFactory("", 0, 20, CommonParams.VERSION, "2.2");
}
@@ -844,7 +837,6 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
lrf = null;
configString = schemaString = null;
initCoreDataDir = null;
- hdfsDataDir = null;
}
/** Validates an update XML String is successful */
diff --git a/solr/test-framework/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/test-framework/src/test-files/solr/collection1/conf/solrconfig.xml
index ea95918291c..f50b7fb7b6a 100644
--- a/solr/test-framework/src/test-files/solr/collection1/conf/solrconfig.xml
+++ b/solr/test-framework/src/test-files/solr/collection1/conf/solrconfig.xml
@@ -43,12 +43,6 @@
<double name="maxWriteMBPerSecFlush">2000000</double>
<double name="maxWriteMBPerSecMerge">3000000</double>
<double name="maxWriteMBPerSecRead">4000000</double>
- <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
- <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
- <bool name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</bool>
- <bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
- <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:10}</int>
- <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
</directoryFactory>
<schemaFactory class="ClassicIndexSchemaFactory"/>