Apache Solr open-source search software
SOLR-15405 AwaitsFix enough HDFS tests to get 3 consecutive runs with no hdfs failures. (#127)
Gus Heck3ed47d7cdfea
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.javaindex d88b1d1fbf4..9386b9153ad 100644--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java@@ -19,6 +19,7 @@ package org.apache.solr.cloud.hdfs;import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;import org.apache.hadoop.hdfs.MiniDFSCluster;+import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.QuickPatchThreadsFilter;import org.apache.solr.SolrIgnoredThreadsFilter;import org.apache.solr.client.solrj.SolrQuery;@@ -37,6 +38,7 @@ import org.junit.BeforeClass;QuickPatchThreadsFilter.class,BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)})[email protected](bugUrl = "SOLR-15405")public class HDFSCollectionsAPITest extends SolrCloudTestCase {private static MiniDFSCluster dfsCluster;diff --git a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.javaindex 7285d346e62..3b08babd3b9 100644--- a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java+++ b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path;import org.apache.hadoop.hdfs.MiniDFSCluster;import org.apache.lucene.index.BaseTestCheckIndex;import org.apache.lucene.store.Directory;+import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.QuickPatchThreadsFilter;import org.apache.solr.SolrIgnoredThreadsFilter;import org.apache.solr.SolrTestCaseJ4;@@ -49,6 +50,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)})@SolrTestCaseJ4.SuppressSSL[email protected](bugUrl = "SOLR-15405")// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {private static MiniDFSCluster dfsCluster;diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.javaindex eb4592c8daf..17a1d015a00 100644--- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java+++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.hdfs.MiniDFSCluster;+import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.QuickPatchThreadsFilter;import org.apache.solr.SolrIgnoredThreadsFilter;import org.apache.solr.SolrTestCaseJ4;@@ -68,6 +69,7 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactorBadHdfsThreadsFilter.class // hdfs currently leaks thread(s)})// TODO: longer term this should be combined with TestRecovery somehow ??[email protected](bugUrl = "SOLR-15405")public class TestRecoveryHdfs extends SolrTestCaseJ4 {// means that we've seen the leader and have version info (i.e. we are a non-leader replica)private static final String FROM_LEADER = DistribPhase.FROMLEADER.toString();@@ -78,17 +80,17 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {private static MiniDFSCluster dfsCluster;private static String hdfsUri;private static FileSystem fs;-+@Afterpublic void afterTest() {TestInjection.reset(); // do after every test, don't wait for AfterClass}-+@BeforeClasspublic static void beforeClass() throws Exception {dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());hdfsUri = HdfsTestUtil.getURI(dfsCluster);-+try {URI uri = new URI(hdfsUri);Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);@@ -98,10 +100,10 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {}System.setProperty("solr.ulog.dir", hdfsUri + "/solr/shard1");-+initCore("solrconfig-tlog.xml","schema15.xml");}-+@AfterClasspublic static void afterClass() throws Exception {IOUtils.closeQuietly(fs);@@ -123,10 +125,10 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {@Testpublic void testReplicationFactor() throws Exception {- clearIndex();-+ clearIndex();+HdfsUpdateLog ulog = (HdfsUpdateLog) h.getCore().getUpdateHandler().getUpdateLog();-+assertU(commit());addAndGetVersion(sdoc("id", "REP1"), null);assertU(commit());@@ -140,10 +142,10 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {break;}}-+assertTrue("Expected to find tlogs with a replication factor of 2", foundRep2);}-+@Testpublic void testLogReplay() throws Exception {try {@@ -650,7 +652,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {req().close();}}-+private void addDocs(int nDocs, int start, LinkedList<Long> versions) throws Exception {for (int i=0; i<nDocs; i++) {versions.addFirst( addAndGetVersion( sdoc("id",Integer.toString(start + nDocs)) , null) );@@ -679,7 +681,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {assertU(commit());String logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();-+h.close();String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));@@ -788,16 +790,16 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {assertU(adoc("id","F1"));assertU(adoc("id","F2"));assertU(adoc("id","F3"));-+h.close();--++String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));Arrays.sort(files);FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length-1]));-+dos.writeLong(0xffffffffffffffffL);dos.writeChars("This should be appended to a good log file, representing a bad partially written record.");dos.close();@@ -837,7 +839,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {TestInjection.skipIndexWriterCommitOnClose = true;String logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();-+clearIndex();assertU(commit());@@ -970,7 +972,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {System.arraycopy(to, 0, data, idx, to.length);}}-+private static int indexOf(byte[] target, byte[] data, int start) {outer: for (int i=start; i<data.length - target.length; i++) {for (int j=0; j<target.length; j++) {diff --git a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.javaindex 2ac5051544b..2e65db5d019 100644--- a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java+++ b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java@@ -23,6 +23,7 @@ import java.net.URISyntaxException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.hdfs.MiniDFSCluster;+import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.QuickPatchThreadsFilter;import org.apache.solr.SolrIgnoredThreadsFilter;import org.apache.solr.SolrTestCaseJ4;@@ -41,16 +42,17 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;QuickPatchThreadsFilter.class,BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)})[email protected](bugUrl = "SOLR-15405")public class TestHdfsUpdateLog extends SolrTestCaseJ4 {private static MiniDFSCluster dfsCluster;private static String hdfsUri;private static FileSystem fs;-+@BeforeClasspublic static void beforeClass() throws Exception {dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());hdfsUri = HdfsTestUtil.getURI(dfsCluster);-+try {URI uri = new URI(hdfsUri);Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);@@ -60,10 +62,10 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {}System.setProperty("solr.ulog.dir", hdfsUri + "/solr/shard1");-+initCore("solrconfig-tlog.xml","schema15.xml");}-+@AfterClasspublic static void afterClass() throws Exception {IOUtils.closeQuietly(fs);@@ -90,14 +92,14 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {((DirectUpdateHandler2) uhandler).getCommitTracker().setTimeUpperBound(100);((DirectUpdateHandler2) uhandler).getCommitTracker().setOpenSearcher(false);final UpdateLog ulog = uhandler.getUpdateLog();-+clearIndex();assertU(commit());-+// we hammer on init in a background thread to make// sure we don't run into any filesystem already closed// problems (SOLR-7113)-+Thread thread = new Thread() {public void run() {int cnt = 0;@@ -114,7 +116,7 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {}}};-+Thread thread2 = new Thread() {public void run() {int cnt = 0;