Apache Solr open-source search software
SOLR-15405: Make HDFS handle little endian LUCENE-9047
Kevin Risden64c02c05667a
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 336ca023a57..e15150344d9 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -599,6 +599,8 @@ Bug Fixes* SOLR-15324: Upgrade jaegertracing to 1.6.0 and libthrift to 0.14.1 (wcmrnd1, janhoy)+* SOLR-15405: Make HDFS handle little endian LUCENE-9047 (Kevin Risden)+================== 8.11.0 ==================Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java b/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.javaindex c138c073a6c..560be653dff 100644--- a/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java+++ b/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java@@ -38,7 +38,7 @@ public abstract class CustomBufferedIndexInput extends IndexInput {private int bufferLength = 0; // end of valid bytesprivate int bufferPosition = 0; // next byte to read- private Store store;+ private final Store store;@Overridepublic byte readByte() throws IOException {@@ -116,14 +116,26 @@ public abstract class CustomBufferedIndexInput extends IndexInput {}}}-++ @Override+ public short readShort() throws IOException {+ if (2 <= (bufferLength - bufferPosition)) {+ final byte b1 = buffer[bufferPosition++];+ final byte b2 = buffer[bufferPosition++];+ return (short) ((b2 & 0xFF) << 8 | (b1 & 0xFF));+ } else {+ return super.readShort();+ }+ }+@Overridepublic int readInt() throws IOException {if (4 <= (bufferLength - bufferPosition)) {- return ((buffer[bufferPosition++] & 0xFF) << 24)- | ((buffer[bufferPosition++] & 0xFF) << 16)- | ((buffer[bufferPosition++] & 0xFF) << 8)- | (buffer[bufferPosition++] & 0xFF);+ final byte b1 = buffer[bufferPosition++];+ final byte b2 = buffer[bufferPosition++];+ final byte b3 = buffer[bufferPosition++];+ final byte b4 = buffer[bufferPosition++];+ return (b4 & 0xFF) << 24 | (b3 & 0xFF) << 16 | (b2 & 0xFF) << 8 | (b1 & 0xFF);} else {return super.readInt();}@@ -132,15 +144,7 @@ public abstract class CustomBufferedIndexInput extends IndexInput {@Overridepublic long readLong() throws IOException {if (8 <= (bufferLength - bufferPosition)) {- final int i1 = ((buffer[bufferPosition++] & 0xff) << 24)- | ((buffer[bufferPosition++] & 0xff) << 16)- | ((buffer[bufferPosition++] & 0xff) << 8)- | (buffer[bufferPosition++] & 0xff);- final int i2 = ((buffer[bufferPosition++] & 0xff) << 24)- | ((buffer[bufferPosition++] & 0xff) << 16)- | ((buffer[bufferPosition++] & 0xff) << 8)- | (buffer[bufferPosition++] & 0xff);- return (((long) i1) << 32) | (i2 & 0xFFFFFFFFL);+ return (readInt() & 0xFFFFFFFFL) | (((long) readInt()) << 32);} else {return super.readLong();}@@ -150,12 +154,22 @@ public abstract class CustomBufferedIndexInput extends IndexInput {public int readVInt() throws IOException {if (5 <= (bufferLength - bufferPosition)) {byte b = buffer[bufferPosition++];+ if (b >= 0) return b;int i = b & 0x7F;- for (int shift = 7; (b & 0x80) != 0; shift += 7) {- b = buffer[bufferPosition++];- i |= (b & 0x7F) << shift;- }- return i;+ b = buffer[bufferPosition++];+ i |= (b & 0x7F) << 7;+ if (b >= 0) return i;+ b = buffer[bufferPosition++];+ i |= (b & 0x7F) << 14;+ if (b >= 0) return i;+ b = buffer[bufferPosition++];+ i |= (b & 0x7F) << 21;+ if (b >= 0) return i;+ b = buffer[bufferPosition++];+ // Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:+ i |= (b & 0x0F) << 28;+ if ((b & 0xF0) == 0) return i;+ throw new RuntimeException("Invalid vInt detected (too many bits)");} else {return super.readVInt();}@@ -164,13 +178,34 @@ public abstract class CustomBufferedIndexInput extends IndexInput {@Overridepublic long readVLong() throws IOException {if (9 <= bufferLength - bufferPosition) {- byte b = buffer[bufferPosition++];- long i = b & 0x7F;- for (int shift = 7; (b & 0x80) != 0; shift += 7) {- b = buffer[bufferPosition++];- i |= (b & 0x7FL) << shift;- }- return i;+ byte b =buffer[bufferPosition++];+ if (b >= 0) return b;+ long i = b & 0x7FL;+ b = buffer[bufferPosition++];+ i |= (b & 0x7FL) << 7;+ if (b >= 0) return i;+ b = buffer[bufferPosition++];+ i |= (b & 0x7FL) << 14;+ if (b >= 0) return i;+ b = buffer[bufferPosition++];+ i |= (b & 0x7FL) << 21;+ if (b >= 0) return i;+ b = buffer[bufferPosition++];+ i |= (b & 0x7FL) << 28;+ if (b >= 0) return i;+ b = buffer[bufferPosition++];+ i |= (b & 0x7FL) << 35;+ if (b >= 0) return i;+ b = buffer[bufferPosition++];+ i |= (b & 0x7FL) << 42;+ if (b >= 0) return i;+ b = buffer[bufferPosition++];+ i |= (b & 0x7FL) << 49;+ if (b >= 0) return i;+ b = buffer[bufferPosition++];+ i |= (b & 0x7FL) << 56;+ if (b >= 0) return i;+ throw new RuntimeException("Invalid vLong detected (negative values disallowed)");} else {return super.readVLong();}diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.javaindex b5a387e3053..83bd930999e 100644--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java@@ -20,7 +20,6 @@ package org.apache.solr.cloud.hdfs;import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;import org.apache.hadoop.hdfs.MiniDFSCluster;-import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.QuickPatchThreadsFilter;import org.apache.solr.SolrIgnoredThreadsFilter;import org.apache.solr.client.solrj.SolrQuery;@@ -39,7 +38,6 @@ import org.junit.BeforeClass;QuickPatchThreadsFilter.class,BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)})[email protected](bugUrl = "SOLR-15405")@ThreadLeakLingering(linger = 10)public class HDFSCollectionsAPITest extends SolrCloudTestCase {diff --git a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.javaindex 3b08babd3b9..28e82ca6edf 100644--- a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java+++ b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.Path;import org.apache.hadoop.hdfs.MiniDFSCluster;import org.apache.lucene.index.BaseTestCheckIndex;import org.apache.lucene.store.Directory;-import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.QuickPatchThreadsFilter;import org.apache.solr.SolrIgnoredThreadsFilter;import org.apache.solr.SolrTestCaseJ4;@@ -50,8 +49,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)})@SolrTestCaseJ4.SuppressSSL[email protected](bugUrl = "SOLR-15405")-// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {private static MiniDFSCluster dfsCluster;private static Path path;diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.javaindex 17a1d015a00..9e57717d493 100644--- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java+++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java@@ -39,7 +39,6 @@ import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.hdfs.MiniDFSCluster;-import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.QuickPatchThreadsFilter;import org.apache.solr.SolrIgnoredThreadsFilter;import org.apache.solr.SolrTestCaseJ4;@@ -69,7 +68,6 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactorBadHdfsThreadsFilter.class // hdfs currently leaks thread(s)})// TODO: longer term this should be combined with TestRecovery somehow ??[email protected](bugUrl = "SOLR-15405")public class TestRecoveryHdfs extends SolrTestCaseJ4 {// means that we've seen the leader and have version info (i.e. we are a non-leader replica)private static final String FROM_LEADER = DistribPhase.FROMLEADER.toString();diff --git a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.javaindex 2e65db5d019..59f8fa93db8 100644--- a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java+++ b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java@@ -23,7 +23,6 @@ import java.net.URISyntaxException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.hdfs.MiniDFSCluster;-import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.QuickPatchThreadsFilter;import org.apache.solr.SolrIgnoredThreadsFilter;import org.apache.solr.SolrTestCaseJ4;@@ -42,7 +41,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;QuickPatchThreadsFilter.class,BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)})[email protected](bugUrl = "SOLR-15405")public class TestHdfsUpdateLog extends SolrTestCaseJ4 {private static MiniDFSCluster dfsCluster;private static String hdfsUri;