Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15405: Make HDFS handle little endian LUCENE-9047

Kevin Risden64c02c05667a
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 336ca023a57..e15150344d9 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -599,6 +599,8 @@ Bug Fixes
* SOLR-15324: Upgrade jaegertracing to 1.6.0 and libthrift to 0.14.1 (wcmrnd1, janhoy)
+* SOLR-15405: Make HDFS handle little endian LUCENE-9047 (Kevin Risden)
+
================== 8.11.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java b/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
index c138c073a6c..560be653dff 100644
--- a/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
+++ b/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
@@ -38,7 +38,7 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
private int bufferLength = 0; // end of valid bytes
private int bufferPosition = 0; // next byte to read
- private Store store;
+ private final Store store;
@Override
public byte readByte() throws IOException {
@@ -116,14 +116,26 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
}
}
}
-
+
+ @Override
+ public short readShort() throws IOException {
+ if (2 <= (bufferLength - bufferPosition)) {
+ final byte b1 = buffer[bufferPosition++];
+ final byte b2 = buffer[bufferPosition++];
+ return (short) ((b2 & 0xFF) << 8 | (b1 & 0xFF));
+ } else {
+ return super.readShort();
+ }
+ }
+
@Override
public int readInt() throws IOException {
if (4 <= (bufferLength - bufferPosition)) {
- return ((buffer[bufferPosition++] & 0xFF) << 24)
- | ((buffer[bufferPosition++] & 0xFF) << 16)
- | ((buffer[bufferPosition++] & 0xFF) << 8)
- | (buffer[bufferPosition++] & 0xFF);
+ final byte b1 = buffer[bufferPosition++];
+ final byte b2 = buffer[bufferPosition++];
+ final byte b3 = buffer[bufferPosition++];
+ final byte b4 = buffer[bufferPosition++];
+ return (b4 & 0xFF) << 24 | (b3 & 0xFF) << 16 | (b2 & 0xFF) << 8 | (b1 & 0xFF);
} else {
return super.readInt();
}
@@ -132,15 +144,7 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
@Override
public long readLong() throws IOException {
if (8 <= (bufferLength - bufferPosition)) {
- final int i1 = ((buffer[bufferPosition++] & 0xff) << 24)
- | ((buffer[bufferPosition++] & 0xff) << 16)
- | ((buffer[bufferPosition++] & 0xff) << 8)
- | (buffer[bufferPosition++] & 0xff);
- final int i2 = ((buffer[bufferPosition++] & 0xff) << 24)
- | ((buffer[bufferPosition++] & 0xff) << 16)
- | ((buffer[bufferPosition++] & 0xff) << 8)
- | (buffer[bufferPosition++] & 0xff);
- return (((long) i1) << 32) | (i2 & 0xFFFFFFFFL);
+ return (readInt() & 0xFFFFFFFFL) | (((long) readInt()) << 32);
} else {
return super.readLong();
}
@@ -150,12 +154,22 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
public int readVInt() throws IOException {
if (5 <= (bufferLength - bufferPosition)) {
byte b = buffer[bufferPosition++];
+ if (b >= 0) return b;
int i = b & 0x7F;
- for (int shift = 7; (b & 0x80) != 0; shift += 7) {
- b = buffer[bufferPosition++];
- i |= (b & 0x7F) << shift;
- }
- return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7F) << 7;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7F) << 14;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7F) << 21;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ // Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:
+ i |= (b & 0x0F) << 28;
+ if ((b & 0xF0) == 0) return i;
+ throw new RuntimeException("Invalid vInt detected (too many bits)");
} else {
return super.readVInt();
}
@@ -164,13 +178,34 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
@Override
public long readVLong() throws IOException {
if (9 <= bufferLength - bufferPosition) {
- byte b = buffer[bufferPosition++];
- long i = b & 0x7F;
- for (int shift = 7; (b & 0x80) != 0; shift += 7) {
- b = buffer[bufferPosition++];
- i |= (b & 0x7FL) << shift;
- }
- return i;
+ byte b =buffer[bufferPosition++];
+ if (b >= 0) return b;
+ long i = b & 0x7FL;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 7;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 14;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 21;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 28;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 35;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 42;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 49;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 56;
+ if (b >= 0) return i;
+ throw new RuntimeException("Invalid vLong detected (negative values disallowed)");
} else {
return super.readVLong();
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
index b5a387e3053..83bd930999e 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
@@ -20,7 +20,6 @@ package org.apache.solr.cloud.hdfs;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.client.solrj.SolrQuery;
@@ -39,7 +38,6 @@ import org.junit.BeforeClass;
QuickPatchThreadsFilter.class,
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
[email protected](bugUrl = "SOLR-15405")
@ThreadLeakLingering(linger = 10)
public class HDFSCollectionsAPITest extends SolrCloudTestCase {
diff --git a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
index 3b08babd3b9..28e82ca6edf 100644
--- a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
+++ b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.index.BaseTestCheckIndex;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -50,8 +49,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
@SolrTestCaseJ4.SuppressSSL
[email protected](bugUrl = "SOLR-15405")
-// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
private static MiniDFSCluster dfsCluster;
private static Path path;
diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
index 17a1d015a00..9e57717d493 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -69,7 +68,6 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactor
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
// TODO: longer term this should be combined with TestRecovery somehow ??
[email protected](bugUrl = "SOLR-15405")
public class TestRecoveryHdfs extends SolrTestCaseJ4 {
// means that we've seen the leader and have version info (i.e. we are a non-leader replica)
private static final String FROM_LEADER = DistribPhase.FROMLEADER.toString();
diff --git a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
index 2e65db5d019..59f8fa93db8 100644
--- a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
+++ b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
@@ -23,7 +23,6 @@ import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -42,7 +41,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
QuickPatchThreadsFilter.class,
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
[email protected](bugUrl = "SOLR-15405")
public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
private static MiniDFSCluster dfsCluster;
private static String hdfsUri;