Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15244 Failing tests

Mike Drobab6e695973ca
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
index b6e2f43d0fe..6d5f9be98c7 100644
--- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
@@ -52,7 +52,6 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.common.annotation.SolrThreadUnsafe;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.filestore.PackageStoreAPI.MetaData;
-import org.apache.solr.util.SimplePostTool;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.server.ByteBufferInputStream;
@@ -95,7 +94,8 @@ public class DistribPackageStore implements PackageStore {
if (!path.isEmpty() && path.charAt(0) != File.separatorChar) {
path = File.separator + path;
}
- return solrHome.resolve(PackageStoreAPI.PACKAGESTORE_DIRECTORY).resolve(path);
+ // Use concat because path might start with a slash and be incorrectly interpreted as absolute
+ return solrHome.resolve(PackageStoreAPI.PACKAGESTORE_DIRECTORY + path);
}
class FileInfo {
@@ -196,11 +196,13 @@ public class DistribPackageStore implements PackageStore {
ByteBuffer filedata = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
baseUrl + "/node/files" + path,
Utils.newBytesConsumer((int) MAX_PKG_SIZE));
+ filedata.mark();
String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(filedata));
String expected = (String) m.get("sha512");
if (!sha512.equals(expected)) {
throw new SolrException(SERVER_ERROR, "sha512 mismatch downloading : " + path + " from node : " + fromNode);
}
+ filedata.reset();
persistToFile(filedata, metadata);
return true;
} catch (SolrException e) {
@@ -589,12 +591,12 @@ public class DistribPackageStore implements PackageStore {
}
Path metadataPath = _getRealPath(_getMetapath(path), solrHome);
- try (SeekableByteChannel channel = Files.newByteChannel(metadataPath, StandardOpenOption.WRITE)) {
+ try (SeekableByteChannel channel = Files.newByteChannel(metadataPath, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) {
channel.write(meta);
}
IOUtils.fsync(metadataPath, false);
- try (SeekableByteChannel channel = Files.newByteChannel(realpath, StandardOpenOption.WRITE)) {
+ try (SeekableByteChannel channel = Files.newByteChannel(realpath, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) {
channel.write(data);
}
IOUtils.fsync(realpath, false);
@@ -615,9 +617,7 @@ public class DistribPackageStore implements PackageStore {
if (keyFiles == null) return result;
for (File keyFile : keyFiles) {
if (keyFile.isFile() && !isMetaDataFile(keyFile.getName())) {
- try (InputStream fis = new FileInputStream(keyFile)) {
- result.put(keyFile.getName(), SimplePostTool.inputStreamToByteArray(fis).array());
- }
+ result.put(keyFile.getName(), Files.readAllBytes(keyFile.toPath()));
}
}
return result;
diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
index db2e05ebdcd..6a10315ce66 100644
--- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
+++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
@@ -199,14 +199,14 @@ public class PackageStoreAPI {
validateName(path, true);
ContentStream stream = streams.iterator().next();
try {
- ByteBuffer buf = SimplePostTool.inputStreamToByteArray(stream.getStream());
+ byte[] buf = stream.getStream().readAllBytes();
List<String> signatures = readSignatures(req, buf);
MetaData meta = _createJsonMetaData(buf, signatures);
PackageStore.FileType type = packageStore.getType(path, true);
if(type != PackageStore.FileType.NOFILE) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path already exists "+ path);
}
- packageStore.put(new PackageStore.FileEntry(buf, meta, path));
+ packageStore.put(new PackageStore.FileEntry(ByteBuffer.wrap(buf), meta, path));
rsp.add(CommonParams.FILE, path);
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
@@ -226,7 +226,7 @@ public class PackageStoreAPI {
}
}
- private List<String> readSignatures(SolrQueryRequest req, ByteBuffer buf)
+ private List<String> readSignatures(SolrQueryRequest req, byte[] buf)
throws SolrException, IOException {
String[] signatures = req.getParams().getParams("sig");
if (signatures == null || signatures.length == 0) return null;
@@ -237,7 +237,7 @@ public class PackageStoreAPI {
}
private void validate(List<String> sigs,
- ByteBuffer buf) throws SolrException, IOException {
+ byte[] buf) throws SolrException, IOException {
Map<String, byte[]> keys = packageStore.getKeys();
if (keys == null || keys.isEmpty()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
@@ -251,9 +251,9 @@ public class PackageStoreAPI {
"Error parsing public keys in Package store");
}
for (String sig : sigs) {
- if (cryptoKeys.verify(sig, buf) == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig +" len: "+buf.limit()+ " content sha512: "+
- DigestUtils.sha512Hex(new ByteBufferInputStream(buf)));
+ if (cryptoKeys.verify(sig, ByteBuffer.wrap(buf)) == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig +" len: "+buf.length+ " content sha512: "+
+ DigestUtils.sha512Hex(buf));
}
}
@@ -262,11 +262,11 @@ public class PackageStoreAPI {
}
/**
- * Creates a JSON string with the metadata
+ * Creates a JSON string with the metadata.
* @lucene.internal
*/
- public static MetaData _createJsonMetaData(ByteBuffer buf, List<String> signatures) throws IOException {
- String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(buf));
+ public static MetaData _createJsonMetaData(byte[] buf, List<String> signatures) throws IOException {
+ String sha512 = DigestUtils.sha512Hex(buf);
Map<String, Object> vals = new HashMap<>();
vals.put(MetaData.SHA512, sha512);
if (signatures != null) {
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 7f781df7ae1..1d89878d4b0 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -53,6 +53,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
+import java.util.stream.Collectors;
import java.util.zip.Adler32;
import java.util.zip.Checksum;
import java.util.zip.InflaterInputStream;
@@ -945,7 +946,7 @@ public class IndexFetcher {
// this is called before copying the files to the original conf dir
// so that if there is an exception avoid corrupting the original files.
terminateAndWaitFsyncService();
- copyTmpConfFiles2Conf(tmpconfDir);
+ copyTmpConfFiles2Conf(tmpConfPath);
} finally {
delTree(tmpconfDir);
}
@@ -1279,36 +1280,36 @@ public class IndexFetcher {
/**
* Make file list
+ * TODO: return the stream directly
*/
- private List<File> makeTmpConfDirFileList(File dir, List<File> fileList) {
- File[] files = dir.listFiles();
- for (File file : files) {
- if (file.isFile()) {
- fileList.add(file);
- } else if (file.isDirectory()) {
- fileList = makeTmpConfDirFileList(file, fileList);
- }
+ private List<Path> makeTmpConfDirFileList(Path dir) {
+ try {
+ return Files.walk(dir)
+ .filter(Files::isRegularFile)
+ .collect(Collectors.toList());
+ } catch (IOException e) {
+ log.warn("Could not walk file tree", e);
+ return Collections.emptyList();
}
- return fileList;
}
/**
* The conf files are copied to the tmp dir to the conf dir. A backup of the old file is maintained
*/
- private void copyTmpConfFiles2Conf(File tmpconfDir) {
+ private void copyTmpConfFiles2Conf(Path tmpconfDir) {
boolean status = false;
Path confPath = solrCore.getResourceLoader().getConfigPath();
- for (File file : makeTmpConfDirFileList(tmpconfDir, new ArrayList<>())) {
- Path oldPath = confPath.resolve(file.getPath().substring(tmpconfDir.getPath().length()));
- File oldFile = oldPath.toFile();
- if (!oldFile.getParentFile().exists()) {
- status = oldFile.getParentFile().mkdirs();
- if (!status) {
- throw new SolrException(ErrorCode.SERVER_ERROR,
- "Unable to mkdirs: " + oldFile.getParentFile());
- }
+ int numTempPathElements = tmpconfDir.getNameCount();
+ for (Path path : makeTmpConfDirFileList(tmpconfDir)) {
+ Path oldPath = confPath.resolve(path.subpath(numTempPathElements, path.getNameCount()));
+ try {
+ Files.createDirectories(oldPath.getParent());
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR,
+ "Unable to mkdirs: " + oldPath.getParent(), e);
}
- if (oldFile.exists()) {
+ if (Files.exists(oldPath)) {
+ File oldFile = oldPath.toFile(); // TODO drop this
File backupFile = new File(oldFile.getPath() + "." + getDateAsStr(new Date(oldFile.lastModified())));
if (!backupFile.getParentFile().exists()) {
status = backupFile.getParentFile().mkdirs();
@@ -1323,10 +1324,11 @@ public class IndexFetcher {
"Unable to rename: " + oldFile + " to: " + backupFile);
}
}
- status = file.renameTo(oldFile);
- if (!status) {
+ try {
+ Files.move(path, oldPath);
+ } catch (IOException e) {
throw new SolrException(ErrorCode.SERVER_ERROR,
- "Unable to rename: " + file + " to: " + oldFile);
+ "Unable to rename: " + path + " to: " + oldPath, e);
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java
index d5220c00ef1..7d74f8496b1 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java
@@ -257,10 +257,9 @@ public class PackageUtils {
return "/api/collections/" + collection + "/config/params";
}
- public static void uploadKey(byte bytes[], String path, Path home, HttpSolrClient client) throws IOException {
- ByteBuffer buf = ByteBuffer.wrap(bytes);
- PackageStoreAPI.MetaData meta = PackageStoreAPI._createJsonMetaData(buf, null);
- DistribPackageStore._persistToFile(home, path, buf, ByteBuffer.wrap(Utils.toJSON(meta)));
+ public static void uploadKey(byte[] bytes, String path, Path home, HttpSolrClient client) throws IOException {
+ PackageStoreAPI.MetaData meta = PackageStoreAPI._createJsonMetaData(bytes, null);
+ DistribPackageStore._persistToFile(home, path, ByteBuffer.wrap(bytes), ByteBuffer.wrap(Utils.toJSON(meta)));
}
}