Apache Solr open-source search software
SOLR-15244 Failing tests
Mike Drobab6e695973ca
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.javaindex b6e2f43d0fe..6d5f9be98c7 100644--- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java+++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java@@ -52,7 +52,6 @@ import org.apache.solr.common.util.Utils;import org.apache.solr.common.annotation.SolrThreadUnsafe;import org.apache.solr.core.CoreContainer;import org.apache.solr.filestore.PackageStoreAPI.MetaData;-import org.apache.solr.util.SimplePostTool;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;import org.apache.zookeeper.server.ByteBufferInputStream;@@ -95,7 +94,8 @@ public class DistribPackageStore implements PackageStore {if (!path.isEmpty() && path.charAt(0) != File.separatorChar) {path = File.separator + path;}- return solrHome.resolve(PackageStoreAPI.PACKAGESTORE_DIRECTORY).resolve(path);+ // Use concat because path might start with a slash and be incorrectly interpreted as absolute+ return solrHome.resolve(PackageStoreAPI.PACKAGESTORE_DIRECTORY + path);}class FileInfo {@@ -196,11 +196,13 @@ public class DistribPackageStore implements PackageStore {ByteBuffer filedata = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),baseUrl + "/node/files" + path,Utils.newBytesConsumer((int) MAX_PKG_SIZE));+ filedata.mark();String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(filedata));String expected = (String) m.get("sha512");if (!sha512.equals(expected)) {throw new SolrException(SERVER_ERROR, "sha512 mismatch downloading : " + path + " from node : " + fromNode);}+ filedata.reset();persistToFile(filedata, metadata);return true;} catch (SolrException e) {@@ -589,12 +591,12 @@ public class DistribPackageStore implements PackageStore {}Path metadataPath = _getRealPath(_getMetapath(path), solrHome);- try (SeekableByteChannel channel = Files.newByteChannel(metadataPath, StandardOpenOption.WRITE)) {+ try (SeekableByteChannel channel = Files.newByteChannel(metadataPath, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) {channel.write(meta);}IOUtils.fsync(metadataPath, false);- try (SeekableByteChannel channel = Files.newByteChannel(realpath, StandardOpenOption.WRITE)) {+ try (SeekableByteChannel channel = Files.newByteChannel(realpath, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) {channel.write(data);}IOUtils.fsync(realpath, false);@@ -615,9 +617,7 @@ public class DistribPackageStore implements PackageStore {if (keyFiles == null) return result;for (File keyFile : keyFiles) {if (keyFile.isFile() && !isMetaDataFile(keyFile.getName())) {- try (InputStream fis = new FileInputStream(keyFile)) {- result.put(keyFile.getName(), SimplePostTool.inputStreamToByteArray(fis).array());- }+ result.put(keyFile.getName(), Files.readAllBytes(keyFile.toPath()));}}return result;diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.javaindex db2e05ebdcd..6a10315ce66 100644--- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java+++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java@@ -199,14 +199,14 @@ public class PackageStoreAPI {validateName(path, true);ContentStream stream = streams.iterator().next();try {- ByteBuffer buf = SimplePostTool.inputStreamToByteArray(stream.getStream());+ byte[] buf = stream.getStream().readAllBytes();List<String> signatures = readSignatures(req, buf);MetaData meta = _createJsonMetaData(buf, signatures);PackageStore.FileType type = packageStore.getType(path, true);if(type != PackageStore.FileType.NOFILE) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path already exists "+ path);}- packageStore.put(new PackageStore.FileEntry(buf, meta, path));+ packageStore.put(new PackageStore.FileEntry(ByteBuffer.wrap(buf), meta, path));rsp.add(CommonParams.FILE, path);} catch (IOException e) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);@@ -226,7 +226,7 @@ public class PackageStoreAPI {}}- private List<String> readSignatures(SolrQueryRequest req, ByteBuffer buf)+ private List<String> readSignatures(SolrQueryRequest req, byte[] buf)throws SolrException, IOException {String[] signatures = req.getParams().getParams("sig");if (signatures == null || signatures.length == 0) return null;@@ -237,7 +237,7 @@ public class PackageStoreAPI {}private void validate(List<String> sigs,- ByteBuffer buf) throws SolrException, IOException {+ byte[] buf) throws SolrException, IOException {Map<String, byte[]> keys = packageStore.getKeys();if (keys == null || keys.isEmpty()) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,@@ -251,9 +251,9 @@ public class PackageStoreAPI {"Error parsing public keys in Package store");}for (String sig : sigs) {- if (cryptoKeys.verify(sig, buf) == null) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig +" len: "+buf.limit()+ " content sha512: "+- DigestUtils.sha512Hex(new ByteBufferInputStream(buf)));+ if (cryptoKeys.verify(sig, ByteBuffer.wrap(buf)) == null) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig +" len: "+buf.length+ " content sha512: "++ DigestUtils.sha512Hex(buf));}}@@ -262,11 +262,11 @@ public class PackageStoreAPI {}/**- * Creates a JSON string with the metadata+ * Creates a JSON string with the metadata.* @lucene.internal*/- public static MetaData _createJsonMetaData(ByteBuffer buf, List<String> signatures) throws IOException {- String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(buf));+ public static MetaData _createJsonMetaData(byte[] buf, List<String> signatures) throws IOException {+ String sha512 = DigestUtils.sha512Hex(buf);Map<String, Object> vals = new HashMap<>();vals.put(MetaData.SHA512, sha512);if (signatures != null) {diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.javaindex 7f781df7ae1..1d89878d4b0 100644--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java@@ -53,6 +53,7 @@ import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;import java.util.function.BooleanSupplier;import java.util.function.Function;+import java.util.stream.Collectors;import java.util.zip.Adler32;import java.util.zip.Checksum;import java.util.zip.InflaterInputStream;@@ -945,7 +946,7 @@ public class IndexFetcher {// this is called before copying the files to the original conf dir// so that if there is an exception avoid corrupting the original files.terminateAndWaitFsyncService();- copyTmpConfFiles2Conf(tmpconfDir);+ copyTmpConfFiles2Conf(tmpConfPath);} finally {delTree(tmpconfDir);}@@ -1279,36 +1280,36 @@ public class IndexFetcher {/*** Make file list+ * TODO: return the stream directly*/- private List<File> makeTmpConfDirFileList(File dir, List<File> fileList) {- File[] files = dir.listFiles();- for (File file : files) {- if (file.isFile()) {- fileList.add(file);- } else if (file.isDirectory()) {- fileList = makeTmpConfDirFileList(file, fileList);- }+ private List<Path> makeTmpConfDirFileList(Path dir) {+ try {+ return Files.walk(dir)+ .filter(Files::isRegularFile)+ .collect(Collectors.toList());+ } catch (IOException e) {+ log.warn("Could not walk file tree", e);+ return Collections.emptyList();}- return fileList;}/*** The conf files are copied to the tmp dir to the conf dir. A backup of the old file is maintained*/- private void copyTmpConfFiles2Conf(File tmpconfDir) {+ private void copyTmpConfFiles2Conf(Path tmpconfDir) {boolean status = false;Path confPath = solrCore.getResourceLoader().getConfigPath();- for (File file : makeTmpConfDirFileList(tmpconfDir, new ArrayList<>())) {- Path oldPath = confPath.resolve(file.getPath().substring(tmpconfDir.getPath().length()));- File oldFile = oldPath.toFile();- if (!oldFile.getParentFile().exists()) {- status = oldFile.getParentFile().mkdirs();- if (!status) {- throw new SolrException(ErrorCode.SERVER_ERROR,- "Unable to mkdirs: " + oldFile.getParentFile());- }+ int numTempPathElements = tmpconfDir.getNameCount();+ for (Path path : makeTmpConfDirFileList(tmpconfDir)) {+ Path oldPath = confPath.resolve(path.subpath(numTempPathElements, path.getNameCount()));+ try {+ Files.createDirectories(oldPath.getParent());+ } catch (IOException e) {+ throw new SolrException(ErrorCode.SERVER_ERROR,+ "Unable to mkdirs: " + oldPath.getParent(), e);}- if (oldFile.exists()) {+ if (Files.exists(oldPath)) {+ File oldFile = oldPath.toFile(); // TODO drop thisFile backupFile = new File(oldFile.getPath() + "." + getDateAsStr(new Date(oldFile.lastModified())));if (!backupFile.getParentFile().exists()) {status = backupFile.getParentFile().mkdirs();@@ -1323,10 +1324,11 @@ public class IndexFetcher {"Unable to rename: " + oldFile + " to: " + backupFile);}}- status = file.renameTo(oldFile);- if (!status) {+ try {+ Files.move(path, oldPath);+ } catch (IOException e) {throw new SolrException(ErrorCode.SERVER_ERROR,- "Unable to rename: " + file + " to: " + oldFile);+ "Unable to rename: " + path + " to: " + oldPath, e);}}}diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.javaindex d5220c00ef1..7d74f8496b1 100644--- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java+++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java@@ -257,10 +257,9 @@ public class PackageUtils {return "/api/collections/" + collection + "/config/params";}- public static void uploadKey(byte bytes[], String path, Path home, HttpSolrClient client) throws IOException {- ByteBuffer buf = ByteBuffer.wrap(bytes);- PackageStoreAPI.MetaData meta = PackageStoreAPI._createJsonMetaData(buf, null);- DistribPackageStore._persistToFile(home, path, buf, ByteBuffer.wrap(Utils.toJSON(meta)));+ public static void uploadKey(byte[] bytes, String path, Path home, HttpSolrClient client) throws IOException {+ PackageStoreAPI.MetaData meta = PackageStoreAPI._createJsonMetaData(bytes, null);+ DistribPackageStore._persistToFile(home, path, ByteBuffer.wrap(bytes), ByteBuffer.wrap(Utils.toJSON(meta)));}}