Apache Solr open-source search software
SOLR-17284: Remove deprecated BlobRepository (#2447)
Eric Pughf5b36c8a819b
diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.javadeleted file mode 100644index b2960a67151..00000000000--- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java+++ /dev/null@@ -1,367 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.core;--import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;-import static org.apache.solr.common.SolrException.ErrorCode.SERVICE_UNAVAILABLE;--import java.io.InputStream;-import java.lang.invoke.MethodHandles;-import java.math.BigInteger;-import java.nio.ByteBuffer;-import java.security.MessageDigest;-import java.security.NoSuchAlgorithmException;-import java.util.ArrayList;-import java.util.Collections;-import java.util.HashSet;-import java.util.List;-import java.util.Locale;-import java.util.Map;-import java.util.Random;-import java.util.Set;-import java.util.concurrent.Callable;-import java.util.concurrent.ConcurrentHashMap;-import java.util.regex.Pattern;-import org.apache.http.HttpEntity;-import org.apache.http.HttpResponse;-import org.apache.http.client.HttpClient;-import org.apache.http.client.methods.HttpGet;-import org.apache.solr.common.SolrException;-import org.apache.solr.common.cloud.ClusterState;-import org.apache.solr.common.cloud.DocCollection;-import org.apache.solr.common.cloud.Replica;-import org.apache.solr.common.cloud.Slice;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.params.CollectionAdminParams;-import org.apache.solr.common.util.StrUtils;-import org.apache.solr.common.util.Utils;-import org.apache.zookeeper.server.ByteBufferInputStream;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--/**- * The purpose of this class is to store the Jars loaded in memory and to keep only one copy of the- * Jar in a single node.- */-public class BlobRepository {-- private static final long MAX_JAR_SIZE =- Long.parseLong(System.getProperty("runtime.lib.size", String.valueOf(5 * 1024 * 1024)));- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- public static final Random RANDOM;- static final Pattern BLOB_KEY_PATTERN_CHECKER = Pattern.compile(".*/\\d+");-- static {- // We try to make things reproducible in the context of our tests by initializing the random- // instance based on the current seed- String seed = System.getProperty("tests.seed");- if (seed == null) {- RANDOM = new Random();- } else {- RANDOM = new Random(seed.hashCode());- }- }-- private final CoreContainer coreContainer;-- @SuppressWarnings({"rawtypes"})- private Map<String, BlobContent> blobs = createMap();-- // for unit tests to override- @SuppressWarnings({"rawtypes"})- ConcurrentHashMap<String, BlobContent> createMap() {- return new ConcurrentHashMap<>();- }-- public BlobRepository(CoreContainer coreContainer) {- this.coreContainer = coreContainer;- }-- // I wanted to {@link SolrCore#loadDecodeAndCacheBlob(String, Decoder)} below but precommit- // complains-- /**- * Returns the contents of a blob containing a ByteBuffer and increments a reference count. Please- * return the same object to decrease the refcount. This is normally used for storing jar files,- * and binary raw data. If you are caching Java Objects you want to use {@code- * SolrCore#loadDecodeAndCacheBlob(String, Decoder)}- *- * @param key it is a combination of blobname and version like blobName/version- * @return The reference of a blob- */- public BlobContentRef<ByteBuffer> getBlobIncRef(String key) {- return getBlobIncRef(key, () -> addBlob(key));- }-- /**- * Internal method that returns the contents of a blob and increments a reference count. Please- * return the same object to decrease the refcount. Only the decoded content will be cached when- * this method is used. Component authors attempting to share objects across cores should use- * {@code SolrCore#loadDecodeAndCacheBlob(String, Decoder)} which ensures that a proper close hook- * is also created.- *- * @param key it is a combination of blob name and version like blobName/version- * @param decoder a decoder that knows how to interpret the bytes from the blob- * @return The reference of a blob- */- <T> BlobContentRef<T> getBlobIncRef(String key, Decoder<T> decoder) {- return getBlobIncRef(key.concat(decoder.getName()), () -> addBlob(key, decoder));- }-- <T> BlobContentRef<T> getBlobIncRef(String key, Decoder<T> decoder, String url, String sha512) {- StringBuilder keyBuilder = new StringBuilder(key);- if (decoder != null) keyBuilder.append(decoder.getName());- keyBuilder.append("/").append(sha512);-- return getBlobIncRef(- keyBuilder.toString(),- () -> new BlobContent<>(key, fetchBlobAndVerify(key, url, sha512), decoder));- }-- // do the actual work returning the appropriate type...- @SuppressWarnings({"unchecked"})- private <T> BlobContentRef<T> getBlobIncRef(String key, Callable<BlobContent<T>> blobCreator) {- BlobContent<T> aBlob;- if (this.coreContainer.isZooKeeperAware()) {- synchronized (blobs) {- aBlob = blobs.get(key);- if (aBlob == null) {- try {- aBlob = blobCreator.call();- } catch (Exception e) {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "Blob loading failed: " + e.getMessage(), e);- }- }- }- } else {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "Blob loading is not supported in non-cloud mode");- // todo- }- BlobContentRef<T> ref = new BlobContentRef<>(aBlob);- synchronized (aBlob.references) {- aBlob.references.add(ref);- }- return ref;- }-- // For use cases sharing raw bytes- private BlobContent<ByteBuffer> addBlob(String key) {- ByteBuffer b = fetchBlob(key);- BlobContent<ByteBuffer> aBlob = new BlobContent<>(key, b);- blobs.put(key, aBlob);- return aBlob;- }-- // for use cases sharing java objects- private <T> BlobContent<T> addBlob(String key, Decoder<T> decoder) {- ByteBuffer b = fetchBlob(key);- String keyPlusName = key + decoder.getName();- BlobContent<T> aBlob = new BlobContent<>(keyPlusName, b, decoder);- blobs.put(keyPlusName, aBlob);- return aBlob;- }-- static String INVALID_JAR_MSG =- "Invalid jar from {0} , expected sha512 hash : {1} , actual : {2}";-- private ByteBuffer fetchBlobAndVerify(String key, String url, String sha512) {- ByteBuffer byteBuffer = fetchFromUrl(key, url);- String computedDigest = sha512Digest(byteBuffer);- if (!computedDigest.equals(sha512)) {- throw new SolrException(- SERVER_ERROR, StrUtils.formatString(INVALID_JAR_MSG, url, sha512, computedDigest));- }- return byteBuffer;- }-- public static String sha512Digest(ByteBuffer byteBuffer) {- MessageDigest digest = null;- try {- digest = MessageDigest.getInstance("SHA-512");- } catch (NoSuchAlgorithmException e) {- // unlikely- throw new SolrException(SERVER_ERROR, e);- }- digest.update(byteBuffer);- return String.format(Locale.ROOT, "%0128x", new BigInteger(1, digest.digest()));- }-- /** Package local for unit tests only please do not use elsewhere */- ByteBuffer fetchBlob(String key) {- Replica replica = getSystemCollReplica();- String url =- replica.getBaseUrl()- + "/"- + CollectionAdminParams.SYSTEM_COLL- + "/blob/"- + key- + "?wt=filestream";- return fetchFromUrl(key, url);- }-- ByteBuffer fetchFromUrl(String key, String url) {- HttpClient httpClient = coreContainer.getUpdateShardHandler().getDefaultHttpClient();- HttpGet httpGet = new HttpGet(url);- ByteBuffer b;- HttpResponse response = null;- HttpEntity entity = null;- try {- response = httpClient.execute(httpGet);- entity = response.getEntity();- int statusCode = response.getStatusLine().getStatusCode();- if (statusCode != 200) {- throw new SolrException(- SolrException.ErrorCode.NOT_FOUND, "no such blob or version available: " + key);- }-- try (InputStream is = entity.getContent()) {- b = Utils.toByteArray(is, MAX_JAR_SIZE);- }- } catch (Exception e) {- if (e instanceof SolrException) {- throw (SolrException) e;- } else {- throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "could not load : " + key, e);- }- } finally {- Utils.consumeFully(entity);- }- return b;- }-- private Replica getSystemCollReplica() {- ZkStateReader zkStateReader = this.coreContainer.getZkController().getZkStateReader();- ClusterState cs = zkStateReader.getClusterState();- DocCollection coll = cs.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL);- if (coll == null)- throw new SolrException(- SERVICE_UNAVAILABLE, CollectionAdminParams.SYSTEM_COLL + " collection not available");- ArrayList<Slice> slices = new ArrayList<>(coll.getActiveSlices());- if (slices.isEmpty())- throw new SolrException(- SERVICE_UNAVAILABLE,- "No active slices for " + CollectionAdminParams.SYSTEM_COLL + " collection");- Collections.shuffle(slices, RANDOM); // do load balancing-- Replica replica = null;- for (Slice slice : slices) {- List<Replica> replicas = new ArrayList<>(slice.getReplicasMap().values());- Collections.shuffle(replicas, RANDOM);- for (Replica r : replicas) {- if (r.getState() == Replica.State.ACTIVE) {- if (zkStateReader- .getClusterState()- .getLiveNodes()- .contains(r.get(ZkStateReader.NODE_NAME_PROP))) {- replica = r;- break;- } else {- if (log.isInfoEnabled()) {- log.info(- "replica {} says it is active but not a member of live nodes",- r.get(ZkStateReader.NODE_NAME_PROP));- }- }- }- }- }- if (replica == null) {- throw new SolrException(- SERVICE_UNAVAILABLE,- "No active replica available for " + CollectionAdminParams.SYSTEM_COLL + " collection");- }- return replica;- }-- /**- * This is to decrement a ref count- *- * @param ref The reference that is already there. Doing multiple calls with same ref will not- * matter- */- public void decrementBlobRefCount(BlobContentRef<?> ref) {- if (ref == null) return;- synchronized (ref.blob.references) {- if (!ref.blob.references.remove(ref)) {- log.error("Multiple releases for the same reference");- }- if (ref.blob.references.isEmpty()) {- blobs.remove(ref.blob.key);- }- }- }-- public static class BlobContent<T> {- public final String key;- // holds byte buffer or cached object, holding both is a waste of memory ref counting mechanism- private final T content;- private final Set<BlobContentRef<T>> references = new HashSet<>();-- @SuppressWarnings("unchecked")- public BlobContent(String key, ByteBuffer buffer, Decoder<T> decoder) {- this.key = key;- this.content =- decoder == null ? (T) buffer : decoder.decode(new ByteBufferInputStream(buffer));- }-- @SuppressWarnings("unchecked")- public BlobContent(String key, ByteBuffer buffer) {- this.key = key;- this.content = (T) buffer;- }-- /**- * Get the cached object.- *- * @return the object representing the content that is cached.- */- public T get() {- return this.content;- }- }-- public interface Decoder<T> {-- /**- * A name by which to distinguish this decoding. This only needs to be implemented if you want- * to support decoding the same blob content with more than one decoder.- *- * @return The name of the decoding, defaults to empty string.- */- default String getName() {- return "";- }-- /**- * A routine that knows how to convert the stream of bytes from the blob into a Java object.- *- * @param inputStream the bytes from a blob- * @return A Java object of the specified type.- */- T decode(InputStream inputStream);- }-- public static class BlobContentRef<T> {- public final BlobContent<T> blob;-- private BlobContentRef(BlobContent<T> blob) {- this.blob = blob;- }- }-}diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex ac27230d548..d0978fcbba2 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -251,8 +251,6 @@ public class CoreContainer {private volatile String hostName;- private final BlobRepository blobRepository = new BlobRepository(this);-private volatile boolean asyncSolrCoreLoad;protected volatile SecurityConfHandler securityConfHandler;@@ -2319,10 +2317,6 @@ public class CoreContainer {return core;}- public BlobRepository getBlobRepository() {- return blobRepository;- }-/*** If using asyncSolrCoreLoad=true, calling this after {@link #load()} will not return until all* cores have finished loading.diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.javaindex 2b1bd65f292..92a963f6bbe 100644--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java@@ -89,7 +89,6 @@ import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.CommonParams.EchoParamStyle;import org.apache.solr.common.params.SolrParams;@@ -3540,40 +3539,6 @@ public class SolrCore implements SolrInfoBean, Closeable {return ImplicitHolder.INSTANCE;}- /**- * Convenience method to load a blob. This method minimizes the degree to which component and- * other code needs to depend on the structure of solr's object graph and ensures that a proper- * close hook is registered. This method should normally be called in {@link- * SolrCoreAware#inform(SolrCore)}, and should never be called during request processing. The- * Decoder will only run on the first invocations, subsequent invocations will return the cached- * object.- *- * @param key A key in the format of name/version for a blob stored in the {@link- * CollectionAdminParams#SYSTEM_COLL} blob store via the Blob Store API- * @param decoder a decoder with which to convert the blob into a Java Object representation- * (first time only)- * @return a reference to the blob that has already cached the decoded version.- */- public <T> BlobRepository.BlobContentRef<T> loadDecodeAndCacheBlob(- String key, BlobRepository.Decoder<T> decoder) {- // make sure component authors don't give us oddball keys with no version...- if (!BlobRepository.BLOB_KEY_PATTERN_CHECKER.matcher(key).matches()) {- throw new IllegalArgumentException(- "invalid key format, must end in /N where N is the version number");- }- // define the blob- BlobRepository.BlobContentRef<T> blobRef =- coreContainer.getBlobRepository().getBlobIncRef(key, decoder);- addCloseHook(- new CloseHook() {- @Override- public void postClose(SolrCore core) {- coreContainer.getBlobRepository().decrementBlobRefCount(blobRef);- }- });- return blobRef;- }-public CancellableQueryTracker getCancellableQueryTracker() {return cancellableQueryTracker;}diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.javaindex 52dee0f5e9a..dc420d37b31 100644--- a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java+++ b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java@@ -45,7 +45,6 @@ import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.ContentStream;import org.apache.solr.common.util.StrUtils;import org.apache.solr.common.util.Utils;-import org.apache.solr.core.BlobRepository;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrCore;import org.apache.solr.pkg.PackageAPI;@@ -84,7 +83,7 @@ public class FileStoreAPI {coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes();ArrayList<String> l = new ArrayList<>(liveNodes);l.remove(coreContainer.getZkController().getNodeName());- Collections.shuffle(l, BlobRepository.RANDOM);+ Collections.shuffle(l, Utils.RANDOM);return l;}diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.javaindex d6e765b0d4a..c0d50dd85e2 100644--- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java+++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java@@ -48,7 +48,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;-import org.apache.solr.core.BlobRepository;import org.apache.solr.filestore.DistribFileStore;import org.apache.solr.filestore.FileStoreAPI;import org.apache.solr.packagemanager.SolrPackage.Manifest;@@ -209,7 +208,7 @@ public class PackageUtils {NamedList<Object> response = solrClient.request(request);String manifestJson = (String) response.get("response");String calculatedSHA512 =- BlobRepository.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8)));+ Utils.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8)));if (expectedSHA512.equals(calculatedSHA512) == false) {throw new SolrException(ErrorCode.UNAUTHORIZED,diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.javaindex 109468f854c..d92d027d1c0 100644--- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java+++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java@@ -50,7 +50,7 @@ import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;-import org.apache.solr.core.BlobRepository;+import org.apache.solr.common.util.Utils;import org.apache.solr.filestore.FileStoreAPI;import org.apache.solr.packagemanager.SolrPackage.Artifact;import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease;@@ -193,8 +193,7 @@ public class RepositoryManager {}String manifestJson = getMapper().writeValueAsString(release.manifest);String manifestSHA512 =- BlobRepository.sha512Digest(- ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8)));+ Utils.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8)));PackageUtils.postFile(solrClient,ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8)),diff --git a/solr/core/src/test-files/solr/configsets/resource-sharing/schema.xml b/solr/core/src/test-files/solr/configsets/resource-sharing/schema.xmldeleted file mode 100644index 287d4fe0149..00000000000--- a/solr/core/src/test-files/solr/configsets/resource-sharing/schema.xml+++ /dev/null@@ -1,21 +0,0 @@-<?xml version="1.0" encoding="UTF-8" ?>-<!--- Licensed to the Apache Software Foundation (ASF) under one or more- contributor license agreements. See the NOTICE file distributed with- this work for additional information regarding copyright ownership.- The ASF licenses this file to You under the Apache License, Version 2.0- (the "License"); you may not use this file except in compliance with- the License. You may obtain a copy of the License at-- http://www.apache.org/licenses/LICENSE-2.0-- Unless required by applicable law or agreed to in writing, software- distributed under the License is distributed on an "AS IS" BASIS,- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- See the License for the specific language governing permissions and- limitations under the License.--->-<schema name="minimal" version="1.1">- <fieldType name="string" class="solr.StrField"/>- <dynamicField name="*" type="string" indexed="true" stored="true"/>-</schema>diff --git a/solr/core/src/test-files/solr/configsets/resource-sharing/solrconfig.xml b/solr/core/src/test-files/solr/configsets/resource-sharing/solrconfig.xmldeleted file mode 100644index 1dd92feef2e..00000000000--- a/solr/core/src/test-files/solr/configsets/resource-sharing/solrconfig.xml+++ /dev/null@@ -1,51 +0,0 @@-<?xml version="1.0" ?>--<!--- Licensed to the Apache Software Foundation (ASF) under one or more- contributor license agreements. See the NOTICE file distributed with- this work for additional information regarding copyright ownership.- The ASF licenses this file to You under the Apache License, Version 2.0- (the "License"); you may not use this file except in compliance with- the License. You may obtain a copy of the License at-- http://www.apache.org/licenses/LICENSE-2.0-- Unless required by applicable law or agreed to in writing, software- distributed under the License is distributed on an "AS IS" BASIS,- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- See the License for the specific language governing permissions and- limitations under the License.--->--<!-- Minimal solrconfig.xml with /select, /admin and /update only -->--<config>-- <dataDir>${solr.data.dir:}</dataDir>-- <directoryFactory name="DirectoryFactory"- class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>- <schemaFactory class="ClassicIndexSchemaFactory"/>-- <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>-- <updateHandler class="solr.DirectUpdateHandler2">- <commitWithin>- <softCommit>${solr.commitwithin.softcommit:true}</softCommit>- </commitWithin>-- </updateHandler>- <searchComponent name="testComponent" class="org.apache.solr.handler.component.ResourceSharingTestComponent" />-- <requestHandler name="/select" class="solr.SearchHandler">- <lst name="defaults">- <str name="echoParams">explicit</str>- <str name="indent">true</str>- <str name="df">text</str>- </lst>- <arr name="first-components">- <str>testComponent</str>- </arr>- </requestHandler>-</config>-diff --git a/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java b/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.javadeleted file mode 100644index 8fe8c34f79b..00000000000--- a/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java+++ /dev/null@@ -1,127 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.core;--import java.io.IOException;-import java.nio.ByteBuffer;-import java.nio.charset.StandardCharsets;-import java.nio.file.Path;-import org.apache.solr.client.solrj.SolrQuery;-import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.impl.CloudSolrClient;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.response.QueryResponse;-import org.apache.solr.cloud.SolrCloudTestCase;-import org.apache.solr.common.SolrDocumentList;-import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.params.CollectionAdminParams;-import org.apache.solr.handler.TestBlobHandler;-import org.junit.BeforeClass;-import org.junit.Test;--public class BlobRepositoryCloudTest extends SolrCloudTestCase {-- public static final Path TEST_PATH = getFile("solr/configsets").toPath();-- @BeforeClass- public static void setupCluster() throws Exception {- configureCluster(1) // only sharing *within* a node- .addConfig("configname", TEST_PATH.resolve("resource-sharing"))- .configure();- // Thread.sleep(2000);- CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 1)- .process(cluster.getSolrClient());- // test component will fail if it can't find a blob with this data by this name- TestBlobHandler.postData(- cluster.getSolrClient(),- findLiveNodeURI(),- "testResource",- ByteBuffer.wrap("foo,bar\nbaz,bam".getBytes(StandardCharsets.UTF_8)));- // Thread.sleep(2000);- // if these don't load we probably failed to post the blob above- CollectionAdminRequest.createCollection("col1", "configname", 1, 1)- .process(cluster.getSolrClient());- CollectionAdminRequest.createCollection("col2", "configname", 1, 1)- .process(cluster.getSolrClient());-- SolrInputDocument document = new SolrInputDocument();- document.addField("id", "1");- document.addField("text", "col1");- CloudSolrClient solrClient = cluster.getSolrClient();- solrClient.add("col1", document);- solrClient.commit("col1");- document = new SolrInputDocument();- document.addField("id", "1");- document.addField("text", "col2");- solrClient.add("col2", document);- solrClient.commit("col2");- Thread.sleep(2000);- }-- @Test- public void test() throws Exception {- // This test relies on the installation of ResourceSharingTestComponent which has 2 useful- // properties:- // 1. it will fail to initialize if it doesn't find a 2 line CSV like foo,bar\nbaz,bam thus- // validating that we are properly pulling data from the blob store- // 2. It replaces any q for a query request to /select with "text:<name>" where <name> is- // the name of the last collection to run a query. It does this by caching a shared resource of- // type ResourceSharingTestComponent.TestObject, and the following sequence is proof that either- // collection can tell if it was (or was not) the last collection to issue a query by consulting- // the shared object- assertLastQueryNotToCollection("col1");- assertLastQueryNotToCollection("col2");- assertLastQueryNotToCollection("col1");- assertLastQueryToCollection("col1");- assertLastQueryNotToCollection("col2");- assertLastQueryToCollection("col2");- }-- // TODO: move this up to parent class?- private static String findLiveNodeURI() {- ZkStateReader zkStateReader = cluster.getZkStateReader();- return zkStateReader.getBaseUrlForNodeName(- zkStateReader- .getClusterState()- .getCollection(".system")- .getSlices()- .iterator()- .next()- .getLeader()- .getNodeName());- }-- private void assertLastQueryToCollection(String collection)- throws SolrServerException, IOException {- assertEquals(1, getSolrDocuments(collection).size());- }-- private void assertLastQueryNotToCollection(String collection)- throws SolrServerException, IOException {- assertEquals(0, getSolrDocuments(collection).size());- }-- private SolrDocumentList getSolrDocuments(String collection)- throws SolrServerException, IOException {- SolrQuery query = new SolrQuery("*:*");- CloudSolrClient client = cluster.getSolrClient();- QueryResponse resp1 = client.query(collection, query);- return resp1.getResults();- }-}diff --git a/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java b/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.javadeleted file mode 100644index cf37174bbc3..00000000000--- a/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java+++ /dev/null@@ -1,191 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.core;--import static org.mockito.Mockito.mock;-import static org.mockito.Mockito.reset;-import static org.mockito.Mockito.verify;-import static org.mockito.Mockito.when;--import java.io.IOException;-import java.io.InputStream;-import java.io.InputStreamReader;-import java.io.StringWriter;-import java.nio.ByteBuffer;-import java.nio.charset.Charset;-import java.nio.charset.StandardCharsets;-import java.util.Objects;-import java.util.concurrent.ConcurrentHashMap;-import org.apache.solr.SolrTestCaseJ4;-import org.apache.solr.common.SolrException;-import org.junit.Before;-import org.junit.BeforeClass;-import org.junit.Test;--public class BlobRepositoryMockingTest extends SolrTestCaseJ4 {-- private static final Charset UTF8 = StandardCharsets.UTF_8;- private static final String[][] PARSED =- new String[][] {{"foo", "bar", "baz"}, {"bang", "boom", "bash"}};- private static final String BLOBSTR = "foo,bar,baz\nbang,boom,bash";- private CoreContainer mockContainer = mock(CoreContainer.class);-- @SuppressWarnings({"unchecked", "rawtypes"})- private ConcurrentHashMap<String, BlobRepository.BlobContent> blobStorage;-- BlobRepository repository;- ByteBuffer blobData = ByteBuffer.wrap(BLOBSTR.getBytes(UTF8));- boolean blobFetched = false;- String blobKey = "";- String url = null;- ByteBuffer filecontent = null;-- @BeforeClass- public static void beforeClass() {- SolrTestCaseJ4.assumeWorkingMockito();- }-- @Override- @Before- public void setUp() throws Exception {- super.setUp();- blobFetched = false;- blobKey = "";- reset(mockContainer);- blobStorage = new ConcurrentHashMap<>();- repository =- new BlobRepository(mockContainer) {- @Override- ByteBuffer fetchBlob(String key) {- blobKey = key;- blobFetched = true;- return blobData;- }-- @Override- ByteBuffer fetchFromUrl(String key, String url) {- if (!Objects.equals(url, BlobRepositoryMockingTest.this.url)) return null;- blobKey = key;- blobFetched = true;- return filecontent;- }-- @Override- @SuppressWarnings({"rawtypes"})- ConcurrentHashMap<String, BlobContent> createMap() {- return blobStorage;- }- };- }-- @Test(expected = SolrException.class)- public void testCloudOnly() {- when(mockContainer.isZooKeeperAware()).thenReturn(false);- try {- repository.getBlobIncRef("foo!");- } catch (SolrException e) {- verify(mockContainer).isZooKeeperAware();- throw e;- }- }-- @Test- public void testGetBlobIncrRefString() {- when(mockContainer.isZooKeeperAware()).thenReturn(true);- BlobRepository.BlobContentRef<ByteBuffer> ref = repository.getBlobIncRef("foo!");- assertEquals("foo!", blobKey);- assertTrue(blobFetched);- assertNotNull(ref.blob);- assertEquals(blobData, ref.blob.get());- verify(mockContainer).isZooKeeperAware();- assertNotNull(blobStorage.get("foo!"));- }-- @Test- public void testGetBlobIncrRefByUrl() throws Exception {- when(mockContainer.isZooKeeperAware()).thenReturn(true);- filecontent = TestSolrConfigHandler.getFileContent("runtimecode/runtimelibs_v2.jar.bin");- url = "http://localhost:8080/myjar/location.jar";- BlobRepository.BlobContentRef<?> ref =- repository.getBlobIncRef(- "filefoo",- null,- url,- "bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417");- assertEquals("filefoo", blobKey);- assertTrue(blobFetched);- assertNotNull(ref.blob);- assertEquals(filecontent, ref.blob.get());- verify(mockContainer).isZooKeeperAware();- try {- repository.getBlobIncRef("filefoo", null, url, "WRONG-SHA512-KEY");- fail("expected exception");- } catch (Exception e) {- assertTrue(e.getMessage().contains(" expected sha512 hash : WRONG-SHA512-KEY , actual :"));- }-- url = null;- filecontent = null;- }-- @Test- public void testCachedAlready() {- when(mockContainer.isZooKeeperAware()).thenReturn(true);- blobStorage.put("foo!", new BlobRepository.BlobContent<BlobRepository>("foo!", blobData));- BlobRepository.BlobContentRef<ByteBuffer> ref = repository.getBlobIncRef("foo!");- assertEquals("", blobKey);- assertFalse(blobFetched);- assertNotNull(ref.blob);- assertEquals(blobData, ref.blob.get());- verify(mockContainer).isZooKeeperAware();- assertNotNull("Key was not mapped to a BlobContent instance.", blobStorage.get("foo!"));- }-- @Test- public void testGetBlobIncrRefStringDecoder() {- when(mockContainer.isZooKeeperAware()).thenReturn(true);- BlobRepository.BlobContentRef<Object> ref =- repository.getBlobIncRef(- "foo!",- new BlobRepository.Decoder<>() {- @Override- public String[][] decode(InputStream inputStream) {- StringWriter writer = new StringWriter();- try {- new InputStreamReader(inputStream, UTF8).transferTo(writer);- } catch (IOException e) {- throw new RuntimeException(e);- }-- assertEquals(BLOBSTR, writer.toString());- return PARSED;- }-- @Override- public String getName() {- return "mocked";- }- });- assertEquals("foo!", blobKey);- assertTrue(blobFetched);- assertNotNull(ref.blob);- assertEquals(PARSED, ref.blob.get());- verify(mockContainer).isZooKeeperAware();- assertNotNull(blobStorage.get("foo!mocked"));- }-}diff --git a/solr/core/src/test/org/apache/solr/handler/component/ResourceSharingTestComponent.java b/solr/core/src/test/org/apache/solr/handler/component/ResourceSharingTestComponent.javadeleted file mode 100644index e653b1f6de3..00000000000--- a/solr/core/src/test/org/apache/solr/handler/component/ResourceSharingTestComponent.java+++ /dev/null@@ -1,142 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.handler.component;--import static org.junit.Assert.assertEquals;--import java.io.BufferedReader;-import java.io.InputStream;-import java.io.InputStreamReader;-import java.lang.invoke.MethodHandles;-import java.nio.charset.StandardCharsets;-import java.util.HashMap;-import java.util.Map;-import java.util.stream.Stream;-import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.params.SolrParams;-import org.apache.solr.core.BlobRepository;-import org.apache.solr.core.SolrCore;-import org.apache.solr.util.plugin.SolrCoreAware;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--public class ResourceSharingTestComponent extends SearchComponent implements SolrCoreAware {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- private SolrCore core;- private volatile BlobRepository.BlobContent<TestObject> blob;-- @SuppressWarnings("SynchronizeOnNonFinalField")- @Override- public void prepare(ResponseBuilder rb) {- SolrParams params = rb.req.getParams();- ModifiableSolrParams mParams = new ModifiableSolrParams(params);- String q = "text:" + getTestObj().getLastCollection();- mParams.set("q", q); // search for the last collection name.- // This should cause the param to show up in the response...- rb.req.setParams(mParams);- getTestObj().setLastCollection(core.getCoreDescriptor().getCollectionName());- }-- @Override- public void process(ResponseBuilder rb) {}-- @Override- public String getDescription() {- return "ResourceSharingTestComponent";- }-- TestObject getTestObj() {- return this.blob.get();- }-- @SuppressWarnings("unchecked")- @Override- public void inform(SolrCore core) {- log.info("Informing test component...");- this.core = core;- this.blob = core.loadDecodeAndCacheBlob(getKey(), new DumbCsvDecoder()).blob;- log.info("Test component informed!");- }-- private String getKey() {- return getResourceName() + "/" + getResourceVersion();- }-- public String getResourceName() {- return "testResource";- }-- public String getResourceVersion() {- return "1";- }-- class DumbCsvDecoder implements BlobRepository.Decoder<TestObject> {- private final Map<String, String> dict = new HashMap<>();-- public DumbCsvDecoder() {}-- void processSimpleCsvRow(String string) {- String[] row = string.split(","); // dumbest csv parser ever... :)- getDict().put(row[0], row[1]);- }-- public Map<String, String> getDict() {- return dict;- }-- @Override- public TestObject decode(InputStream inputStream) {- // loading a tiny csv like:- //- // foo,bar- // baz,bam-- try (Stream<String> lines =- new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)).lines()) {- lines.forEach(this::processSimpleCsvRow);- } catch (Exception e) {- log.error("failed to read dictionary {}", getResourceName());- throw new RuntimeException("Cannot load dictionary ", e);- }-- assertEquals("bar", dict.get("foo"));- assertEquals("bam", dict.get("baz"));- if (log.isInfoEnabled()) {- log.info("Loaded {} using {}", getDict().size(), this.getClass().getClassLoader());- }-- // if we get here we have seen the data from the blob and all we need is to test that two- // collections are able to see the same object...- return new TestObject();- }- }-- public static class TestObject {- public static final String NEVER_UPDATED = "never updated";- private volatile String lastCollection = NEVER_UPDATED;-- public String getLastCollection() {- return this.lastCollection;- }-- public void setLastCollection(String lastCollection) {- this.lastCollection = lastCollection;- }- }-}diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.javaindex 57d5a6b47a8..90a7685d83e 100644--- a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java+++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java@@ -70,7 +70,7 @@ public class TestSchemaDesignerAPI extends SolrCloudTestCase implements SchemaDeconfigureCluster(1).addConfig(DEFAULT_CONFIGSET_NAME, new File(ExternalPaths.DEFAULT_CONFIGSET).toPath()).configure();- // SchemaDesignerAPI depends on the blob store+ // SchemaDesignerAPI depends on the blob store ".system" collection existing.CollectionAdminRequest.createCollection(BLOB_STORE_ID, 1, 1).process(cluster.getSolrClient());cluster.waitForActiveCollection(BLOB_STORE_ID, 1, 1);}diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adocindex a712bbe54e8..e1491a6d74f 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc@@ -55,3 +55,7 @@ has been removed. Please use `-Dsolr.hiddenSysProps` or the envVar `SOLR_HIDDEN_* The node configuration file `/solr.xml` can no longer be loaded from Zookeeper. Solr startup will fail if it is present.* The legacy Circuit Breaker named `CircuitBreakerManager`, is removed. Please use individual Circuit Breaker plugins instead.++* The `BlobRepository`, which was deprecated in 8x in favour of the `FileStore` approach is removed.+Users should migrate to the `FileStore` implementation (per node stored file) and the still existing `BlobHandler` (across the cluster storage backed by `.system` collection).+Please note this also removes the ability to share resource intensive objects across multiple cores as this feature was tied to the `BlobRepository` implementation.diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.javaindex f37038e1c6c..98899633bff 100644--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java@@ -19,6 +19,7 @@ package org.apache.solr.common.util;import static java.nio.charset.StandardCharsets.UTF_8;import static java.util.Collections.singletonList;import static java.util.concurrent.TimeUnit.NANOSECONDS;+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;import com.fasterxml.jackson.annotation.JsonAnyGetter;import java.io.ByteArrayInputStream;@@ -38,10 +39,13 @@ import java.lang.invoke.MethodType;import java.lang.reflect.Field;import java.lang.reflect.Method;import java.lang.reflect.Modifier;+import java.math.BigInteger;import java.net.URL;import java.nio.BufferOverflowException;import java.nio.ByteBuffer;import java.nio.charset.StandardCharsets;+import java.security.MessageDigest;+import java.security.NoSuchAlgorithmException;import java.util.AbstractMap;import java.util.ArrayList;import java.util.Arrays;@@ -52,8 +56,10 @@ import java.util.HashMap;import java.util.HashSet;import java.util.LinkedHashMap;import java.util.List;+import java.util.Locale;import java.util.Map;import java.util.Objects;+import java.util.Random;import java.util.Set;import java.util.TreeMap;import java.util.TreeSet;@@ -90,6 +96,31 @@ public class Utils {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ public static final Random RANDOM;++ static {+ // We try to make things reproducible in the context of our tests by initializing the random+ // instance based on the current seed+ String seed = System.getProperty("tests.seed");+ if (seed == null) {+ RANDOM = new Random();+ } else {+ RANDOM = new Random(seed.hashCode());+ }+ }++ public static String sha512Digest(ByteBuffer byteBuffer) {+ MessageDigest digest;+ try {+ digest = MessageDigest.getInstance("SHA-512");+ } catch (NoSuchAlgorithmException e) {+ // unlikely+ throw new SolrException(SERVER_ERROR, e);+ }+ digest.update(byteBuffer);+ return String.format(Locale.ROOT, "%0128x", new BigInteger(1, digest.digest()));+ }+@SuppressWarnings({"rawtypes"})public static Map getDeepCopy(Map<?, ?> map, int maxDepth) {return getDeepCopy(map, maxDepth, true, false);