Apache Solr open-source search software
SOLR-14155: Load all other SolrCore plugins from packages (#1666)
Noble Paul9466af576a4a
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex a62ca52eeee..7c8a26ff12d 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -249,6 +249,8 @@ Improvements* SOLR-15040: Improvements to postlogs timestamp handling (Joel Bernstein)+* SOLR-14155: Load all other SolrCore plugins from packages (noble)+Optimizations---------------------* SOLR-14975: Optimize CoreContainer.getAllCoreNames, getLoadedCoreNames and getCoreDescriptors. (Bruno Roustant)diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.javaindex 57366921c05..619e7c9771c 100644--- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java+++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java@@ -419,7 +419,7 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,final DirectoryFactory dirFactory;if (info != null) {log.debug(info.className);- dirFactory = config.getResourceLoader().newInstance(info.className, DirectoryFactory.class);+ dirFactory = config.getResourceLoader().newInstance (info, DirectoryFactory.class, true);// allow DirectoryFactory instances to access the CoreContainerdirFactory.initCoreContainer(cc);dirFactory.init(info.initArgs);diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.javaindex bdafa722690..427cd7eb13f 100644--- a/solr/core/src/java/org/apache/solr/core/PluginBag.java+++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java@@ -396,7 +396,6 @@ public class PluginBag<T> implements AutoCloseable {if (pluginInfo != null) return pluginInfo.className;return null;}-public PluginInfo getPluginInfo() {return pluginInfo;}diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.javaindex a388bf3a3d9..c01f4c4b5c2 100644--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java@@ -755,7 +755,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {IndexReaderFactory indexReaderFactory;PluginInfo info = solrConfig.getPluginInfo(IndexReaderFactory.class.getName());if (info != null) {- indexReaderFactory = resourceLoader.newInstance(info.className, IndexReaderFactory.class);+ indexReaderFactory = resourceLoader.newInstance(info, IndexReaderFactory.class, true);indexReaderFactory.init(info.initArgs);} else {indexReaderFactory = new StandardIndexReaderFactory();@@ -955,6 +955,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {this.solrConfig = configSet.getSolrConfig();this.resourceLoader = configSet.getSolrConfig().getResourceLoader();+ this.resourceLoader.initCore(this);IndexSchema schema = configSet.getIndexSchema();this.configSetProperties = configSet.getProperties();@@ -1241,7 +1242,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {private void checkVersionFieldExistsInSchema(IndexSchema schema, CoreDescriptor coreDescriptor) {if (null != coreDescriptor.getCloudDescriptor()) {- // we are evidently running in cloud mode.+ // we are evidently running in cloud mode.//// In cloud mode, version field is required for correct consistency// ideally this check would be more fine grained, and individual features@@ -1405,7 +1406,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {final PluginInfo info = solrConfig.getPluginInfo(CodecFactory.class.getName());final CodecFactory factory;if (info != null) {- factory = resourceLoader.newInstance(info.className, CodecFactory.class);+ factory = resourceLoader.newInstance( info, CodecFactory.class, true);factory.init(info.initArgs);} else {factory = new CodecFactory() {@@ -1443,8 +1444,8 @@ public final class SolrCore implements SolrInfoBean, Closeable {final StatsCache cache;PluginInfo pluginInfo = solrConfig.getPluginInfo(StatsCache.class.getName());if (pluginInfo != null && pluginInfo.className != null && pluginInfo.className.length() > 0) {- cache = createInitInstance(pluginInfo, StatsCache.class, null,- LocalStatsCache.class.getName());+ cache = resourceLoader.newInstance( pluginInfo, StatsCache.class, true);+ initPlugin(pluginInfo ,cache);if (log.isDebugEnabled()) {log.debug("Using statsCache impl: {}", cache.getClass().getName());}@@ -2134,7 +2135,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {newReader = currentReader;}- // for now, turn off caches if this is for a realtime reader+ // for now, turn off caches if this is for a realtime reader// (caches take a little while to instantiate)final boolean useCaches = !realtime;final String newName = realtime ? "realtime" : "main";@@ -3177,7 +3178,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {public void cleanupOldIndexDirectories(boolean reload) {final DirectoryFactory myDirFactory = getDirectoryFactory();final String myDataDir = getDataDir();- final String myIndexDir = getNewIndexDir(); // ensure the latest replicated index is protected+ final String myIndexDir = getNewIndexDir(); // ensure the latest replicated index is protectedfinal String coreName = getName();if (myDirFactory != null && myDataDir != null && myIndexDir != null) {Thread cleanupThread = new Thread(() -> {@@ -3280,6 +3281,9 @@ public final class SolrCore implements SolrInfoBean, Closeable {this.coreName = coreName;this.coreId = coreId;}+ public void reload() {+ coreContainer.reload(coreName, coreId);+ }public void withCore(Consumer<SolrCore> r) {try(SolrCore core = coreContainer.getCore(coreName, coreId)) {diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.javaindex fa183c62b72..288259ac6e2 100644--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java@@ -42,6 +42,7 @@ import java.util.List;import java.util.Map;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;+import java.util.function.Function;import java.util.regex.Matcher;import java.util.regex.Pattern;import java.util.stream.Collectors;@@ -64,6 +65,7 @@ import org.apache.solr.handler.component.SearchComponent;import org.apache.solr.handler.component.ShardHandlerFactory;import org.apache.solr.logging.DeprecationLog;import org.apache.solr.pkg.PackageListeningClassLoader;+import org.apache.solr.pkg.PackageLoader;import org.apache.solr.request.SolrRequestHandler;import org.apache.solr.response.QueryResponseWriter;import org.apache.solr.rest.RestManager;@@ -100,6 +102,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLprivate CoreContainer coreContainer;private PackageListeningClassLoader schemaLoader ;+ private PackageListeningClassLoader coreReloadingClassLoader ;private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());@@ -109,7 +112,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL// Provide a registry so that managed resources can register themselves while the XML configuration// documents are being parsed ... after all are registered, they are asked by the RestManager to// initialize themselves. This two-step process is required because not all resources are available- // (such as the SolrZkClient) when XML docs are being parsed.+ // (such as the SolrZkClient) when XML docs are being parsed.private RestManager.Registry managedResourceRegistry;/** @see #reloadLuceneSPI() */@@ -165,7 +168,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLif (instanceDir == null) {throw new NullPointerException("SolrResourceLoader instanceDir must be non-null");}-+this.instanceDir = instanceDir;log.debug("new SolrResourceLoader for directory: '{}'", this.instanceDir);@@ -645,15 +648,29 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL}}+ void initCore(SolrCore core) {+ this.coreName = core.getName();+ this.config = core.getSolrConfig();+ this.coreId = core.uniqueId;+ this.coreContainer = core.getCoreContainer();+ SolrCore.Provider coreProvider = core.coreProvider;++ this.coreReloadingClassLoader = new PackageListeningClassLoader(core.getCoreContainer(),+ this, s -> config.maxPackageVersion(s), null){+ @Override+ protected void doReloadAction(Ctx ctx) {+ log.info("Core reloading classloader issued reload for: {}/{} ", coreName, coreId);+ coreProvider.reload();+ }+ };+ core.getPackageListeners().addListener(coreReloadingClassLoader, true);++ }/*** Tell all {@link SolrCoreAware} instances about the SolrCore*/public void inform(SolrCore core) {- this.coreName = core.getName();- this.config = core.getSolrConfig();- this.coreId = core.uniqueId;- this.coreContainer = core.getCoreContainer();if(getSchemaLoader() != null) core.getPackageListeners().addListener(schemaLoader);// make a copy to avoid potential deadlock of a callback calling newInstance and trying to@@ -834,6 +851,47 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLpublic List<SolrInfoBean> getInfoMBeans() {return Collections.unmodifiableList(infoMBeans);}+ /**+ * Load a class using an appropriate {@link SolrResourceLoader} depending of the package on that class+ * @param registerCoreReloadListener register a listener for the package and reload the core if the package is changed.+ * Use this sparingly. This will result in core reloads across all the cores in+ * all collections using this configset+ */+ public <T> Class<? extends T> findClass( PluginInfo info, Class<T> type, boolean registerCoreReloadListener) {+ if(info.cName.pkg == null) return findClass(info.className, type);+ return _classLookup(info,+ (Function<PackageLoader.Package.Version, Class<? extends T>>) ver -> ver.getLoader().findClass(info.cName.className, type), registerCoreReloadListener);++ }+++ private <T> T _classLookup(PluginInfo info, Function<PackageLoader.Package.Version, T> fun, boolean registerCoreReloadListener ) {+ PluginInfo.ClassName cName = info.cName;+ if (registerCoreReloadListener) {+ if (coreReloadingClassLoader == null) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Core not set");+ }+ return fun.apply(coreReloadingClassLoader.findPackageVersion(cName, true));+ } else {+ return fun.apply(coreReloadingClassLoader.findPackageVersion(cName, false));+ }+ }++ /**+ *Create a n instance of a class using an appropriate {@link SolrResourceLoader} depending on the package of that class+ * @param registerCoreReloadListener register a listener for the package and reload the core if the package is changed.+ * Use this sparingly. This will result in core reloads across all the cores in+ * all collections using this configset+ */+ public <T> T newInstance(PluginInfo info, Class<T> type, boolean registerCoreReloadListener) {+ if(info.cName.pkg == null) {+ return newInstance(info.cName.className == null?+ type.getName():+ info.cName.className ,+ type);+ }+ return _classLookup( info, version -> version.getLoader().newInstance(info.cName.className, type), registerCoreReloadListener);+ }private PackageListeningClassLoader createSchemaLoader() {CoreContainer cc = getCoreContainer();diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.javaindex 0b97896f504..db8ec17bde5 100644--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java@@ -48,6 +48,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;import org.apache.solr.client.solrj.io.stream.expr.Expressible;import org.apache.solr.cloud.ZkController;import org.apache.solr.cloud.ZkSolrResourceLoader;+import org.apache.solr.common.MapSerializable;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.DocCollection;@@ -69,6 +70,7 @@ import org.apache.solr.core.RequestParams;import org.apache.solr.core.SolrConfig;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrResourceLoader;+import org.apache.solr.pkg.PackageAPI;import org.apache.solr.pkg.PackageListeners;import org.apache.solr.request.LocalSolrQueryRequest;import org.apache.solr.request.SolrQueryRequest;@@ -250,25 +252,28 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwaString componentName = req.getParams().get("componentName");if (componentName != null) {@SuppressWarnings({"rawtypes"})- Map map = (Map) val.get(parts.get(1));- if (map != null) {- Object o = map.get(componentName);- val.put(parts.get(1), makeMap(componentName, o));+ Map pluginNameVsPluginInfo = (Map) val.get(parts.get(1));+ if (pluginNameVsPluginInfo != null) {+ @SuppressWarnings({"rawtypes"})+ Object o = pluginNameVsPluginInfo instanceof MapSerializable ?+ pluginNameVsPluginInfo:+ pluginNameVsPluginInfo.get(componentName);+ @SuppressWarnings({"rawtypes"})+ Map pluginInfo = o instanceof MapSerializable? ((MapSerializable) o).toMap(new LinkedHashMap<>()): (Map) o;+ val.put(parts.get(1),pluginNameVsPluginInfo instanceof PluginInfo? pluginInfo : makeMap(componentName, pluginInfo));if (req.getParams().getBool("meta", false)) {// meta=true is asking for the package info of the plugin// We go through all the listeners and see if there is one registered for this pluginList<PackageListeners.Listener> listeners = req.getCore().getPackageListeners().getListeners();for (PackageListeners.Listener listener :listeners) {- PluginInfo info = listener.pluginInfo();- if(info == null) continue;- if (info.type.equals(parts.get(1)) && info.name.equals(componentName)) {- if (o instanceof Map) {- @SuppressWarnings({"rawtypes"})- Map m1 = (Map) o;- m1.put("_packageinfo_", listener.getPackageVersion(info.cName));+ Map<String, PackageAPI.PkgVersion> infos = listener.packageDetails();+ if(infos == null || infos.isEmpty()) continue;+ infos.forEach((s, mapWriter) -> {+ if(s.equals(pluginInfo.get("class"))) {+ (pluginInfo).put("_packageinfo_", mapWriter);}- }+ });}}}diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.javaindex f5c2474c4a8..c0c536fbca7 100644--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java@@ -16,16 +16,6 @@*/package org.apache.solr.handler.component;-import java.io.PrintWriter;-import java.io.StringWriter;-import java.lang.invoke.MethodHandles;-import java.util.ArrayList;-import java.util.HashSet;-import java.util.LinkedList;-import java.util.List;-import java.util.Set;-import java.util.concurrent.atomic.AtomicLong;-import org.apache.commons.lang3.StringUtils;import org.apache.lucene.index.ExitableDirectoryReader;import org.apache.lucene.search.TotalHits;@@ -44,6 +34,7 @@ import org.apache.solr.core.CoreContainer;import org.apache.solr.core.PluginInfo;import org.apache.solr.core.SolrCore;import org.apache.solr.handler.RequestHandlerBase;+import org.apache.solr.pkg.PackageAPI;import org.apache.solr.pkg.PackageListeners;import org.apache.solr.pkg.PackageLoader;import org.apache.solr.request.SolrQueryRequest;@@ -61,10 +52,13 @@ import org.apache.solr.util.plugin.SolrCoreAware;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-import static org.apache.solr.common.params.CommonParams.DISTRIB;-import static org.apache.solr.common.params.CommonParams.FAILURE;-import static org.apache.solr.common.params.CommonParams.PATH;-import static org.apache.solr.common.params.CommonParams.STATUS;+import java.io.PrintWriter;+import java.io.StringWriter;+import java.lang.invoke.MethodHandles;+import java.util.*;+import java.util.concurrent.atomic.AtomicLong;++import static org.apache.solr.common.params.CommonParams.*;/**@@ -162,8 +156,8 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware,}@Override- public PluginInfo pluginInfo() {- return null;+ public Map<String , PackageAPI.PkgVersion> packageDetails() {+ return Collections.emptyMap();}@Overridediff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.javaindex 1895b6d3069..1281d999236 100644--- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java@@ -47,6 +47,14 @@ public class PackageListeners {public synchronized void addListener(Listener listener) {listeners.add(new SoftReference<>(listener));+ }++ public synchronized void addListener(Listener listener, boolean addFirst) {+ if(addFirst) {+ listeners.add(0, new SoftReference<>(listener));+ } else {+ addListener(listener);+ }}@@ -71,7 +79,7 @@ public class PackageListeners {invokeListeners(pkgInfo, ctx);}} finally {- ctx.runLaterTasks(core::runAsync);+ ctx.runLaterTasks(r -> core.getCoreContainer().runAsync(r));MDCLoggingContext.clear();}}@@ -102,7 +110,10 @@ public class PackageListeners {/**Name of the package or null to listen to all package changes */String packageName();- PluginInfo pluginInfo();+ /** fetch the package versions of class names+ *+ */+ Map<String, PackageAPI.PkgVersion> packageDetails();/**A callback when the package is updated */void changed(PackageLoader.Package pkg, Ctx ctx);diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.javaindex bbdf66b6cc2..14617aa9e61 100644--- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java@@ -26,9 +26,8 @@ import org.apache.solr.common.cloud.SolrClassLoader;import org.apache.solr.core.SolrResourceLoader;import java.io.IOException;-import java.util.HashMap;-import java.util.Map;-import java.util.Objects;+import java.util.*;+import java.util.concurrent.ConcurrentHashMap;import java.util.function.Function;/*** A {@link SolrClassLoader} that is designed to listen to a set of packages.@@ -37,22 +36,29 @@ import java.util.function.Function;* */public class PackageListeningClassLoader implements SolrClassLoader , PackageListeners.Listener {private final CoreContainer coreContainer;- private final SolrResourceLoader coreResourceLoader;+ private final SolrClassLoader fallbackClassLoader;private final Function<String, String> pkgVersionSupplier;/** package name and the versions that we are tracking*/- private Map<String ,PackageAPI.PkgVersion> packageVersions = new HashMap<>(1);+ private Map<String ,PackageAPI.PkgVersion> packageVersions = new ConcurrentHashMap<>(1);+ private Map<String, String> classNameVsPackageName = new ConcurrentHashMap<>();private final Runnable onReload;+ /**+ * @param fallbackClassLoader The {@link SolrClassLoader} to use if no package is specified+ * @param pkgVersionSupplier Get the version configured for a given package+ * @param onReload The callback function that should be run if a package is updated+ */public PackageListeningClassLoader(CoreContainer coreContainer,- SolrResourceLoader coreResourceLoader,+ SolrClassLoader fallbackClassLoader,Function<String, String> pkgVersionSupplier,Runnable onReload) {this.coreContainer = coreContainer;- this.coreResourceLoader = coreResourceLoader;+ this.fallbackClassLoader = fallbackClassLoader;this.pkgVersionSupplier = pkgVersionSupplier;this.onReload = () -> {- packageVersions = new HashMap<>();+ packageVersions = new ConcurrentHashMap<>();+ classNameVsPackageName = new ConcurrentHashMap<>();onReload.run();};}@@ -61,18 +67,28 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis@Overridepublic <T> T newInstance(String cname, Class<T> expectedType, String... subpackages) {PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);- if(cName.pkg == null){- return coreResourceLoader.newInstance(cname, expectedType, subpackages);+ if(cName.pkg == null) {+ return fallbackClassLoader.newInstance(cname, expectedType, subpackages);} else {- PackageLoader.Package.Version version = findPkgVersion(cName);- return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subpackages));-+ PackageLoader.Package.Version version = findPackageVersion(cName, true);+ T obj = version.getLoader().newInstance(cName.className, expectedType, subpackages);+ classNameVsPackageName.put(cName.original, cName.pkg);+ return applyResourceLoaderAware(version, obj);}}- private PackageLoader.Package.Version findPkgVersion(PluginInfo.ClassName cName) {++ /**+ * This looks up for package and also listens for that package if required+ * @param cName The class name+ */+ public PackageLoader.Package.Version findPackageVersion(PluginInfo.ClassName cName, boolean registerListener) {PackageLoader.Package.Version theVersion = coreContainer.getPackageLoader().getPackage(cName.pkg).getLatest(pkgVersionSupplier.apply(cName.pkg));- packageVersions.put(cName.pkg, theVersion.getPkgVersion());+ if(registerListener) {+ classNameVsPackageName.put(cName.original, cName.pkg);+ PackageAPI.PkgVersion pkgVersion = theVersion.getPkgVersion();+ if(pkgVersion !=null) packageVersions.put(cName.pkg, pkgVersion);+ }return theVersion;}@@ -101,10 +117,12 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLispublic <T> T newInstance(String cname, Class<T> expectedType, String[] subPackages, Class[] params, Object[] args) {PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);if (cName.pkg == null) {- return coreResourceLoader.newInstance(cname, expectedType, subPackages, params, args);+ return fallbackClassLoader.newInstance(cname, expectedType, subPackages, params, args);} else {- PackageLoader.Package.Version version = findPkgVersion(cName);- return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subPackages, params, args));+ PackageLoader.Package.Version version = findPackageVersion(cName, true);+ T obj = version.getLoader().newInstance(cName.className, expectedType, subPackages, params, args);+ classNameVsPackageName.put(cName.original, cName.pkg);+ return applyResourceLoaderAware(version, obj);}}@@ -112,10 +130,12 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLispublic <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);if (cName.pkg == null) {- return coreResourceLoader.findClass(cname, expectedType);+ return fallbackClassLoader.findClass(cname, expectedType);} else {- PackageLoader.Package.Version version = findPkgVersion(cName);- return version.getLoader().findClass(cName.className, expectedType);+ PackageLoader.Package.Version version = findPackageVersion(cName, true);+ Class<? extends T> klas = version.getLoader().findClass(cName.className, expectedType);+ classNameVsPackageName.put(cName.original, cName.pkg);+ return klas;}}@@ -126,8 +146,10 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis}@Override- public PluginInfo pluginInfo() {- return null;+ public Map<String, PackageAPI.PkgVersion> packageDetails() {+ Map<String, PackageAPI.PkgVersion> result = new LinkedHashMap<>();+ classNameVsPackageName.forEach((k, v) -> result.put(k, packageVersions.get(v)));+ return result;}@Override@@ -146,6 +168,11 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis//no need to updatereturn;}+ doReloadAction(ctx);+ }++ protected void doReloadAction(Ctx ctx) {+ if(onReload == null) return;ctx.runLater(null, onReload);}}diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.javaindex 332de74f962..79626de61dc 100644--- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java+++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java@@ -19,6 +19,8 @@ package org.apache.solr.pkg;import java.io.IOException;import java.lang.invoke.MethodHandles;+import java.util.Collections;+import java.util.Map;import org.apache.lucene.util.ResourceLoaderAware;import org.apache.solr.common.MapWriter;@@ -57,8 +59,8 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {}@Override- public PluginInfo pluginInfo() {- return info;+ public Map<String, PackageAPI.PkgVersion> packageDetails() {+ return Collections.singletonMap(info.cName.original, pkgVersion.getPkgVersion());}@Override@@ -75,6 +77,12 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {});}+ public static <T> PluginBag.PluginHolder<T> createHolder(T inst, Class<T> type) {+ SolrConfig.SolrPluginInfo plugin = SolrConfig.classVsSolrPluginInfo.get(type.getName());+ PluginInfo info = new PluginInfo(plugin.tag, Collections.singletonMap("class", inst.getClass().getName()));+ return new PluginBag.PluginHolder<T>(info,inst);+ }+public static <T> PluginBag.PluginHolder<T> createHolder(PluginInfo info, SolrCore core, Class<T> type, String msg) {if(info.cName.pkg == null) {return new PluginBag.PluginHolder<T>(info, core.createInitInstance(info, type,msg, null));@@ -108,7 +116,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {if (log.isInfoEnabled()) {log.info("loading plugin: {} -> {} using package {}:{}",- pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion());+ pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion());}initNewInstance(newest, core);@@ -119,7 +127,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {@SuppressWarnings({"unchecked"})protected Object initNewInstance(PackageLoader.Package.Version newest, SolrCore core) {Object instance = SolrCore.createInstance(pluginInfo.className,- pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());+ pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());PluginBag.initInstance(instance, pluginInfo);handleAwareCallbacks(newest.getLoader(), instance, core);T old = inst;diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.javaindex d8c41c6bd84..1520f80620f 100644--- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java+++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java@@ -23,11 +23,15 @@ import java.util.HashMap;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import java.util.function.Supplier;import org.apache.solr.common.MapSerializable;import org.apache.solr.common.SolrException;import org.apache.solr.common.util.DOMUtil;import org.apache.solr.common.util.StrUtils;++import org.apache.solr.core.PluginInfo;+import org.apache.solr.core.SolrConfig;import org.apache.solr.core.SolrResourceLoader;import org.slf4j.Logger;@@ -46,11 +50,17 @@ import static org.apache.solr.common.params.CommonParams.NAME;*/public class CacheConfig implements MapSerializable{private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-+private String nodeName;++ /**+ * When this object is created, the core is not yet available . So, if the class is to be+ * loaded from a package we should have a corresponding core+ *+ */@SuppressWarnings({"rawtypes"})- private Class<? extends SolrCache> clazz;+ private Supplier<Class<? extends SolrCache>> clazz;private Map<String,String> args;private CacheRegenerator regenerator;@@ -64,9 +74,10 @@ public class CacheConfig implements MapSerializable{@SuppressWarnings({"rawtypes"})public CacheConfig(Class<? extends SolrCache> clazz, Map<String,String> args, CacheRegenerator regenerator) {- this.clazz = clazz;+ this.clazz = () -> clazz;this.args = args;this.regenerator = regenerator;+ this.nodeName = args.get(NAME);}public CacheRegenerator getRegenerator() {@@ -85,7 +96,7 @@ public class CacheConfig implements MapSerializable{Node node = nodes.item(i);if ("true".equals(DOMUtil.getAttrOrDefault(node, "enabled", "true"))) {CacheConfig config = getConfig(solrConfig, node.getNodeName(),- DOMUtil.toMap(node.getAttributes()), configPath);+ DOMUtil.toMap(node.getAttributes()), configPath);result.put(config.args.get(NAME), config);}}@@ -133,20 +144,32 @@ public class CacheConfig implements MapSerializable{SolrResourceLoader loader = solrConfig.getResourceLoader();config.cacheImpl = config.args.get("class");- if(config.cacheImpl == null) config.cacheImpl = "solr.CaffeineCache";+ if (config.cacheImpl == null) config.cacheImpl = "solr.CaffeineCache";+ config.clazz = new Supplier<>() {+ @SuppressWarnings("rawtypes")+ Class<SolrCache> loadedClass;++ @Override+ @SuppressWarnings("rawtypes")+ public Class<? extends SolrCache> get() {+ if (loadedClass != null) return loadedClass;+ return loadedClass = (Class<SolrCache>) loader.findClass(+ new PluginInfo("cache", Collections.singletonMap("class", config.cacheImpl)),+ SolrCache.class, true);+ }+ };config.regenImpl = config.args.get("regenerator");- config.clazz = loader.findClass(config.cacheImpl, SolrCache.class);if (config.regenImpl != null) {config.regenerator = loader.newInstance(config.regenImpl, CacheRegenerator.class);}-+return config;}@SuppressWarnings({"rawtypes"})public SolrCache newInstance() {try {- SolrCache cache = clazz.getConstructor().newInstance();+ SolrCache cache = clazz.get().getConstructor().newInstance();persistence[0] = cache.init(args, persistence[0], regenerator);return cache;} catch (Exception e) {@@ -158,11 +181,8 @@ public class CacheConfig implements MapSerializable{}@Override- @SuppressWarnings({"unchecked"})public Map<String, Object> toMap(Map<String, Object> map) {- @SuppressWarnings({"rawtypes"})- Map result = Collections.unmodifiableMap(args);- return result;+ return new HashMap<>(args);}public String getNodeName() {diff --git a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateHandler.javaindex 14b78af267a..ade92401370 100644--- a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java+++ b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java@@ -20,12 +20,7 @@ import java.io.IOException;import java.lang.invoke.MethodHandles;import java.util.Vector;-import org.apache.solr.core.DirectoryFactory;-import org.apache.solr.core.HdfsDirectoryFactory;-import org.apache.solr.core.PluginInfo;-import org.apache.solr.core.SolrCore;-import org.apache.solr.core.SolrEventListener;-import org.apache.solr.core.SolrInfoBean;+import org.apache.solr.core.*;import org.apache.solr.metrics.SolrMetricsContext;import org.apache.solr.schema.FieldType;import org.apache.solr.schema.SchemaField;@@ -108,7 +103,7 @@ public abstract class UpdateHandler implements SolrInfoBean {public UpdateHandler(SolrCore core) {this(core, null);}-+public UpdateHandler(SolrCore core, UpdateLog updateLog) {this.core=core;idField = core.getLatestSchema().getUniqueKeyField();@@ -123,8 +118,8 @@ public abstract class UpdateHandler implements SolrInfoBean {if (dirFactory instanceof HdfsDirectoryFactory) {ulog = new HdfsUpdateLog(((HdfsDirectoryFactory)dirFactory).getConfDir());} else {- String className = ulogPluginInfo.className == null ? UpdateLog.class.getName() : ulogPluginInfo.className;- ulog = core.getResourceLoader().newInstance(className, UpdateLog.class);+ ulog = ulogPluginInfo.className == null ? new UpdateLog():+ core.getResourceLoader().newInstance(ulogPluginInfo, UpdateLog.class, true);}if (!core.isReloaded() && !dirFactory.isPersistent()) {diff --git a/solr/core/src/test-files/solr/configsets/conf2/conf/schema.xml b/solr/core/src/test-files/solr/configsets/conf2/conf/schema.xmlnew file mode 100644index 00000000000..85f7ed35453--- /dev/null+++ b/solr/core/src/test-files/solr/configsets/conf2/conf/schema.xml@@ -0,0 +1,43 @@+<?xml version="1.0" encoding="UTF-8" ?>+<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->+<schema name="minimal" version="1.1">+ <fieldType name="string" class="solr.StrField"/>+ <fieldType name="int" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>+ <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>+ <dynamicField name="*" type="string" indexed="true" stored="true"/>+ <!-- for versioning -->+ <field name="_version_" type="long" indexed="true" stored="true"/>+ <field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>+ <field name="id" type="string" indexed="true" stored="true"/>+ <dynamicField name="*_s" type="string" indexed="true" stored="true" />+ <uniqueKey>id</uniqueKey>+ <query>+ <filterCache+ size="0"+ initialSize="0"+ autowarmCount="0"/>+ <queryResultCache+ size="0"+ initialSize="0"+ autowarmCount="0"/>+ <documentCache+ size="0"+ initialSize="0"+ autowarmCount="0"/>+ </query>+</schema>diff --git a/solr/core/src/test-files/solr/configsets/conf2/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/conf2/conf/solrconfig.xmlnew file mode 100644index 00000000000..e69d16ff17f--- /dev/null+++ b/solr/core/src/test-files/solr/configsets/conf2/conf/solrconfig.xml@@ -0,0 +1,68 @@+<?xml version="1.0" ?>++<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<!-- Minimal solrconfig.xml with /select, /admin and /update only -->++<config>++ <dataDir>${solr.data.dir:}</dataDir>++ <directoryFactory name="DirectoryFactory"+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>+ <schemaFactory class="ClassicIndexSchemaFactory"/>++ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>++ <updateHandler class="solr.DirectUpdateHandler2">+ <commitWithin>+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>+ </commitWithin>+ <updateLog class="${solr.ulog:solr.UpdateLog}"></updateLog>+ </updateHandler>++ <requestHandler name="/select" class="solr.SearchHandler">+ <lst name="defaults">+ <str name="echoParams">explicit</str>+ <str name="indent">true</str>+ <str name="df">text</str>+ </lst>++ </requestHandler>+ <query>+ <filterCache+ class = "mypkg:org.apache.solr.search.CaffeineCache"+ size="512"+ initialSize="512"+ autowarmCount="0" />++ <queryResultCache+ size="512"+ initialSize="512"+ autowarmCount="0" />++ <documentCache+ size="512"+ initialSize="512"+ autowarmCount="0" />+ </query>+ <indexConfig>+ <mergeScheduler class="${solr.mscheduler:org.apache.lucene.index.ConcurrentMergeScheduler}"/>+ </indexConfig>+</config>+diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.javaindex bea5d0c1104..2e908ef7458 100644--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java@@ -17,17 +17,6 @@package org.apache.solr.pkg;-import java.io.FileOutputStream;-import java.io.IOException;-import java.nio.ByteBuffer;-import java.util.Arrays;-import java.util.Collections;-import java.util.LinkedHashMap;-import java.util.Map;-import java.util.concurrent.Callable;-import java.util.zip.ZipEntry;-import java.util.zip.ZipOutputStream;-import org.apache.commons.codec.digest.DigestUtils;import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory;@@ -40,11 +29,7 @@ import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.embedded.JettySolrRunner;import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;import org.apache.solr.client.solrj.impl.HttpSolrClient;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.request.GenericSolrRequest;-import org.apache.solr.client.solrj.request.RequestWriter;-import org.apache.solr.client.solrj.request.UpdateRequest;-import org.apache.solr.client.solrj.request.V2Request;+import org.apache.solr.client.solrj.request.*;import org.apache.solr.client.solrj.request.beans.Package;import org.apache.solr.client.solrj.response.QueryResponse;import org.apache.solr.client.solrj.util.ClientUtils;@@ -75,13 +60,22 @@ import org.junit.After;import org.junit.Before;import org.junit.Test;+import java.io.FileOutputStream;+import java.io.IOException;+import java.nio.ByteBuffer;+import java.util.Arrays;+import java.util.Collections;+import java.util.LinkedHashMap;+import java.util.Map;+import java.util.concurrent.Callable;+import java.util.zip.ZipEntry;+import java.util.zip.ZipOutputStream;+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;import static org.apache.solr.common.params.CommonParams.JAVABIN;import static org.apache.solr.common.params.CommonParams.WT;import static org.apache.solr.core.TestSolrConfigHandler.getFileContent;-import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;-import static org.apache.solr.filestore.TestDistribPackageStore.readFile;-import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;+import static org.apache.solr.filestore.TestDistribPackageStore.*;@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")public class TestPackages extends SolrCloudTestCase {@@ -102,6 +96,69 @@ public class TestPackages extends SolrCloudTestCase {@JsonProperty("class")public String klass;}+++ public void testCoreReloadingPlugin() throws Exception {+ MiniSolrCloudCluster cluster =+ configureCluster(4)+ .withJettyConfig(jetty -> jetty.enableV2(true))+ .addConfig("conf", configset("conf2"))+ .configure();+ try {+ String FILE1 = "/mypkg/runtimelibs.jar";+ String COLLECTION_NAME = "testCoreReloadingPluginColl";+ byte[] derFile = readFile("cryptokeys/pub_key512.der");+ uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster);+ postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1,+ "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");++ Package.AddVersion add = new Package.AddVersion();+ add.version = "1.0";+ add.pkg = "mypkg";+ add.files = Arrays.asList(new String[]{FILE1});+ V2Request req = new V2Request.Builder("/cluster/package")+ .forceV2(true)+ .withMethod(SolrRequest.METHOD.POST)+ .withPayload(Collections.singletonMap("add", add))+ .build();++ req.process(cluster.getSolrClient());+ TestDistribPackageStore.assertResponseValues(10,+ () -> new V2Request.Builder("/cluster/package").+ withMethod(SolrRequest.METHOD.GET)+ .build().process(cluster.getSolrClient()),+ Utils.makeMap(+ ":result:packages:mypkg[0]:version", "1.0",+ ":result:packages:mypkg[0]:files[0]", FILE1+ ));++ CollectionAdminRequest+ .createCollection(COLLECTION_NAME, "conf", 2, 2)+ .process(cluster.getSolrClient());+ cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4);++ verifyComponent(cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", add.pkg, add.version);+++ add.version = "2.0";+ req.process(cluster.getSolrClient());+ TestDistribPackageStore.assertResponseValues(10,+ () -> new V2Request.Builder("/cluster/package").+ withMethod(SolrRequest.METHOD.GET)+ .build().process(cluster.getSolrClient()),+ Utils.makeMap(+ ":result:packages:mypkg[1]:version", "2.0",+ ":result:packages:mypkg[1]:files[0]", FILE1+ ));+ new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION_NAME);++ verifyComponent(cluster.getSolrClient(),+ COLLECTION_NAME, "query", "filterCache",+ "mypkg", "2.0" );+ } finally {+ cluster.shutdown();+ }+ }@Test@SuppressWarnings({"unchecked"})public void testPluginLoading() throws Exception {