Apache Solr open-source search software
SOLR-14151: fixed the classloading issue
noblepaul515608a087f8
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.javaindex 7a10f8964d0..532f1dd3896 100644--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java@@ -16,8 +16,12 @@*/package org.apache.solr.core;-import com.google.common.annotations.VisibleForTesting;-import java.io.*;+import java.io.Closeable;+import java.io.File;+import java.io.FileOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;import java.lang.invoke.MethodHandles;import java.lang.reflect.Constructor;import java.net.MalformedURLException;@@ -30,14 +34,26 @@ import java.nio.file.DirectoryStream;import java.nio.file.Files;import java.nio.file.Path;import java.nio.file.PathMatcher;-import java.util.*;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Collections;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.regex.Matcher;import java.util.regex.Pattern;import java.util.stream.Collectors;+import com.google.common.annotations.VisibleForTesting;+import com.google.common.collect.ImmutableSet;import org.apache.lucene.analysis.WordlistLoader;-import org.apache.lucene.analysis.util.*;+import org.apache.lucene.analysis.util.CharFilterFactory;+import org.apache.lucene.analysis.util.ResourceLoader;+import org.apache.lucene.analysis.util.ResourceLoaderAware;+import org.apache.lucene.analysis.util.TokenFilterFactory;+import org.apache.lucene.analysis.util.TokenizerFactory;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.DocValuesFormat;import org.apache.lucene.codecs.PostingsFormat;@@ -83,27 +99,6 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLprivate CoreContainer coreContainer;private PackageListeningClassLoader schemaLoader ;- private PackageListeningClassLoader createSchemaLoader() {- CoreContainer cc = getCoreContainer();- if (cc == null) {- //corecontainer not available . can't load from packages- return null;- }- return new PackageListeningClassLoader(cc, this, pkg -> {- if (getSolrConfig() == null) return null;- return getSolrConfig().maxPackageVersion(pkg);- }, () -> {- if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;- try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {- if (c != null) {- c.fetchLatestSchema();- }- }- });- }---private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());@@ -476,8 +471,9 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL}}}-Class<? extends T> clazz = null;+ clazz = getPackageClass(cname, expectedType);+ if(clazz != null) return clazz;try {// first try legacy analysis patterns, now replaced by Lucene's Analysis package:final Matcher m = legacyAnalysisPattern.matcher(cname);@@ -540,6 +536,24 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL}}+ private <T> Class<? extends T> getPackageClass(String cname, Class<T> expectedType) {+ PluginInfo.ClassName cName = PluginInfo.parseClassName(cname);+ if (cName.pkg == null) return null;+ ResourceLoaderAware aware = CURRENT_AWARE.get();+ if (aware != null) {+ //this is invoked from a component+ //let's check if it's a schema component+ @SuppressWarnings("rawtypes")+ Class type = assertAwareCompatibility(ResourceLoaderAware.class, aware);+ if (schemaResourceLoaderComponents.contains(type)) {+ //this is a schema component+ //lets use schema classloader+ return getSchemaLoader().findClass(cname, expectedType);+ }+ }+ return null;+ }+static final String[] empty = new String[0];@Override@@ -679,7 +693,13 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL}for (ResourceLoaderAware aware : arr) {- aware.inform(loader);+ CURRENT_AWARE.set(aware);+ try{+ aware.inform(loader);+ } finally {+ CURRENT_AWARE.remove();+ }+}}}@@ -749,6 +769,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLResourceLoaderAware.class, new Class<?>[]{// DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY// VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311+ // evaluate if this must go into schemaResourceLoaderComponentsCharFilterFactory.class,TokenFilterFactory.class,TokenizerFactory.class,@@ -758,11 +779,21 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL);}+ /**If these components are trying to load classes, use schema classloader+ *+ */+ @SuppressWarnings("rawtypes")+ private static final ImmutableSet<Class> schemaResourceLoaderComponents = ImmutableSet.of(+ CharFilterFactory.class,+ TokenFilterFactory.class,+ TokenizerFactory.class,+ FieldType.class);+/*** Utility function to throw an exception if the class is invalid*/@SuppressWarnings({"rawtypes"})- public static void assertAwareCompatibility(Class aware, Object obj) {+ public static Class assertAwareCompatibility(Class aware, Object obj) {Class[] valid = awareCompatibility.get(aware);if (valid == null) {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,@@ -770,7 +801,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL}for (Class v : valid) {if (v.isInstance(obj)) {- return;+ return v;}}StringBuilder builder = new StringBuilder();@@ -800,6 +831,25 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLreturn Collections.unmodifiableList(infoMBeans);}+ private PackageListeningClassLoader createSchemaLoader() {+ CoreContainer cc = getCoreContainer();+ if (cc == null) {+ //corecontainer not available . can't load from packages+ return null;+ }+ return new PackageListeningClassLoader(cc, this, pkg -> {+ if (getSolrConfig() == null) return null;+ return getSolrConfig().maxPackageVersion(pkg);+ }, () -> {+ if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;+ try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {+ if (c != null) {+ c.fetchLatestSchema();+ }+ }+ });+ }+public static void persistConfLocally(SolrResourceLoader loader, String resourceName, byte[] content) {// Persist locally@@ -831,4 +881,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL}}+ //This is to verify if this requires to use the schema classloader for classes loaded from packages+ public static final ThreadLocal<ResourceLoaderAware> CURRENT_AWARE = new ThreadLocal<>();+}diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.javaindex 9fd52431a44..f8c0c4fd1b4 100644--- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java+++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java@@ -66,10 +66,10 @@ import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.SolrNamedThreadFactory;import org.apache.solr.core.SolrConfig;import org.apache.solr.core.SolrResourceLoader;import org.apache.solr.rest.schema.FieldTypeXmlAdapter;-import org.apache.solr.common.util.SolrNamedThreadFactory;import org.apache.solr.util.FileUtils;import org.apache.solr.util.RTimer;import org.apache.zookeeper.CreateMode;@@ -1343,10 +1343,13 @@ public final class ManagedIndexSchema extends IndexSchema {TokenFilterFactory[] filters = chain.getTokenFilterFactories();for (TokenFilterFactory next : filters) {if (next instanceof ResourceLoaderAware) {+ SolrResourceLoader.CURRENT_AWARE.set((ResourceLoaderAware) next);try {((ResourceLoaderAware) next).inform(loader);} catch (IOException e) {throw new SolrException(ErrorCode.SERVER_ERROR, e);+ } finally {+ SolrResourceLoader.CURRENT_AWARE.remove();}}}diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java b/solr/core/src/java/org/apache/solr/schema/SchemaManager.javaindex 3731a176275..c04c9c36c77 100644--- a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java+++ b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java@@ -31,13 +31,13 @@ import org.apache.solr.cloud.ZkController;import org.apache.solr.cloud.ZkSolrResourceLoader;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.util.CommandOperation;import org.apache.solr.common.util.TimeSource;import org.apache.solr.core.CoreDescriptor;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrResourceLoader;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.rest.BaseSolrResource;-import org.apache.solr.common.util.CommandOperation;import org.apache.solr.util.TimeOut;import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;@@ -189,6 +189,7 @@ public class SchemaManager {mgr.managedIndexSchema = mgr.managedIndexSchema.addFieldTypes(singletonList(fieldType), false);return true;} catch (Exception e) {+ log.error("err", e);op.addError(getErrorStr(e));return false;}@@ -223,6 +224,7 @@ public class SchemaManager {mgr.managedIndexSchema = mgr.managedIndexSchema.addCopyFields(src, dests, maxChars);return true;} catch (Exception e) {+ log.error("err", e);op.addError(getErrorStr(e));return false;}@@ -240,6 +242,7 @@ public class SchemaManager {= mgr.managedIndexSchema.addFields(singletonList(field), Collections.emptyMap(), false);return true;} catch (Exception e) {+ log.error("err", e);op.addError(getErrorStr(e));return false;}@@ -257,6 +260,7 @@ public class SchemaManager {= mgr.managedIndexSchema.addDynamicFields(singletonList(field), Collections.emptyMap(), false);return true;} catch (Exception e) {+ log.error("err", e);op.addError(getErrorStr(e));return false;}@@ -275,6 +279,7 @@ public class SchemaManager {mgr.managedIndexSchema = mgr.managedIndexSchema.deleteFieldTypes(singleton(name));return true;} catch (Exception e) {+ log.error("err", e);op.addError(getErrorStr(e));return false;}@@ -331,6 +336,7 @@ public class SchemaManager {mgr.managedIndexSchema = mgr.managedIndexSchema.deleteDynamicFields(singleton(name));return true;} catch (Exception e) {+ log.error("err", e);op.addError(getErrorStr(e));return false;}@@ -346,6 +352,7 @@ public class SchemaManager {mgr.managedIndexSchema = mgr.managedIndexSchema.replaceFieldType(name, className, op.getDataMap());return true;} catch (Exception e) {+ log.error("err", e);op.addError(getErrorStr(e));return false;}@@ -366,6 +373,7 @@ public class SchemaManager {mgr.managedIndexSchema = mgr.managedIndexSchema.replaceField(name, ft, op.getValuesExcluding(NAME, TYPE));return true;} catch (Exception e) {+ log.error("err", e);op.addError(getErrorStr(e));return false;}diff --git a/solr/core/src/test-files/runtimecode/payload-component.jar.bin b/solr/core/src/test-files/runtimecode/payload-component.jar.binnew file mode 100644index 00000000000..47a4305f543Binary files /dev/null and b/solr/core/src/test-files/runtimecode/payload-component.jar.bin differdiff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.javaindex 8d8583c11a4..9473d4140e4 100644--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java@@ -19,13 +19,20 @@ package org.apache.solr.pkg;import java.io.IOException;import java.nio.ByteBuffer;-import java.util.*;+import java.util.Arrays;+import java.util.Collections;+import java.util.LinkedHashMap;+import java.util.Map;import java.util.concurrent.Callable;import org.apache.commons.codec.digest.DigestUtils;import org.apache.lucene.analysis.util.ResourceLoader;import org.apache.lucene.analysis.util.ResourceLoaderAware;-import org.apache.solr.client.solrj.*;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.SolrQuery;+import org.apache.solr.client.solrj.SolrRequest;+import org.apache.solr.client.solrj.SolrResponse;+import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.embedded.JettySolrRunner;import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;import org.apache.solr.client.solrj.impl.HttpSolrClient;@@ -69,9 +76,9 @@ import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;import static org.apache.solr.common.params.CommonParams.JAVABIN;import static org.apache.solr.common.params.CommonParams.WT;import static org.apache.solr.core.TestSolrConfigHandler.getFileContent;+import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;import static org.apache.solr.filestore.TestDistribPackageStore.readFile;import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;-import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;import static org.hamcrest.CoreMatchers.containsString;@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")@@ -655,10 +662,14 @@ public class TestPackages extends SolrCloudTestCase {postFileAndWait(cluster, "runtimecode/schema-plugins.jar.bin", FILE1,"iSRhrogDyt9P1htmSf/krh1kx9oty3TYyWm4GKHQGlb8a+X4tKCe9kKk+3tGs+bU9zq5JBZ5txNXsn96aZem5A==");+ String FILE2 = "/schemapkg/payload-component.jar";+ postFileAndWait(cluster, "runtimecode/payload-component.jar.bin", FILE2,+ "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q==");+Package.AddVersion add = new Package.AddVersion();add.version = "1.0";add.pkg = "schemapkg";- add.files = Arrays.asList(new String[]{FILE1});+ add.files = Arrays.asList(FILE1,FILE2);V2Request req = new V2Request.Builder("/cluster/package").forceV2(true).withMethod(SolrRequest.METHOD.POST)@@ -697,7 +708,7 @@ public class TestPackages extends SolrCloudTestCase {String tokenizer =" 'tokenizer' : { 'class':'schemapkg:my.pkg.MyWhitespaceTokenizerFactory' },\n";String filters =- " 'filters' : [{ 'class':'solr.ASCIIFoldingFilterFactory' }]\n";+ " 'filters' : [{ 'class':'solr.DelimitedPayloadTokenFilterFactory', 'encoder' : 'schemapkg:com.o19s.payloads.Base64Encoder'}]\n";String suffix = " }\n" +"}}";cluster.getSolrClient().request(new SolrRequest(SolrRequest.METHOD.POST, "/schema") {