Apache Solr open-source search software
SOLR-15258: ConfigSetService add CRUD operations, subsume ZkConfigManager (#23)
Nazerke Seidan1a922dc99aa2
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex a73487ab4d5..7ce63ffcd78 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -241,6 +241,10 @@ Other Changes* SOLR-15292: SignatureUpdateProcessorFactory will fail to initialize if it is used in a SolrCloud cluster in a way that isknown to be problematic with multiple replicas. (hossman)+* SOLR-15258: Consolidate ConfigSet handling (C.R.U.D.) into ConfigSetService to allow for viable+ alternative implementations of where ConfigSets come from.+ (Nazerke Seidan, David Smiley)+Bug Fixes---------------------* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order executiondiff --git a/solr/core/src/java/org/apache/solr/cloud/CloudConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/CloudConfigSetService.javadeleted file mode 100644index 88fbe781ed5..00000000000--- a/solr/core/src/java/org/apache/solr/cloud/CloudConfigSetService.java+++ /dev/null@@ -1,146 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.cloud;--import java.lang.invoke.MethodHandles;-import java.lang.ref.WeakReference;-import java.util.Map;-import java.util.concurrent.ConcurrentHashMap;--import org.apache.solr.client.solrj.cloud.SolrCloudManager;-import org.apache.solr.cloud.api.collections.CreateCollectionCmd;-import org.apache.solr.common.ConfigNode;-import org.apache.solr.common.SolrException;-import org.apache.solr.common.cloud.ZkConfigManager;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.cloud.ZooKeeperException;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.core.ConfigSetProperties;-import org.apache.solr.core.ConfigSetService;-import org.apache.solr.core.CoreDescriptor;-import org.apache.solr.core.SolrConfig;-import org.apache.solr.core.SolrResourceLoader;-import org.apache.zookeeper.KeeperException;-import org.apache.zookeeper.data.Stat;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--/**- * SolrCloud ConfigSetService impl.- */-public class CloudConfigSetService extends ConfigSetService {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- private Map<String, ConfigCacheEntry> cache = new ConcurrentHashMap<>();- private final ZkController zkController;-- public CloudConfigSetService(SolrResourceLoader loader, boolean shareSchema, ZkController zkController) {- super(loader, shareSchema);- this.zkController = zkController;- }-- public void storeConfig(String resource, ConfigNode config, int znodeVersion) {- cache.put(resource, new ConfigCacheEntry(config, znodeVersion));- }-- public ConfigNode getConfig(String resource, int znodeVersion) {- ConfigCacheEntry e = cache.get(resource);- if (e == null) return null;- ConfigNode configNode = e.configNode.get();- if (configNode == null) cache.remove(resource);- if (e.znodeVersion == znodeVersion) return configNode;- if (e.znodeVersion < znodeVersion) cache.remove(resource);- return null;- }-- @Override- public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) {- final String colName = cd.getCollectionName();-- // For back compat with cores that can create collections without the collections API- try {- if (!zkController.getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + colName, true)) {- // TODO remove this functionality or maybe move to a CLI mechanism- log.warn("Auto-creating collection (in ZK) from core descriptor (on disk). This feature may go away!");- CreateCollectionCmd.createCollectionZkNode(zkController.getSolrCloudManager().getDistribStateManager(), colName, cd.getCloudDescriptor().getParams());- }- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted auto-creating collection", e);- } catch (KeeperException e) {- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "Failure auto-creating collection", e);- }-- // The configSet is read from ZK and populated. Ignore CD's pre-existing configSet; only populated in standalone- final String configSetName;- try {- configSetName = zkController.getZkStateReader().readConfigName(colName);- cd.setConfigSet(configSetName);- } catch (KeeperException ex) {- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "Trouble resolving configSet for collection " + colName + ": " + ex.getMessage());- }-- return new ZkSolrResourceLoader(cd.getInstanceDir(), configSetName, parentLoader.getClassLoader(), zkController);- }-- @Override- @SuppressWarnings({"rawtypes"})- protected NamedList loadConfigSetFlags(CoreDescriptor cd, SolrResourceLoader loader) {- try {- return ConfigSetProperties.readFromResourceLoader(loader, ".");- } catch (Exception ex) {- log.debug("No configSet flags", ex);- return null;- }- }-- @Override- protected Long getCurrentSchemaModificationVersion(String configSet, SolrConfig solrConfig, String schemaFile) {- String zkPath = ZkConfigManager.CONFIGS_ZKNODE + "/" + configSet + "/" + schemaFile;- Stat stat;- try {- stat = zkController.getZkClient().exists(zkPath, null, true);- } catch (KeeperException e) {- log.warn("Unexpected exception when getting modification time of {}", zkPath, e);- return null; // debatable; we'll see an error soon if there's a real problem- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);- }- if (stat == null) { // not found- return null;- }- return (long) stat.getVersion();- }-- @Override- public String configSetName(CoreDescriptor cd) {- return "configset " + cd.getConfigSet();- }-- private static class ConfigCacheEntry {- final WeakReference<ConfigNode> configNode;- final int znodeVersion;-- private ConfigCacheEntry(ConfigNode configNode, int znodeVersion) {- this.configNode = new WeakReference<>(configNode);- this.znodeVersion = znodeVersion;- }- }- public SolrCloudManager getSolrCloudManager() {- return zkController.getSolrCloudManager();- }-}diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.javaindex d3819d0cda6..1724f6c8cb7 100644--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java@@ -90,7 +90,7 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessorfinal OverseerCollectionMessageHandler collMessageHandler = new OverseerCollectionMessageHandler(zkStateReader, myId, shardHandlerFactory, adminPath, stats, overseer, overseerNodePrioritizer);final OverseerConfigSetMessageHandler configMessageHandler = new OverseerConfigSetMessageHandler(- zkStateReader);+ zkStateReader, overseer.getCoreContainer()); //coreContainer is passed instead of configSetService as configSetService is loaded latereturn new OverseerMessageHandlerSelector() {@Overridepublic void close() throws IOException {diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.javaindex 9af5a6f8c89..3a6434e03d7 100644--- a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java@@ -30,15 +30,14 @@ import java.util.Set;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.cloud.DocCollection;-import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.ConfigSetParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.core.ConfigSetProperties;-import org.apache.zookeeper.CreateMode;+import org.apache.solr.core.ConfigSetService;+import org.apache.solr.core.CoreContainer;import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -71,6 +70,8 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {private ZkStateReader zkStateReader;+ private CoreContainer coreContainer;+// we essentially implement a read/write lock for the ConfigSet exclusivity as follows:// WRITE: CREATE/DELETE on the ConfigSet under operation// READ: for the Base ConfigSet being copied in CREATE.@@ -84,8 +85,9 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- public OverseerConfigSetMessageHandler(ZkStateReader zkStateReader) {+ public OverseerConfigSetMessageHandler(ZkStateReader zkStateReader, CoreContainer coreContainer) {this.zkStateReader = zkStateReader;+ this.coreContainer = coreContainer;this.configSetWriteWip = new HashSet<>();this.configSetReadWip = new HashSet<>();}@@ -227,17 +229,8 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {}@SuppressWarnings({"rawtypes"})- private NamedList getConfigSetProperties(String path) throws IOException {- byte[] oldPropsData = null;- try {- oldPropsData = zkStateReader.getZkClient().getData(path, null, null, true);- } catch (KeeperException.NoNodeException e) {- log.info("no existing ConfigSet properties found");- } catch (KeeperException | InterruptedException e) {- throw new IOException("Error reading old properties",- SolrZkClient.checkInterrupted(e));- }-+ private NamedList getConfigSetProperties(ConfigSetService configSetService, String configName, String propertyPath) throws IOException {+ byte[] oldPropsData = configSetService.downloadFileFromConfig(configName, propertyPath);if (oldPropsData != null) {InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream(oldPropsData), StandardCharsets.UTF_8);try {@@ -285,10 +278,6 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {return null;}- private String getPropertyPath(String configName, String propertyPath) {- return ZkConfigManager.CONFIGS_ZKNODE + "/" + configName + "/" + propertyPath;- }-private void createConfigSet(ZkNodeProps message) throws IOException {String configSetName = getTaskKey(message);if (configSetName == null || configSetName.length() == 0) {@@ -297,13 +286,12 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {String baseConfigSetName = message.getStr(BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);- ZkConfigManager configManager = new ZkConfigManager(zkStateReader.getZkClient());- if (configManager.configExists(configSetName)) {+ if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet already exists: " + configSetName);}// is there a base config that already exists- if (!configManager.configExists(baseConfigSetName)) {+ if (!coreContainer.getConfigSetService().checkConfigExists(baseConfigSetName)) {throw new SolrException(ErrorCode.BAD_REQUEST,"Base ConfigSet does not exist: " + baseConfigSetName);}@@ -313,25 +301,17 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {if (props != null) {// read the old config properties and do a merge, if necessary@SuppressWarnings({"rawtypes"})- NamedList oldProps = getConfigSetProperties(getPropertyPath(baseConfigSetName, propertyPath));+ NamedList oldProps = getConfigSetProperties(coreContainer.getConfigSetService(), baseConfigSetName, propertyPath);if (oldProps != null) {mergeOldProperties(props, oldProps);}}byte[] propertyData = getPropertyData(props);- Set<String> copiedToZkPaths = new HashSet<String>();try {- configManager.copyConfigDir(baseConfigSetName, configSetName, copiedToZkPaths);+ coreContainer.getConfigSetService().copyConfig(baseConfigSetName, configSetName);if (propertyData != null) {- try {- zkStateReader.getZkClient().makePath(- getPropertyPath(configSetName, propertyPath),- propertyData, CreateMode.PERSISTENT, null, false, true);- } catch (KeeperException | InterruptedException e) {- throw new IOException("Error writing new properties",- SolrZkClient.checkInterrupted(e));- }+ coreContainer.getConfigSetService().uploadFileToConfig(configSetName, propertyPath, propertyData, true);}} catch (Exception e) {// copying the config dir or writing the properties file may have failed.@@ -340,7 +320,7 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {// the entire baseConfig set with the old properties, including immutable,// that would make it impossible for the user to delete.try {- if (configManager.configExists(configSetName) && copiedToZkPaths.size() > 0) {+ if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {deleteConfigSet(configSetName, true);}} catch (IOException ioe) {@@ -360,8 +340,7 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {}private void deleteConfigSet(String configSetName, boolean force) throws IOException {- ZkConfigManager configManager = new ZkConfigManager(zkStateReader.getZkClient());- if (!configManager.configExists(configSetName)) {+ if (!coreContainer.getConfigSetService().checkConfigExists(configSetName)) {throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet does not exist to delete: " + configSetName);}@@ -380,7 +359,7 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;@SuppressWarnings({"rawtypes"})- NamedList properties = getConfigSetProperties(getPropertyPath(configSetName, propertyPath));+ NamedList properties = getConfigSetProperties(coreContainer.getConfigSetService(), configSetName, propertyPath);if (properties != null) {Object immutable = properties.get(ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG);boolean isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;@@ -388,6 +367,6 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {throw new SolrException(ErrorCode.BAD_REQUEST, "Requested delete of immutable ConfigSet: " + configSetName);}}- configManager.deleteConfigDir(configSetName);+ coreContainer.getConfigSetService().deleteConfig(configSetName);}}diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.javaindex ce2f613dfc4..a54be75141f 100644--- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java+++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java@@ -41,7 +41,8 @@ import org.apache.commons.io.FileUtils;import org.apache.commons.io.IOUtils;import org.apache.solr.common.cloud.ClusterProperties;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;+import org.apache.solr.common.cloud.ZkMaintenanceUtils;+import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.CoreContainer;import org.apache.solr.util.CLIO;import org.apache.zookeeper.CreateMode;@@ -71,7 +72,7 @@ public class ZkCLI implements CLIO {static final String UPCONFIG = "upconfig";static final String EXCLUDE_REGEX_SHORT = "x";static final String EXCLUDE_REGEX = "excluderegex";- static final String EXCLUDE_REGEX_DEFAULT = ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX;+ static final String EXCLUDE_REGEX_DEFAULT = ConfigSetService.UPLOAD_FILENAME_EXCLUDE_REGEX;private static final String COLLECTION = "collection";private static final String CLEAR = "clear";private static final String LIST = "list";@@ -206,13 +207,14 @@ public class ZkCLI implements CLIO {}CoreContainer cc = new CoreContainer(Paths.get(solrHome), new Properties());+ cc.setCoreConfigService(new ZkConfigSetService(zkClient));if(!ZkController.checkChrootPath(zkServerAddress, true)) {stdout.println("A chroot was specified in zkHost but the znode doesn't exist. ");System.exit(1);}- ZkController.bootstrapConf(zkClient, cc);+ ConfigSetService.bootstrapConf(cc);// No need to close the CoreContainer, as it wasn't started// up in the first place...@@ -231,9 +233,8 @@ public class ZkCLI implements CLIO {stdout.println("A chroot was specified in zkHost but the znode doesn't exist. ");System.exit(1);}- ZkConfigManager configManager = new ZkConfigManager(zkClient);final Pattern excludePattern = Pattern.compile(excludeExpr);- configManager.uploadConfigDir(Paths.get(confDir), confName, excludePattern);+ ZkMaintenanceUtils.uploadToZK(zkClient, Paths.get(confDir), ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName, excludePattern);} else if (line.getOptionValue(CMD).equalsIgnoreCase(DOWNCONFIG)) {if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {stdout.println("-" + CONFDIR + " and -" + CONFNAME@@ -242,8 +243,7 @@ public class ZkCLI implements CLIO {}String confDir = line.getOptionValue(CONFDIR);String confName = line.getOptionValue(CONFNAME);- ZkConfigManager configManager = new ZkConfigManager(zkClient);- configManager.downloadConfigDir(confName, Paths.get(confDir));+ ZkMaintenanceUtils.downloadFromZK(zkClient, ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName, Paths.get(confDir));} else if (line.getOptionValue(CMD).equalsIgnoreCase(LINKCONFIG)) {if (!line.hasOption(COLLECTION) || !line.hasOption(CONFNAME)) {stdout.println("-" + COLLECTION + " and -" + CONFNAMEdiff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.javanew file mode 100644index 00000000000..5c0c22b9a36--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java@@ -0,0 +1,324 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cloud;++import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.Collections;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.Objects;++import org.apache.solr.client.solrj.cloud.SolrCloudManager;+import org.apache.solr.cloud.api.collections.CreateCollectionCmd;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.cloud.ZkMaintenanceUtils;+import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.cloud.ZooKeeperException;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.Utils;+import org.apache.solr.core.ConfigSetProperties;+import org.apache.solr.core.ConfigSetService;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.CoreDescriptor;+import org.apache.solr.core.SolrConfig;+import org.apache.solr.core.SolrResourceLoader;+import org.apache.zookeeper.CreateMode;+import org.apache.zookeeper.KeeperException;+import org.apache.zookeeper.data.Stat;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/**+ * SolrCloud Zookeeper ConfigSetService impl.+ */+public class ZkConfigSetService extends ConfigSetService {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ private final ZkController zkController;+ private final SolrZkClient zkClient;+ /** ZkNode where named configs are stored */+ public static final String CONFIGS_ZKNODE = "/configs";++ public ZkConfigSetService(CoreContainer cc) {+ super(cc.getResourceLoader(), cc.getConfig().hasSchemaCache());+ this.zkController = cc.getZkController();+ this.zkClient = cc.getZkController().getZkClient();+ }++ /** This is for ZkCLI and some tests */+ public ZkConfigSetService(SolrZkClient zkClient) {+ super(null, false);+ this.zkController = null;+ this.zkClient = zkClient;+ }++ @Override+ public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) {+ final String colName = cd.getCollectionName();++ // For back compat with cores that can create collections without the collections API+ try {+ if (!zkClient.exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + colName, true)) {+ // TODO remove this functionality or maybe move to a CLI mechanism+ log.warn("Auto-creating collection (in ZK) from core descriptor (on disk). This feature may go away!");+ CreateCollectionCmd.createCollectionZkNode(+ zkController.getSolrCloudManager().getDistribStateManager(),+ colName,+ cd.getCloudDescriptor().getParams(),+ zkController.getCoreContainer().getConfigSetService());+ }+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted auto-creating collection", e);+ } catch (KeeperException e) {+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "Failure auto-creating collection", e);+ }++ // The configSet is read from ZK and populated. Ignore CD's pre-existing configSet; only populated in standalone+ final String configSetName;+ try {+ configSetName = zkController.getZkStateReader().readConfigName(colName);+ cd.setConfigSet(configSetName);+ } catch (KeeperException ex) {+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "Trouble resolving configSet for collection " + colName + ": " + ex.getMessage());+ }++ return new ZkSolrResourceLoader(cd.getInstanceDir(), configSetName, parentLoader.getClassLoader(), zkController);+ }++ @Override+ @SuppressWarnings({"rawtypes"})+ protected NamedList loadConfigSetFlags(CoreDescriptor cd, SolrResourceLoader loader) {+ try {+ return ConfigSetProperties.readFromResourceLoader(loader, ".");+ } catch (Exception ex) {+ log.debug("No configSet flags", ex);+ return null;+ }+ }++ @Override+ protected Long getCurrentSchemaModificationVersion(String configSet, SolrConfig solrConfig, String schemaFile) {+ String zkPath = CONFIGS_ZKNODE + "/" + configSet + "/" + schemaFile;+ Stat stat;+ try {+ stat = zkClient.exists(zkPath, null, true);+ } catch (KeeperException e) {+ log.warn("Unexpected exception when getting modification time of {}", zkPath, e);+ return null; // debatable; we'll see an error soon if there's a real problem+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ }+ if (stat == null) { // not found+ return null;+ }+ return (long) stat.getVersion();+ }++ @Override+ public String configSetName(CoreDescriptor cd) {+ return "configset " + cd.getConfigSet();+ }++ @Override+ public boolean checkConfigExists(String configName) throws IOException {+ try {+ Boolean existsConfig = zkClient.exists(CONFIGS_ZKNODE + "/" + configName, true);+ if (existsConfig == null) return false;+ return existsConfig;+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error checking whether config exists",+ SolrZkClient.checkInterrupted(e));+ }+ }++ @Override+ public void deleteConfig(String configName) throws IOException {+ try {+ zkClient.clean(CONFIGS_ZKNODE + "/" + configName);+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error deleting config",+ SolrZkClient.checkInterrupted(e));+ }+ }++ @Override+ public void deleteFilesFromConfig(String configName, List<String> filesToDelete) throws IOException {+ Objects.requireNonNull(filesToDelete);+ try {+ for (String fileToDelete : filesToDelete) {+ if (fileToDelete.endsWith("/")) {+ fileToDelete = fileToDelete.substring(0, fileToDelete.length()-1);+ }+ zkClient.clean(CONFIGS_ZKNODE + "/" + configName + "/" + fileToDelete);+ }+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error deleting files in config", SolrZkClient.checkInterrupted(e));+ }+ }++ public void copyConfig(String fromConfig, String toConfig) throws IOException {+ String fromConfigPath = CONFIGS_ZKNODE + "/" + fromConfig;+ String toConfigPath = CONFIGS_ZKNODE + "/" + toConfig;+ try {+ copyData(fromConfigPath, toConfigPath);+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error config " + fromConfig + " to " + toConfig,+ SolrZkClient.checkInterrupted(e));+ }+ copyConfigDirFromZk(fromConfigPath, toConfigPath);+ }++ @Override+ public void uploadConfig(String configName, Path dir) throws IOException {+ zkClient.uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, ConfigSetService.UPLOAD_FILENAME_EXCLUDE_PATTERN);+ }++ @Override+ public void uploadFileToConfig(String configName, String fileName, byte[] data, boolean overwriteOnExists) throws IOException {+ String filePath = CONFIGS_ZKNODE + "/" + configName + "/" + fileName;+ try {+ // if createNew is true then zkClient#makePath failOnExists is set to false+ zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true);+ } catch (KeeperException.NodeExistsException nodeExistsException) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ "The path " + filePath + " for configSet " + configName + " already exists. "+ + "In order to overwrite, provide overwrite=true or use an HTTP PUT with the V2 API.");+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error creating file in config", SolrZkClient.checkInterrupted(e));+ }+ }++ @Override+ public void setConfigMetadata(String configName, Map<String, Object> data) throws IOException {+ try {+ zkClient.makePath(CONFIGS_ZKNODE + "/" + configName, Utils.toJSON(data), CreateMode.PERSISTENT, null, false,true);+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error setting config metadata", SolrZkClient.checkInterrupted(e));+ }+ }++ @Override+ public Map<String, Object> getConfigMetadata(String configName) throws IOException {+ try {+ @SuppressWarnings("unchecked")+ Map<String, Object> data =+ (Map<String, Object>)+ Utils.fromJSON(zkClient.getData(CONFIGS_ZKNODE + "/" + configName, null, null, true));+ if (data == null) return new HashMap<>();+ return data;+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error getting config metadata", SolrZkClient.checkInterrupted(e));+ }+ }++ @Override+ public void downloadConfig(String configName, Path dir) throws IOException {+ zkClient.downloadFromZK(CONFIGS_ZKNODE + "/" + configName, dir);+ }++ @Override+ public byte[] downloadFileFromConfig(String configName, String filePath) throws IOException {+ try {+ return zkClient.getData(CONFIGS_ZKNODE + "/" + configName + "/" + filePath, null, null, true);+ } catch (KeeperException.NoNodeException e) {+ return null;+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error downloading file from config", SolrZkClient.checkInterrupted(e));+ }+ }++ @Override+ public List<String> listConfigs() throws IOException {+ try {+ return zkClient.getChildren(CONFIGS_ZKNODE, null, true);+ } catch (KeeperException.NoNodeException e) {+ return Collections.emptyList();+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error listing configs", SolrZkClient.checkInterrupted(e));+ }+ }++ @Override+ public List<String> getAllConfigFiles(String configName) throws IOException {+ String zkPath = CONFIGS_ZKNODE + "/" + configName;+ try {+ List<String> filePaths = new ArrayList<>();+ ZkMaintenanceUtils.traverseZkTree(zkClient, zkPath, ZkMaintenanceUtils.VISIT_ORDER.VISIT_POST, filePaths::add);+ filePaths.remove(zkPath);++ String prevPath = "";+ for (int i = 0; i < filePaths.size(); i++) {+ String currPath = filePaths.get(i);++ // stripping /configs/configName/+ assert currPath.startsWith(zkPath + "/");+ currPath = currPath.substring(zkPath.length() + 1);++ // if currentPath is a directory, concatenate '/'+ if (prevPath.startsWith(currPath)) {+ currPath = currPath + "/";+ }+ prevPath = currPath;+ filePaths.set(i, currPath);+ }+ Collections.sort(filePaths);+ return filePaths;+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error getting all configset files", SolrZkClient.checkInterrupted(e));+ }+ }++ // This method is used by configSetUploadTool and CreateTool to resolve the configset directory.+ // Check several possibilities:+ // 1> confDir/solrconfig.xml exists+ // 2> confDir/conf/solrconfig.xml exists+ // 3> configSetDir/confDir/conf/solrconfig.xml exists (canned configs)++ private void copyConfigDirFromZk(String fromZkPath, String toZkPath) throws IOException {+ try {+ List<String> files = zkClient.getChildren(fromZkPath, null, true);+ for (String file : files) {+ List<String> children = zkClient.getChildren(fromZkPath + "/" + file, null, true);+ if (children.size() == 0) {+ copyData(fromZkPath + "/" + file, toZkPath + "/" + file);+ } else {+ copyConfigDirFromZk(fromZkPath + "/" + file, toZkPath + "/" + file);+ }+ }+ } catch (KeeperException | InterruptedException e) {+ throw new IOException("Error copying nodes from zookeeper path " + fromZkPath + " to " + toZkPath,+ SolrZkClient.checkInterrupted(e));+ }+ }++ private void copyData(String fromZkFilePath, String toZkFilePath) throws KeeperException, InterruptedException {+ log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);+ byte[] data = zkClient.getData(fromZkFilePath, null, null, true);+ zkClient.makePath(toZkFilePath, data, true);+ }++ public SolrCloudManager getSolrCloudManager() {+ return zkController.getSolrCloudManager();+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.javaindex 4cafe666e99..9c928926e45 100644--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java@@ -17,7 +17,6 @@package org.apache.solr.cloud;import java.io.Closeable;-import java.io.File;import java.io.IOException;import java.lang.invoke.MethodHandles;import java.net.InetAddress;@@ -25,8 +24,6 @@ import java.net.NetworkInterface;import java.net.URLEncoder;import java.net.UnknownHostException;import java.nio.charset.StandardCharsets;-import java.nio.file.Path;-import java.nio.file.Paths;import java.util.ArrayList;import java.util.Arrays;import java.util.Collection;@@ -51,7 +48,6 @@ import java.util.concurrent.atomic.AtomicReference;import java.util.function.Supplier;import com.google.common.base.Strings;-import org.apache.commons.lang3.StringUtils;import org.apache.solr.client.solrj.cloud.SolrCloudManager;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.impl.HttpSolrClient;@@ -83,11 +79,9 @@ import org.apache.solr.core.CoreContainer;import org.apache.solr.core.CoreDescriptor;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrCoreInitializationException;-import org.apache.solr.handler.admin.ConfigSetsHandler;import org.apache.solr.handler.component.HttpShardHandler;import org.apache.solr.logging.MDCLoggingContext;import org.apache.solr.search.SolrIndexSearcher;-import org.apache.solr.servlet.SolrDispatchFilter;import org.apache.solr.update.UpdateLog;import org.apache.solr.util.RTimer;import org.apache.solr.util.RefCounted;@@ -751,15 +745,6 @@ public class ZkController implements Closeable {}- /**- * Returns true if config file exists- */- public boolean configFileExists(String collection, String fileName)- throws KeeperException, InterruptedException {- Stat stat = zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + collection + "/" + fileName, null, true);- return stat != null;- }-/*** @return information about the cluster from ZooKeeper*/@@ -791,22 +776,6 @@ public class ZkController implements Closeable {return cloudManager;}- /**- * Returns config file data (in bytes)- */- public byte[] getConfigFileData(String zkConfigName, String fileName)- throws KeeperException, InterruptedException {- String zkPath = ZkConfigManager.CONFIGS_ZKNODE + "/" + zkConfigName + "/" + fileName;- byte[] bytes = zkClient.getData(zkPath, null, null, true);- if (bytes == null) {- log.error("Config file contains no data:{}", zkPath);- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,- "Config file contains no data:" + zkPath);- }-- return bytes;- }-// normalize host removing any url scheme.// input can be null, host, or url_prefix://hostprivate String normalizeHostName(String host) {@@ -889,48 +858,10 @@ public class ZkController implements Closeable {cmdExecutor.ensureExists(ZkStateReader.SOLR_SECURITY_CONF_PATH, emptyJson, zkClient);if (zkClient.getACL(ZkStateReader.SOLR_SECURITY_CONF_PATH, null, true).equals(OPEN_ACL_UNSAFE)) {log.warn("Contents of zookeeper /security.json are world-readable;" +- " consider setting up ACLs as described in https://solr.apache.org/guide/zookeeper-access-control.html");- }- bootstrapDefaultConfigSet(zkClient);- }-- private static void bootstrapDefaultConfigSet(SolrZkClient zkClient) throws KeeperException, InterruptedException, IOException {- if (zkClient.exists("/configs/_default", true) == false) {- String configDirPath = getDefaultConfigDirPath();- if (configDirPath == null) {- log.warn("The _default configset could not be uploaded. Please provide 'solr.default.confdir' parameter that points to a configset {} {}"- , "intended to be the default. Current 'solr.default.confdir' value:"- , System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE));- } else {- ZkMaintenanceUtils.upConfig(zkClient, Paths.get(configDirPath), ConfigSetsHandler.DEFAULT_CONFIGSET_NAME);- }+ " consider setting up ACLs as described in https://solr.apache.org/guide/zookeeper-access-control.html");}}- /**- * Gets the absolute filesystem path of the _default configset to bootstrap from.- * First tries the sysprop "solr.default.confdir". If not found, tries to find- * the _default dir relative to the sysprop "solr.install.dir".- * Returns null if not found anywhere.- *- * @lucene.internal- * @see SolrDispatchFilter#SOLR_DEFAULT_CONFDIR_ATTRIBUTE- */- public static String getDefaultConfigDirPath() {- String configDirPath = null;- String serverSubPath = "solr" + File.separator +- "configsets" + File.separator + "_default" +- File.separator + "conf";- String subPath = File.separator + "server" + File.separator + serverSubPath;- if (System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE) != null && new File(System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE)).exists()) {- configDirPath = new File(System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE)).getAbsolutePath();- } else if (System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE) != null &&- new File(System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE) + subPath).exists()) {- configDirPath = new File(System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE) + subPath).getAbsolutePath();- }- return configDirPath;- }-private void init() {try {createClusterZkNodes(zkClient);@@ -2031,31 +1962,6 @@ public class ZkController implements Closeable {}- /**- * If in SolrCloud mode, upload config sets for each SolrCore in solr.xml.- */- public static void bootstrapConf(SolrZkClient zkClient, CoreContainer cc) throws IOException {-- ZkConfigManager configManager = new ZkConfigManager(zkClient);-- //List<String> allCoreNames = cfg.getAllCoreNames();- List<CoreDescriptor> cds = cc.getCoresLocator().discover(cc);-- if (log.isInfoEnabled()) {- log.info("bootstrapping config for {} cores into ZooKeeper using solr.xml from {}", cds.size(), cc.getSolrHome());- }-- for (CoreDescriptor cd : cds) {- String coreName = cd.getName();- String confName = cd.getCollectionName();- if (StringUtils.isEmpty(confName))- confName = coreName;- Path udir = cd.getInstanceDir().resolve("conf");- log.info("Uploading directory {} with name {} for solrCore {}", udir, confName, coreName);- configManager.uploadConfigDir(udir, confName);- }- }-public ZkDistributedQueue getOverseerJobQueue() {if (distributedClusterStateUpdater.isDistributedStateUpdate()) {throw new IllegalStateException("Cluster is configured with distributed state update, not expecting the queue to be retrieved");diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java b/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.javaindex b52d6bbde92..a3e3c8cfca1 100644--- a/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java+++ b/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java@@ -24,7 +24,6 @@ import java.lang.invoke.MethodHandles;import java.nio.file.Path;import org.apache.solr.common.SolrException.ErrorCode;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.cloud.ZooKeeperException;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrResourceLoader;@@ -58,7 +57,7 @@ public class ZkSolrResourceLoader extends SolrResourceLoader {ZkController zooKeeperController) {super(instanceDir, parent);this.zkController = zooKeeperController;- configSetZkPath = ZkConfigManager.CONFIGS_ZKNODE + "/" + configSet;+ configSetZkPath = ZkConfigSetService.CONFIGS_ZKNODE + "/" + configSet;}/**diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.javaindex 2f5dd6a49d6..72dfe5d6cad 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java@@ -121,7 +121,7 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {log.info("Starting to backup ZK data for backupName={}", backupName);//Download the configs- backupMgr.downloadConfigDir(configName);+ backupMgr.downloadConfigDir(configName, cc.getConfigSetService());//Save the collection's state. Can be part of the monolithic clusterstate.json or a individual state.json//Since we don't want to distinguish we extract the state and back it up as a separate jsondiff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.javaindex 18190319664..433e8bb434d 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java@@ -230,7 +230,7 @@ public class CollApiCmds {String configName = (String) message.getProperties().remove(CollectionAdminParams.COLL_CONF);if (configName != null) {- CollectionHandlingUtils.validateConfigOrThrowSolrException(ccc.getSolrCloudManager(), configName);+ CollectionHandlingUtils.validateConfigOrThrowSolrException(ccc.getCoreContainer().getConfigSetService(), configName);CollectionHandlingUtils.createConfNode(ccc.getSolrCloudManager().getDistribStateManager(), configName, collectionName);new ReloadCollectionCmd(ccc).call(clusterState, new ZkNodeProps(NAME, collectionName), results);diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.javaindex 77075664c45..a52d9edea58 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java@@ -37,7 +37,6 @@ import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.cloud.AlreadyExistsException;import org.apache.solr.client.solrj.cloud.BadVersionException;import org.apache.solr.client.solrj.cloud.DistribStateManager;-import org.apache.solr.client.solrj.cloud.SolrCloudManager;import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;import org.apache.solr.client.solrj.impl.HttpSolrClient;import org.apache.solr.client.solrj.request.AbstractUpdateRequest;@@ -58,6 +57,7 @@ import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.common.util.StrUtils;import org.apache.solr.common.util.TimeSource;import org.apache.solr.common.util.Utils;+import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.backup.BackupId;import org.apache.solr.core.backup.repository.BackupRepository;import org.apache.solr.handler.component.ShardHandler;@@ -300,8 +300,8 @@ public class CollectionHandlingUtils {return new AddReplicaCmd(ccc).addReplica(clusterState, message, results, onComplete);}- static void validateConfigOrThrowSolrException(SolrCloudManager cloudManager, String configName) throws IOException, KeeperException, InterruptedException {- boolean isValid = cloudManager.getDistribStateManager().hasData(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName);+ static void validateConfigOrThrowSolrException(ConfigSetService configSetService, String configName) throws IOException {+ boolean isValid = configSetService.checkConfigExists(configName);if (!isValid) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can not find the specified config set: " + configName);}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.javaindex ecf6004ee99..a6e3bbde861 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java@@ -23,7 +23,6 @@ import java.lang.invoke.MethodHandles;import java.util.ArrayList;import java.util.Collections;import java.util.HashMap;-import java.util.HashSet;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;@@ -57,13 +56,13 @@ import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.common.util.TimeSource;import org.apache.solr.common.util.Utils;+import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.CoreContainer;import org.apache.solr.handler.component.ShardHandler;import org.apache.solr.handler.component.ShardRequest;import org.apache.solr.util.TimeOut;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;-import org.apache.zookeeper.KeeperException.NoNodeException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -117,7 +116,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No config set found to associate with the collection.");}- CollectionHandlingUtils.validateConfigOrThrowSolrException(ccc.getSolrCloudManager(), configName);+ CollectionHandlingUtils.validateConfigOrThrowSolrException(ccc.getCoreContainer().getConfigSetService(), configName);String router = message.getStr("router.name", DocRouter.DEFAULT_NAME);@@ -145,7 +144,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {}}- createCollectionZkNode(stateManager, collectionName, collectionParams);+ createCollectionZkNode(stateManager, collectionName, collectionParams, ccc.getCoreContainer().getConfigSetService());// Note that in code below there are two main execution paths: Overseer based cluster state updates and distributed// cluster state updates (look for isDistributedStateUpdate() conditions).@@ -469,29 +468,25 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {return shardNames;}- String getConfigName(String coll, ZkNodeProps message) throws KeeperException, InterruptedException {+ String getConfigName(String coll, ZkNodeProps message) throws IOException {String configName = message.getStr(COLL_CONF);if (configName == null) {// if there is only one conf, use thatList<String> configNames = null;- try {- configNames = ccc.getZkStateReader().getZkClient().getChildren(ZkConfigManager.CONFIGS_ZKNODE, null, true);- if (configNames.contains(DEFAULT_CONFIGSET_NAME)) {- if (CollectionAdminParams.SYSTEM_COLL.equals(coll)) {- return coll;- } else {- String intendedConfigSetName = getSuffixedNameForAutoGeneratedConfigSet(coll);- copyDefaultConfigSetTo(configNames, intendedConfigSetName);- return intendedConfigSetName;- }- } else if (configNames != null && configNames.size() == 1) {- configName = configNames.get(0);- // no config set named, but there is only 1 - use it- log.info("Only one config set found in zk - using it: {}", configName);+ configNames = ccc.getCoreContainer().getConfigSetService().listConfigs();+ if (configNames.contains(DEFAULT_CONFIGSET_NAME)) {+ if (CollectionAdminParams.SYSTEM_COLL.equals(coll)) {+ return coll;+ } else {+ String intendedConfigSetName = getSuffixedNameForAutoGeneratedConfigSet(coll);+ copyDefaultConfigSetTo(configNames, intendedConfigSetName);+ return intendedConfigSetName;}- } catch (KeeperException.NoNodeException e) {-+ } else if (configNames != null && configNames.size() == 1) {+ configName = configNames.get(0);+ // no config set named, but there is only 1 - use it+ log.info("Only one config set found in zk - using it: {}", configName);}}return "".equals(configName)? null: configName;@@ -501,8 +496,6 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {* Copies the _default configset to the specified configset name (overwrites if pre-existing)*/private void copyDefaultConfigSetTo(List<String> configNames, String targetConfig) {- ZkConfigManager configManager = new ZkConfigManager(ccc.getZkStateReader().getZkClient());-// if a configset named collection exists, re-use itif (configNames.contains(targetConfig)) {log.info("There exists a configset by the same name as the collection we're trying to create: {}, re-using it.", targetConfig);@@ -510,13 +503,13 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {}// Copy _default into targetConfigtry {- configManager.copyConfigDir(DEFAULT_CONFIGSET_NAME, targetConfig, new HashSet<>());+ ccc.getCoreContainer().getConfigSetService().copyConfig(DEFAULT_CONFIGSET_NAME, targetConfig);} catch (Exception e) {throw new SolrException(ErrorCode.INVALID_STATE, "Error while copying _default to " + targetConfig, e);}}- public static void createCollectionZkNode(DistribStateManager stateManager, String collection, Map<String,String> params) {+ public static void createCollectionZkNode(DistribStateManager stateManager, String collection, Map<String,String> params, ConfigSetService configSetService) {log.debug("Check for collection zkNode: {}", collection);String collectionPath = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;// clean up old terms node@@ -541,7 +534,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {// if the config name wasn't passed in, use the defaultif (!collectionProps.containsKey(ZkController.CONFIGNAME_PROP)) {// users can create the collection node and conf link ahead of time, or this may return another option- getConfName(stateManager, collection, collectionPath, collectionProps);+ getConfName(stateManager, collection, collectionPath, collectionProps, configSetService);}} else if (System.getProperty("bootstrap_confdir") != null) {@@ -566,7 +559,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {// the conf name should should be the collection name of this corecollectionProps.put(ZkController.CONFIGNAME_PROP, collection);} else {- getConfName(stateManager, collection, collectionPath, collectionProps);+ getConfName(stateManager, collection, collectionPath, collectionProps, configSetService);}collectionProps.remove(ZkStateReader.NUM_SHARDS_PROP); // we don't put numShards in the collections properties@@ -602,7 +595,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {}- private static void getConfName(DistribStateManager stateManager, String collection, String collectionPath, Map<String,Object> collectionProps) throws IOException,+ private static void getConfName(DistribStateManager stateManager, String collection, String collectionPath, Map<String,Object> collectionProps, ConfigSetService configSetService) throws IOException,KeeperException, InterruptedException {// check for configNamelog.debug("Looking for collection configName");@@ -626,8 +619,8 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {}try {- configNames = stateManager.listData(ZkConfigManager.CONFIGS_ZKNODE);- } catch (NoSuchElementException | NoNodeException e) {+ configNames = configSetService.listConfigs();+ } catch (NoSuchElementException e) {// just keep trying}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.javaindex bee8de2f371..6612deda1fb 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java@@ -191,7 +191,7 @@ public class DeleteCollectionCmd implements CollApiCmds.CollectionApiCommand {if (!configSetIsUsedByOtherCollection) {// delete the config set- zkStateReader.getConfigManager().deleteConfigDir(configSetName);+ ccc.getCoreContainer().getConfigSetService().deleteConfig(configSetName);}}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.javaindex a5727dc9fda..fa79e5a3829 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java@@ -86,11 +86,11 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,private volatile boolean isClosed;public OverseerCollectionMessageHandler(ZkStateReader zkStateReader, String myId,- final HttpShardHandlerFactory shardHandlerFactory,- String adminPath,- Stats stats,- Overseer overseer,- OverseerNodePrioritizer overseerPrioritizer) {+ final HttpShardHandlerFactory shardHandlerFactory,+ String adminPath,+ Stats stats,+ Overseer overseer,+ OverseerNodePrioritizer overseerPrioritizer) {this.zkStateReader = zkStateReader;this.shardHandlerFactory = shardHandlerFactory;this.adminPath = adminPath;diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.javaindex 860605cda46..d8c1dcb3a6f 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java@@ -39,6 +39,7 @@ import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.common.util.StrUtils;import org.apache.solr.common.util.Utils;+import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.backup.BackupManager;import org.apache.solr.core.backup.BackupProperties;@@ -224,8 +225,8 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {// Avoiding passing RestoreContext arounduploadConfig(rc.backupProperties.getConfigName(),rc.restoreConfigName,- rc.zkStateReader,- rc.backupManager);+ rc.backupManager,+ rc.container.getConfigSetService());log.info("Starting restore into collection={} with backup_name={} at location={}", rc.restoreCollectionName, rc.backupName,rc.location);@@ -274,13 +275,14 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {assert totalReplicasPerShard > 0;}- private void uploadConfig(String configName, String restoreConfigName, ZkStateReader zkStateReader, BackupManager backupMgr) throws IOException {- if (zkStateReader.getConfigManager().configExists(restoreConfigName)) {+ private void uploadConfig(String configName, String restoreConfigName, BackupManager backupMgr, ConfigSetService configSetService) throws IOException {+ if (configSetService.checkConfigExists(restoreConfigName)) {log.info("Using existing config {}", restoreConfigName);//TODO add overwrite option?} else {log.info("Uploading config {}", restoreConfigName);- backupMgr.uploadConfigDir(configName, restoreConfigName);++ backupMgr.uploadConfigDir(configName, restoreConfigName, configSetService);}}diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.javaindex 409851a22a3..f5d5c60ca05 100644--- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java+++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java@@ -16,77 +16,183 @@*/package org.apache.solr.core;-import java.io.FileNotFoundException;+import java.io.File;import java.io.IOException;-import java.io.InputStream;import java.lang.invoke.MethodHandles;import java.lang.reflect.Constructor;-import java.nio.file.Files;import java.nio.file.Path;+import java.nio.file.Paths;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.regex.Pattern;import com.github.benmanes.caffeine.cache.Cache;import com.github.benmanes.caffeine.cache.Caffeine;-import org.apache.solr.cloud.CloudConfigSetService;+import org.apache.solr.cloud.ZkConfigSetService;import org.apache.solr.cloud.ZkController;import org.apache.solr.cloud.ZkSolrResourceLoader;-import org.apache.solr.common.ConfigNode;import org.apache.solr.common.SolrException;+import org.apache.solr.common.StringUtils;import org.apache.solr.common.util.NamedList;+import org.apache.solr.handler.admin.ConfigSetsHandler;import org.apache.solr.schema.IndexSchema;import org.apache.solr.schema.IndexSchemaFactory;-import org.apache.solr.util.DOMConfigNode;-import org.apache.solr.util.DataConfigNode;-import org.apache.solr.util.SystemIdResolver;+import org.apache.solr.servlet.SolrDispatchFilter;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-import org.xml.sax.InputSource;-import org.xml.sax.SAXException;--import javax.xml.parsers.ParserConfigurationException;--import static org.apache.solr.schema.IndexSchema.SCHEMA;-import static org.apache.solr.schema.IndexSchema.SLASH;/**- * Service class used by the CoreContainer to load ConfigSets for use in SolrCore- * creation.+ * Service class used by the CoreContainer to load ConfigSets for use in SolrCore creation.*/public abstract class ConfigSetService {+ public static final String UPLOAD_FILENAME_EXCLUDE_REGEX = "^\\..*$";+ public static final Pattern UPLOAD_FILENAME_EXCLUDE_PATTERN = Pattern.compile(UPLOAD_FILENAME_EXCLUDE_REGEX);private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());public static ConfigSetService createConfigSetService(CoreContainer coreContainer) {+ final ConfigSetService configSetService = instantiate(coreContainer);+ try {+ bootstrapDefaultConfigSet(configSetService);+ } catch (UnsupportedOperationException e) {+ log.info("_default config couldn't be uploaded");+ } catch (IOException e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR, "_default config couldn't be uploaded ", e);+ }+ return configSetService;+ }- NodeConfig nodeConfig = coreContainer.getConfig();- SolrResourceLoader loader = coreContainer.getResourceLoader();- ZkController zkController = coreContainer.getZkController();+ private static ConfigSetService instantiate(CoreContainer coreContainer) {+ final NodeConfig nodeConfig = coreContainer.getConfig();+ final SolrResourceLoader loader = coreContainer.getResourceLoader();+ final ZkController zkController = coreContainer.getZkController();- String configSetServiceClass = nodeConfig.getConfigSetServiceClass();+ final String configSetServiceClass = nodeConfig.getConfigSetServiceClass();- if(configSetServiceClass != null){+ if (configSetServiceClass != null) {try {Class<? extends ConfigSetService> clazz = loader.findClass(configSetServiceClass, ConfigSetService.class);Constructor<? extends ConfigSetService> constructor = clazz.getConstructor(CoreContainer.class);return constructor.newInstance(coreContainer);} catch (Exception e) {- throw new RuntimeException("create configSetService instance faild,configSetServiceClass:" + configSetServiceClass, e);+ throw new RuntimeException("create configSetService instance failed, configSetServiceClass:" + configSetServiceClass, e);}- }else if(zkController == null){- return new Standalone(loader, nodeConfig.hasSchemaCache(), nodeConfig.getConfigSetBaseDirectory());- }else{- return new CloudConfigSetService(loader, nodeConfig.hasSchemaCache(), zkController);+ } else if (zkController == null) {+ return new FileSystemConfigSetService(coreContainer);+ } else {+ return new ZkConfigSetService(coreContainer);}+ }+ private static void bootstrapDefaultConfigSet(ConfigSetService configSetService) throws IOException {+ if (configSetService.checkConfigExists("_default") == false) {+ String configDirPath = getDefaultConfigDirPath();+ if (configDirPath == null) {+ log.warn(+ "The _default configset could not be uploaded. Please provide 'solr.default.confdir' parameter that points to a configset {} {}",+ "intended to be the default. Current 'solr.default.confdir' value:",+ System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE));+ } else {+ configSetService.uploadConfig(ConfigSetsHandler.DEFAULT_CONFIGSET_NAME, Paths.get(configDirPath));+ }+ }}- protected final SolrResourceLoader parentLoader;+ /**+ * Gets the absolute filesystem path of the _default configset to bootstrap from. First tries the+ * sysprop "solr.default.confdir". If not found, tries to find the _default dir relative to the+ * sysprop "solr.install.dir". Returns null if not found anywhere.+ *+ * @lucene.internal+ * @see SolrDispatchFilter#SOLR_DEFAULT_CONFDIR_ATTRIBUTE+ */+ public static String getDefaultConfigDirPath() {+ String configDirPath = null;+ String serverSubPath =+ "solr"+ + File.separator+ + "configsets"+ + File.separator+ + "_default"+ + File.separator+ + "conf";+ String subPath = File.separator + "server" + File.separator + serverSubPath;+ if (System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE) != null+ && new File(System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE))+ .exists()) {+ configDirPath =+ new File(System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE))+ .getAbsolutePath();+ } else if (System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE) != null+ && new File(System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE) + subPath)+ .exists()) {+ configDirPath =+ new File(System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE) + subPath)+ .getAbsolutePath();+ }+ return configDirPath;+ }- /** Optional cache of schemas, key'ed by a bunch of concatenated things */- private final Cache<String, IndexSchema> schemaCache;+ // Order is important here since "confDir" may be+ // 1> a full path to the parent of a solrconfig.xml or parent of /conf/solrconfig.xml+ // 2> one of the canned config sets only, e.g. _default+ // and trying to assemble a path for configsetDir/confDir is A Bad Idea. if confDir is a full path.+ public static Path getConfigsetPath(String confDir, String configSetDir) throws IOException {++ // A local path to the source, probably already includes "conf".+ Path ret = Paths.get(confDir, "solrconfig.xml").normalize();+ if (Files.exists(ret)) {+ return Paths.get(confDir).normalize();+ }++ // a local path to the parent of a "conf" directory+ ret = Paths.get(confDir, "conf", "solrconfig.xml").normalize();+ if (Files.exists(ret)) {+ return Paths.get(confDir, "conf").normalize();+ }++ // one of the canned configsets.+ ret = Paths.get(configSetDir, confDir, "conf", "solrconfig.xml").normalize();+ if (Files.exists(ret)) {+ return Paths.get(configSetDir, confDir, "conf").normalize();+ }++ throw new IllegalArgumentException(String.format(Locale.ROOT,+ "Could not find solrconfig.xml at %s, %s or %s",+ Paths.get(configSetDir, "solrconfig.xml").normalize().toAbsolutePath().toString(),+ Paths.get(configSetDir, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString(),+ Paths.get(configSetDir, confDir, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString()+ ));+ }++ /** If in SolrCloud mode, upload configSets for each SolrCore in solr.xml. */+ public static void bootstrapConf(CoreContainer cc) throws IOException {+ // List<String> allCoreNames = cfg.getAllCoreNames();+ List<CoreDescriptor> cds = cc.getCoresLocator().discover(cc);++ if (log.isInfoEnabled()) {+ log.info(+ "bootstrapping config for {} cores into ZooKeeper using solr.xml from {}",+ cds.size(),+ cc.getSolrHome());+ }++ for (CoreDescriptor cd : cds) {+ String coreName = cd.getName();+ String confName = cd.getCollectionName();+ if (StringUtils.isEmpty(confName)) confName = coreName;+ Path udir = cd.getInstanceDir().resolve("conf");+ log.info("Uploading directory {} with name {} for solrCore {}", udir, confName, coreName);+ cc.getConfigSetService().uploadConfig(confName, udir);+ }+ }/*** Load the ConfigSet for a core+ ** @param dcore the core's CoreDescriptor* @return a ConfigSet*/@@ -119,8 +225,14 @@ public abstract class ConfigSetService {}+ protected final SolrResourceLoader parentLoader;++ /** Optional cache of schemas, key'ed by a bunch of concatenated things */+ private final Cache<String, IndexSchema> schemaCache;+/*** Create a new ConfigSetService+ ** @param loader the CoreContainer's resource loader* @param shareSchema should we share the IndexSchema among cores of same config?*/@@ -131,6 +243,7 @@ public abstract class ConfigSetService {/*** Create a SolrConfig object for a core+ ** @param cd the core's CoreDescriptor* @param loader the core's resource loader* @param isTrusted is the configset trusted?@@ -142,6 +255,7 @@ public abstract class ConfigSetService {/*** Create an IndexSchema object for a core. It might be a cached lookup.+ ** @param cd the core's CoreDescriptor* @param solrConfig the core's SolrConfig* @return an IndexSchema@@ -215,73 +329,110 @@ public abstract class ConfigSetService {*/public abstract String configSetName(CoreDescriptor cd);- public interface ConfigResource {+ /**+ * Upload files from a given path to config+ *+ * @param configName the config name+ * @param dir {@link Path} to the files+ * @throws IOException if an I/O error occurs or the path does not exist+ */+ public abstract void uploadConfig(String configName, Path dir) throws IOException;- ConfigNode get() throws Exception;+ /**+ * Upload a file to config+ * If file does not exist, it will be uploaded+ * If createNew param is set to true then file be overwritten+ *+ * @param configName the name to give the config+ * @param fileName the name of the file+ * @param data the content of the file+ * @param overwriteOnExists if true then file will be overwritten+ * @throws SolrException if file exists and overwriteOnExists == false+ */+ public abstract void uploadFileToConfig(String configName, String fileName, byte[] data, boolean overwriteOnExists) throws IOException;- }- public static ConfigNode getParsedSchema(InputStream is, SolrResourceLoader loader, String name) throws IOException, SAXException, ParserConfigurationException {- XmlConfigFile schemaConf = null;- InputSource inputSource = new InputSource(is);- inputSource.setSystemId(SystemIdResolver.createSystemIdFromResourceName(name));- schemaConf = new XmlConfigFile(loader, SCHEMA, inputSource, SLASH + SCHEMA + SLASH, null);- return new DataConfigNode(new DOMConfigNode(schemaConf.getDocument().getDocumentElement()));+ /**+ * Download all files from this config to the filesystem at dir+ *+ * @param configName the config to download+ * @param dir the {@link Path} to write files under+ */+ public abstract void downloadConfig(String configName, Path dir) throws IOException;- }+ /**+ * Download a file from config+ * If the file does not exist, it returns null+ *+ * @param configName the name of the config+ * @param filePath the file to download+ * @return the content of the file+ */+ public abstract byte[] downloadFileFromConfig(String configName, String filePath) throws IOException;/**- * The Solr standalone version of ConfigSetService.+ * Copy a config*- * Loads a ConfigSet defined by the core's configSet property,- * looking for a directory named for the configSet property value underneath- * a base directory. If no configSet property is set, loads the ConfigSet- * instead from the core's instance directory.+ * @param fromConfig the config to copy from+ * @param toConfig the config to copy to*/- public static class Standalone extends ConfigSetService {+ public abstract void copyConfig(String fromConfig, String toConfig) throws IOException;- private final Path configSetBase;+ /**+ * Check whether a config exists+ *+ * @param configName the config to check if it exists+ * @return whether the config exists or not+ */+ public abstract boolean checkConfigExists(String configName) throws IOException ;- public Standalone(SolrResourceLoader loader, boolean shareSchema, Path configSetBase) {- super(loader, shareSchema);- this.configSetBase = configSetBase;- }+ /**+ * Delete a config (recursively deletes its files if not empty)+ *+ * @param configName the config to delete+ */+ public abstract void deleteConfig(String configName) throws IOException;- @Override- public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) {- Path instanceDir = locateInstanceDir(cd);- SolrResourceLoader solrResourceLoader = new SolrResourceLoader(instanceDir, parentLoader.getClassLoader());- return solrResourceLoader;- }+ /**+ * Delete files in config+ *+ * @param configName the name of the config+ * @param filesToDelete a list of file paths to delete+ */+ public abstract void deleteFilesFromConfig(String configName, List<String> filesToDelete) throws IOException;- @Override- public String configSetName(CoreDescriptor cd) {- return (cd.getConfigSet() == null ? "instancedir " : "configset ") + locateInstanceDir(cd);- }+ /**+ * Set the config metadata+ * If config does not exist, it will be created and set metadata on it+ * Else metadata will be replaced with the provided metadata+ *+ * @param configName the config name+ * @param data the metadata to be set on config+ */+ public abstract void setConfigMetadata(String configName, Map<String, Object> data) throws IOException;- protected Path locateInstanceDir(CoreDescriptor cd) {- String configSet = cd.getConfigSet();- if (configSet == null)- return cd.getInstanceDir();- Path configSetDirectory = configSetBase.resolve(configSet);- if (!Files.isDirectory(configSetDirectory))- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,- "Could not load configuration from directory " + configSetDirectory);- return configSetDirectory;- }+ /**+ * Get the config metadata (mutable, non-null)+ *+ * @param configName the config name+ * @return the config metadata+ */+ public abstract Map<String, Object> getConfigMetadata(String configName) throws IOException;- @Override- protected Long getCurrentSchemaModificationVersion(String configSet, SolrConfig solrConfig, String schemaFileName) {- Path schemaFile = solrConfig.getResourceLoader().getConfigPath().resolve(schemaFileName);- try {- return Files.getLastModifiedTime(schemaFile).toMillis();- } catch (FileNotFoundException e) {- return null; // acceptable- } catch (IOException e) {- log.warn("Unexpected exception when getting modification time of {}", schemaFile, e);- return null; // debatable; we'll see an error soon if there's a real problem- }- }+ /**+ * List the names of configs (non-null)+ *+ * @return list of config names+ */+ public abstract List<String> listConfigs() throws IOException;- }+ /**+ * Get the names of the files in config including dirs (mutable, non-null)+ * sorted lexicographically+ * e.g. solrconfig.xml, lang/, lang/stopwords_en.txt+ *+ * @param configName the config name+ * @return list of file name paths in the config+ */+ public abstract List<String> getAllConfigFiles(String configName) throws IOException;}diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex c4fc7f1ae84..0e6126edd22 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -747,7 +747,7 @@ public class CoreContainer {reloadSecurityProperties();warnUsersOfInsecureSettings();this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins());-+ coreConfigService = ConfigSetService.createConfigSetService(this);createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class);createHandler(ZK_STATUS_PATH, ZookeeperStatusHandler.class.getName(), ZookeeperStatusHandler.class);collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class);@@ -792,7 +792,6 @@ public class CoreContainer {metricManager.loadReporters(metricReporters, loader, this, null, null, SolrInfoBean.Group.jvm);metricManager.loadReporters(metricReporters, loader, this, null, null, SolrInfoBean.Group.jetty);- coreConfigService = ConfigSetService.createConfigSetService(this);containerProperties.putAll(cfg.getSolrProperties());@@ -2041,6 +2040,10 @@ public class CoreContainer {return this.coreConfigService;}+ public void setCoreConfigService(ConfigSetService configSetService) {+ this.coreConfigService = configSetService;+ }+public String getHostName() {return this.hostName;}diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.javanew file mode 100644index 00000000000..9db6245a36d--- /dev/null+++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java@@ -0,0 +1,148 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.core;++import java.io.FileNotFoundException;+import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.List;+import java.util.Map;+import java.util.stream.Collectors;+import java.util.stream.Stream;++import org.apache.solr.common.SolrException;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/**+ * Solr Standalone File System ConfigSetService impl.+ *+ * <p>+ * Loads a ConfigSet defined by the core's configSet property, looking for a directory named for+ * the configSet property value underneath a base directory. If no configSet property is set, loads+ * the ConfigSet instead from the core's instance directory.+ */+public class FileSystemConfigSetService extends ConfigSetService {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ private final Path configSetBase;++ public FileSystemConfigSetService(CoreContainer cc) {+ super(cc.getResourceLoader(), cc.getConfig().hasSchemaCache());+ this.configSetBase = cc.getConfig().getConfigSetBaseDirectory();+ }++ @Override+ public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) {+ Path instanceDir = locateInstanceDir(cd);+ SolrResourceLoader solrResourceLoader = new SolrResourceLoader(instanceDir, parentLoader.getClassLoader());+ return solrResourceLoader;+ }++ @Override+ public String configSetName(CoreDescriptor cd) {+ return (cd.getConfigSet() == null ? "instancedir " : "configset ") + locateInstanceDir(cd);+ }++ @Override+ public boolean checkConfigExists(String configName) throws IOException {+ Path configSetDirectory = configSetBase.resolve(configName);+ return Files.isDirectory(configSetDirectory);+ }++ @Override+ public void deleteConfig(String configName) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public void deleteFilesFromConfig(String configName, List<String> filesToDelete) throws IOException {+ throw new UnsupportedOperationException();+ }++ public void copyConfig(String fromConfig, String toConfig) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public void uploadConfig(String configName, Path dir) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public void uploadFileToConfig(String configName, String fileName, byte[] data, boolean overwriteOnExists) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public void setConfigMetadata(String configName, Map<String, Object> data) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public Map<String, Object> getConfigMetadata(String configName) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public void downloadConfig(String configName, Path dir) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public List<String> listConfigs() throws IOException {+ try (Stream<Path> configs = Files.list(configSetBase)) {+ return configs.map(Path::getFileName)+ .map(Path::toString)+ .collect(Collectors.toList());+ }+ }++ @Override+ public byte[] downloadFileFromConfig(String configName, String filePath) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public List<String> getAllConfigFiles(String configName) throws IOException {+ throw new UnsupportedOperationException();+ }++ protected Path locateInstanceDir(CoreDescriptor cd) {+ String configSet = cd.getConfigSet();+ if (configSet == null) return cd.getInstanceDir();+ Path configSetDirectory = configSetBase.resolve(configSet);+ if (!Files.isDirectory(configSetDirectory))+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,+ "Could not load configuration from directory " + configSetDirectory);+ return configSetDirectory;+ }++ @Override+ protected Long getCurrentSchemaModificationVersion(String configSet, SolrConfig solrConfig, String schemaFileName) {+ Path schemaFile = solrConfig.getResourceLoader().getConfigPath().resolve(schemaFileName);+ try {+ return Files.getLastModifiedTime(schemaFile).toMillis();+ } catch (FileNotFoundException e) {+ return null; // acceptable+ } catch (IOException e) {+ log.warn("Unexpected exception when getting modification time of {}", schemaFile, e);+ return null; // debatable; we'll see an error soon if there's a real problem+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/core/ZkContainer.java b/solr/core/src/java/org/apache/solr/core/ZkContainer.javaindex f1d5c398f2d..f3f80828b24 100644--- a/solr/core/src/java/org/apache/solr/core/ZkContainer.java+++ b/solr/core/src/java/org/apache/solr/core/ZkContainer.java@@ -37,7 +37,7 @@ import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.ClusterProperties;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.UrlScheme;-import org.apache.solr.common.cloud.ZkConfigManager;+import org.apache.solr.common.cloud.ZkMaintenanceUtils;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.cloud.ZooKeeperException;import org.apache.solr.common.util.ExecutorUtil;@@ -158,12 +158,11 @@ public class ZkContainer {throw new IllegalArgumentException("bootstrap_confdir must be a directory of configuration files");String confName = System.getProperty(ZkController.COLLECTION_PARAM_PREFIX+ZkController.CONFIGNAME_PROP, "configuration1");- ZkConfigManager configManager = new ZkConfigManager(zkController.getZkClient());- configManager.uploadConfigDir(configPath, confName);+ ZkMaintenanceUtils.uploadToZK(zkController.getZkClient(), configPath, ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName, ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN);}if(boostrapConf) {- ZkController.bootstrapConf(zkController.getZkClient(), cc);+ ConfigSetService.bootstrapConf(cc);}this.zkController = zkController;diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.javaindex c5b48c77739..b8c8783411d 100644--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java@@ -36,11 +36,10 @@ import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.Utils;+import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.backup.repository.BackupRepository;-import org.apache.solr.core.backup.repository.BackupRepository.PathType;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;@@ -228,32 +227,30 @@ public class BackupManager {** @param sourceConfigName The name of the config to be copied* @param targetConfigName The name of the config to be created.+ * @param configSetService The name of the configset used to upload the config* @throws IOException in case of I/O errors.*/- public void uploadConfigDir(String sourceConfigName, String targetConfigName)+ public void uploadConfigDir(String sourceConfigName, String targetConfigName, ConfigSetService configSetService)throws IOException {URI source = getZkStateDir(CONFIG_STATE_DIR, sourceConfigName);- String zkPath = ZkConfigManager.CONFIGS_ZKNODE + "/" + targetConfigName;-Preconditions.checkState(repository.exists(source), "Path {} does not exist", source);- Preconditions.checkState(repository.getPathType(source) == PathType.DIRECTORY,- "Path {} is not a directory", zkPath);- uploadToZk(zkStateReader.getZkClient(), source, zkPath);+ uploadConfigToSolrCloud(configSetService, source, targetConfigName, "");}/*** This method stores the contents of a specified Solr config at the specified location in repository.** @param configName The name of the config to be saved.+ * @param configSetService The name of the configset used to download the config* @throws IOException in case of I/O errors.*/- public void downloadConfigDir(String configName) throws IOException {+ public void downloadConfigDir(String configName, ConfigSetService configSetService) throws IOException {URI dest = getZkStateDir(CONFIG_STATE_DIR, configName);repository.createDirectory(getZkStateDir());repository.createDirectory(getZkStateDir(CONFIG_STATE_DIR));repository.createDirectory(dest);- downloadFromZK(zkStateReader.getZkClient(), ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, dest);+ downloadConfigToRepo(configSetService, configName, dest);}public void uploadCollectionProperties(String collectionName) throws IOException {@@ -296,51 +293,41 @@ public class BackupManager {}}- private void downloadFromZK(SolrZkClient zkClient, String zkPath, URI dir) throws IOException {- try {- List<String> files = zkClient.getChildren(zkPath, null, true);- for (String file : files) {- List<String> children = zkClient.getChildren(zkPath + "/" + file, null, true);- if (children.size() == 0) {- log.debug("Writing file {}", file);- byte[] data = zkClient.getData(zkPath + "/" + file, null, null, true);- try (OutputStream os = repository.createOutput(repository.resolve(dir, file))) {- os.write(data);- }- } else {- URI uri = repository.resolve(dir, file);- if (!repository.exists(uri)) {- repository.createDirectory(uri);- }- downloadFromZK(zkClient, zkPath + "/" + file, repository.resolve(dir, file));+ private void downloadConfigToRepo(ConfigSetService configSetService, String configName, URI dir) throws IOException {+ List<String> filePaths = configSetService.getAllConfigFiles(configName);+ for (String filePath : filePaths) {+ URI uri = repository.resolve(dir, filePath);+ if (!filePath.endsWith("/")) {+ log.debug("Writing file {}", filePath);+ byte[] data = configSetService.downloadFileFromConfig(configName, filePath);+ try (OutputStream os = repository.createOutput(uri)) {+ os.write(data);+ }+ } else {+ if (!repository.exists(uri)) {+ repository.createDirectory(uri);}}- } catch (KeeperException | InterruptedException e) {- throw new IOException("Error downloading files from zookeeper path " + zkPath + " to " + dir.toString(),- SolrZkClient.checkInterrupted(e));}}- private void uploadToZk(SolrZkClient zkClient, URI sourceDir, String destZkPath) throws IOException {+ private void uploadConfigToSolrCloud(ConfigSetService configSetService, URI sourceDir, String configName, String filePrefix) throws IOException {for (String file : repository.listAll(sourceDir)) {- String zkNodePath = destZkPath + "/" + file;+ String filePath = filePrefix + file;URI path = repository.resolve(sourceDir, file);- PathType t = repository.getPathType(path);+ BackupRepository.PathType t = repository.getPathType(path);switch (t) {case FILE: {try (IndexInput is = repository.openInput(sourceDir, file, IOContext.DEFAULT)) {byte[] arr = new byte[(int) is.length()]; // probably ok since the config file should be small.is.readBytes(arr, 0, (int) is.length());- zkClient.makePath(zkNodePath, arr, true);- } catch (KeeperException | InterruptedException e) {- throw new IOException(SolrZkClient.checkInterrupted(e));+ configSetService.uploadFileToConfig(configName, filePath, arr, false);}break;}-case DIRECTORY: {if (!file.startsWith(".")) {- uploadToZk(zkClient, path, zkNodePath);+ uploadConfigToSolrCloud(configSetService, path, configName, filePath + "/");}break;}diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.javaindex 5e40c0d859b..39608a0d13c 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java@@ -16,15 +16,14 @@*/package org.apache.solr.handler.admin;+import java.io.IOException;import java.io.InputStream;import java.lang.invoke.MethodHandles;import java.nio.charset.StandardCharsets;import java.util.Collections;-import java.util.HashSet;import java.util.Iterator;import java.util.List;import java.util.Map;-import java.util.Set;import java.util.concurrent.TimeUnit;import java.util.zip.ZipEntry;import java.util.zip.ZipInputStream;@@ -37,9 +36,6 @@ import org.apache.solr.cloud.OverseerSolrResponseSerializer;import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;-import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;-import org.apache.solr.common.cloud.ZkMaintenanceUtils;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.params.ConfigSetParams;import org.apache.solr.common.params.ConfigSetParams.ConfigSetAction;@@ -48,6 +44,7 @@ import org.apache.solr.common.util.ContentStream;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.common.util.Utils;+import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.CoreContainer;import org.apache.solr.handler.RequestHandlerBase;import org.apache.solr.request.SolrQueryRequest;@@ -55,7 +52,6 @@ import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.security.AuthenticationPlugin;import org.apache.solr.security.AuthorizationContext;import org.apache.solr.security.PermissionNameProvider;-import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -166,10 +162,9 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN"The configuration name should be provided in the \"name\" parameter");}- SolrZkClient zkClient = coreContainer.getZkController().getZkClient();- String configPathInZk = ZkConfigManager.CONFIGS_ZKNODE + "/" + configSetName;+ ConfigSetService configSetService = coreContainer.getConfigSetService();- boolean overwritesExisting = zkClient.exists(configPathInZk, true);+ boolean overwritesExisting = configSetService.checkConfigExists(configSetName);boolean requestIsTrusted = isTrusted(req, coreContainer.getAuthenticationPlugin());@@ -199,16 +194,10 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN// Cleanup is not allowed while using singleFilePath uploadthrow new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet uploads do not allow cleanup=true when file path is used.");} else {- try {- // Create a node for the configuration in zookeeper- // For creating the baseZnode, the cleanup parameter is only allowed to be true when singleFilePath is not passed.- createBaseZnode(zkClient, overwritesExisting, requestIsTrusted, configPathInZk);- String filePathInZk = configPathInZk + "/" + fixedSingleFilePath;- zkClient.makePath(filePathInZk, IOUtils.toByteArray(inputStream), CreateMode.PERSISTENT, null, !allowOverwrite, true);- } catch(KeeperException.NodeExistsException nodeExistsException) {- throw new SolrException(ErrorCode.BAD_REQUEST,- "The path " + singleFilePath + " for configSet " + configSetName + " already exists. In order to overwrite, provide overwrite=true or use an HTTP PUT with the V2 API.");- }+ // Create a node for the configuration in config+ // For creating the baseNode, the cleanup parameter is only allowed to be true when singleFilePath is not passed.+ createBaseNode(configSetService, overwritesExisting, requestIsTrusted, configSetName);+ configSetService.uploadFileToConfig(configSetName, fixedSingleFilePath, IOUtils.toByteArray(inputStream), allowOverwrite);}return;}@@ -218,33 +207,30 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN"The configuration " + configSetName + " already exists in zookeeper");}- Set<String> filesToDelete;+ List<String> filesToDelete;if (overwritesExisting && cleanup) {- filesToDelete = getAllConfigsetFiles(zkClient, configPathInZk);+ filesToDelete = configSetService.getAllConfigFiles(configSetName);} else {- filesToDelete = Collections.emptySet();+ filesToDelete = Collections.emptyList();}// Create a node for the configuration in zookeeper// For creating the baseZnode, the cleanup parameter is only allowed to be true when singleFilePath is not passed.- createBaseZnode(zkClient, overwritesExisting, requestIsTrusted, configPathInZk);+ createBaseNode(configSetService, overwritesExisting, requestIsTrusted, configSetName);ZipInputStream zis = new ZipInputStream(inputStream, StandardCharsets.UTF_8);ZipEntry zipEntry = null;boolean hasEntry = false;while ((zipEntry = zis.getNextEntry()) != null) {hasEntry = true;- String filePathInZk = configPathInZk + "/" + zipEntry.getName();- if (filePathInZk.endsWith("/")) {- filesToDelete.remove(filePathInZk.substring(0, filePathInZk.length() -1));+ String filePath = zipEntry.getName();+ if (filePath.endsWith("/")) {+ filesToDelete.remove(filePath.substring(0, filePath.length() - 1));} else {- filesToDelete.remove(filePathInZk);+ filesToDelete.remove(filePath);}- if (zipEntry.isDirectory()) {- zkClient.makePath(filePathInZk, false, true);- } else {- createZkNodeIfNotExistsAndSetData(zkClient, filePathInZk,- IOUtils.toByteArray(zis));+ if (!zipEntry.isDirectory()) {+ configSetService.uploadFileToConfig(configSetName, zipEntry.getName(), IOUtils.toByteArray(zis), true);}}zis.close();@@ -252,29 +238,29 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNthrow new SolrException(ErrorCode.BAD_REQUEST,"Either empty zipped data, or non-zipped data was uploaded. In order to upload a configSet, you must zip a non-empty directory to upload.");}- deleteUnusedFiles(zkClient, filesToDelete);+ deleteUnusedFiles(configSetService, configSetName, filesToDelete);// If the request is doing a full trusted overwrite of an untrusted configSet (overwrite=true, cleanup=true), then trust the configSet.- if (cleanup && requestIsTrusted && overwritesExisting && !isCurrentlyTrusted(zkClient, configPathInZk)) {- byte[] baseZnodeData = ("{\"trusted\": true}").getBytes(StandardCharsets.UTF_8);- zkClient.setData(configPathInZk, baseZnodeData, true);+ if (cleanup && requestIsTrusted && overwritesExisting && !isCurrentlyTrusted(configSetService, configSetName)) {+ Map<String, Object> metadata = Collections.singletonMap("trusted", true);+ configSetService.setConfigMetadata(configSetName, metadata);}}- private void createBaseZnode(SolrZkClient zkClient, boolean overwritesExisting, boolean requestIsTrusted, String configPathInZk) throws KeeperException, InterruptedException {- byte[] baseZnodeData = ("{\"trusted\": " + Boolean.toString(requestIsTrusted) + "}").getBytes(StandardCharsets.UTF_8);+ private void createBaseNode(ConfigSetService configSetService, boolean overwritesExisting, boolean requestIsTrusted, String configName) throws IOException {+ Map<String, Object> metadata = Collections.singletonMap("trusted", requestIsTrusted);if (overwritesExisting) {if (!requestIsTrusted) {- ensureOverwritingUntrustedConfigSet(zkClient, configPathInZk);+ ensureOverwritingUntrustedConfigSet(configSetService, configName);}// If the request is trusted and cleanup=true, then the configSet will be set to trusted after the overwriting has been done.} else {- zkClient.makePath(configPathInZk, baseZnodeData, true);+ configSetService.setConfigMetadata(configName, metadata);}}- private void deleteUnusedFiles(SolrZkClient zkClient, Set<String> filesToDelete) throws InterruptedException, KeeperException {+ private void deleteUnusedFiles(ConfigSetService configSetService, String configName, List<String> filesToDelete) throws IOException {if (!filesToDelete.isEmpty()) {if (log.isInfoEnabled()) {log.info("Cleaning up {} unused files", filesToDelete.size());@@ -282,50 +268,22 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNif (log.isDebugEnabled()) {log.debug("Cleaning up unused files: {}", filesToDelete);}- for (String f:filesToDelete) {- try {- zkClient.delete(f, -1, true);- } catch (KeeperException.NoNodeException nne) {- }- }- }- }-- private Set<String> getAllConfigsetFiles(SolrZkClient zkClient, String configPathInZk) throws KeeperException, InterruptedException {- final Set<String> files = new HashSet<>();- if (!configPathInZk.startsWith(ZkConfigManager.CONFIGS_ZKNODE + "/")) {- throw new IllegalArgumentException("\"" + configPathInZk + "\" not recognized as a configset path");+ configSetService.deleteFilesFromConfig(configName, filesToDelete);}- ZkMaintenanceUtils.traverseZkTree(zkClient, configPathInZk, ZkMaintenanceUtils.VISIT_ORDER.VISIT_POST, files::add);- files.remove(configPathInZk);- return files;}/** Fail if an untrusted request tries to update a trusted ConfigSet*/- private void ensureOverwritingUntrustedConfigSet(SolrZkClient zkClient, String configSetZkPath) {- boolean isCurrentlyTrusted = isCurrentlyTrusted(zkClient, configSetZkPath);+ private void ensureOverwritingUntrustedConfigSet(ConfigSetService configSetService, String configName) throws IOException {+ boolean isCurrentlyTrusted = isCurrentlyTrusted(configSetService, configName);if (isCurrentlyTrusted) {throw new SolrException(ErrorCode.BAD_REQUEST, "Trying to make an untrusted ConfigSet update on a trusted configSet");}}- private static boolean isCurrentlyTrusted(SolrZkClient zkClient, String configSetZkPath) {- byte[] configSetNodeContent;- try {- configSetNodeContent = zkClient.getData(configSetZkPath, null, null, true);- if (configSetNodeContent == null || configSetNodeContent.length == 0) {- return true;- }- } catch (KeeperException e) {- throw new SolrException(ErrorCode.SERVER_ERROR, "Exception while fetching current configSet at " + configSetZkPath, e);- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- throw new SolrException(ErrorCode.SERVER_ERROR, "Interrupted while fetching current configSet at " + configSetZkPath, e);- }- @SuppressWarnings("unchecked")- Map<Object, Object> contentMap = (Map<Object, Object>) Utils.fromJSON(configSetNodeContent);+ private static boolean isCurrentlyTrusted(ConfigSetService configSetService, String configName) throws IOException {+ Map<String, Object> contentMap = configSetService.getConfigMetadata(configName);return (boolean) contentMap.getOrDefault("trusted", true);}@@ -338,15 +296,6 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNreturn false;}- private void createZkNodeIfNotExistsAndSetData(SolrZkClient zkClient,- String filePathInZk, byte[] data) throws Exception {- if (!zkClient.exists(filePathInZk, true)) {- zkClient.create(filePathInZk, data, CreateMode.PERSISTENT, true);- } else {- zkClient.setData(filePathInZk, data, true);- }- }-@SuppressWarnings({"unchecked"})private void handleResponse(String operation, ZkNodeProps m,SolrQueryResponse rsp, long timeout) throws KeeperException, InterruptedException {@@ -423,13 +372,12 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNthrow new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet name not specified");}- ZkConfigManager zkConfigManager = new ZkConfigManager(h.coreContainer.getZkController().getZkStateReader().getZkClient());- if (zkConfigManager.configExists(newConfigSetName)) {+ if (h.coreContainer.getConfigSetService().checkConfigExists(newConfigSetName)) {throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet already exists: " + newConfigSetName);}// is there a base config that already exists- if (!zkConfigManager.configExists(baseConfigSetName)) {+ if (!h.coreContainer.getConfigSetService().checkConfigExists(baseConfigSetName)) {throw new SolrException(ErrorCode.BAD_REQUEST,"Base ConfigSet does not exist: " + baseConfigSetName);}@@ -438,7 +386,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNprops.put(BASE_CONFIGSET, baseConfigSetName);if (!DISABLE_CREATE_AUTH_CHECKS &&!isTrusted(req, h.coreContainer.getAuthenticationPlugin()) &&- isCurrentlyTrusted(h.coreContainer.getZkController().getZkClient(), ZkConfigManager.CONFIGS_ZKNODE + "/" + baseConfigSetName)) {+ isCurrentlyTrusted(h.coreContainer.getConfigSetService(), baseConfigSetName)) {throw new SolrException(ErrorCode.UNAUTHORIZED, "Can't create a configset with an unauthenticated request from a trusted " + BASE_CONFIGSET);}return copyPropertiesWithPrefix(req.getParams(), props, PROPERTY_PREFIX + ".");@@ -455,9 +403,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN@Overridepublic Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception {NamedList<Object> results = new NamedList<>();- SolrZkClient zk = h.coreContainer.getZkController().getZkStateReader().getZkClient();- ZkConfigManager zkConfigManager = new ZkConfigManager(zk);- List<String> configSetsList = zkConfigManager.listConfigs();+ List<String> configSetsList = h.coreContainer.getConfigSetService().listConfigs();results.add("configSets", configSetsList);SolrResponse response = new OverseerSolrResponse(results);rsp.getValues().addAll(response.getResponse());diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.javaindex 41b6d8a6245..19c1de2237a 100644--- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java+++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java@@ -64,7 +64,6 @@ import org.apache.solr.common.util.Cache;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Pair;import org.apache.solr.common.util.SimpleOrderedMap;-import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrResourceLoader;import org.apache.solr.request.LocalSolrQueryRequest;@@ -109,7 +108,6 @@ public class IndexSchema {public static final String REQUIRED = "required";public static final String SCHEMA = "schema";public static final String SIMILARITY = "similarity";- public static final String SLASH = "/";public static final String SOURCE = "source";public static final String TYPE = "type";public static final String TYPES = "types";@@ -170,7 +168,7 @@ public class IndexSchema {* By default, this follows the normal config path directory searching rules.* @see SolrResourceLoader#openResource*/- public IndexSchema(String name, ConfigSetService.ConfigResource schemaResource, Version luceneVersion, SolrResourceLoader resourceLoader, Properties substitutableProperties) {+ public IndexSchema(String name, IndexSchemaFactory.ConfigResource schemaResource, Version luceneVersion, SolrResourceLoader resourceLoader, Properties substitutableProperties) {this(luceneVersion, resourceLoader, substitutableProperties);this.resourceName = Objects.requireNonNull(name);@@ -492,7 +490,7 @@ public class IndexSchema {}}- protected void readSchema(ConfigSetService.ConfigResource is) {+ protected void readSchema(IndexSchemaFactory.ConfigResource is) {assert null != is : "schema InputSource should never be null";try {rootNode = is.get();diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java b/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.javaindex c49cb12b452..c9d31871dca 100644--- a/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java+++ b/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java@@ -17,7 +17,9 @@package org.apache.solr.schema;-import org.apache.solr.cloud.CloudConfigSetService;+import javax.xml.parsers.ParserConfigurationException;++import org.apache.solr.cloud.ZkConfigSetService;import org.apache.solr.cloud.ZkSolrResourceLoader;import org.apache.solr.common.ConfigNode;import org.apache.solr.common.SolrException;@@ -26,9 +28,15 @@ import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.PluginInfo;import org.apache.solr.core.SolrConfig;import org.apache.solr.core.SolrResourceLoader;+import org.apache.solr.core.XmlConfigFile;+import org.apache.solr.util.DOMConfigNode;+import org.apache.solr.util.DataConfigNode;+import org.apache.solr.util.SystemIdResolver;import org.apache.solr.util.plugin.NamedListInitializedPlugin;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+import org.xml.sax.InputSource;+import org.xml.sax.SAXException;import java.io.IOException;import java.io.InputStream;@@ -36,6 +44,8 @@ import java.lang.invoke.MethodHandles;import java.util.Map;import java.util.concurrent.ConcurrentHashMap;+import static org.apache.solr.schema.IndexSchema.SCHEMA;+/** Base class for factories for IndexSchema implementations */public abstract class IndexSchemaFactory implements NamedListInitializedPlugin {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());@@ -93,11 +103,11 @@ public abstract class IndexSchemaFactory implements NamedListInitializedPlugin {}@SuppressWarnings("unchecked")- public static ConfigSetService.ConfigResource getConfigResource(ConfigSetService configSetService, InputStream schemaInputStream, SolrResourceLoader loader, String name) throws IOException {- if (configSetService instanceof CloudConfigSetService && schemaInputStream instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {+ public static ConfigResource getConfigResource(ConfigSetService configSetService, InputStream schemaInputStream, SolrResourceLoader loader, String name) throws IOException {+ if (configSetService instanceof ZkConfigSetService && schemaInputStream instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {ZkSolrResourceLoader.ZkByteArrayInputStream is = (ZkSolrResourceLoader.ZkByteArrayInputStream) schemaInputStream;- Map<String, VersionedConfig> configCache = (Map<String, VersionedConfig>) ((CloudConfigSetService) configSetService).getSolrCloudManager().getObjectCache()- .computeIfAbsent(ConfigSetService.ConfigResource.class.getName(), s -> new ConcurrentHashMap<>());+ Map<String, VersionedConfig> configCache = (Map<String, VersionedConfig>) ((ZkConfigSetService) configSetService).getSolrCloudManager().getObjectCache()+ .computeIfAbsent(ConfigResource.class.getName(), s -> new ConcurrentHashMap<>());VersionedConfig cached = configCache.get(is.fileName);if (cached != null) {if (cached.version != is.getStat().getVersion()) {@@ -107,13 +117,27 @@ public abstract class IndexSchemaFactory implements NamedListInitializedPlugin {}}return () -> {- ConfigNode data = ConfigSetService.getParsedSchema(schemaInputStream, loader, name);// either missing or stale. create a new one+ ConfigNode data = getParsedSchema(schemaInputStream, loader, name);// either missing or stale. create a new oneconfigCache.put(is.fileName, new VersionedConfig(is.getStat().getVersion(), data));return data;};}//this is not cacheable as it does not come from ZK- return () -> ConfigSetService.getParsedSchema(schemaInputStream,loader, name);+ return () -> getParsedSchema(schemaInputStream,loader, name);+ }++ public static ConfigNode getParsedSchema(InputStream is, SolrResourceLoader loader, String name) throws IOException, SAXException, ParserConfigurationException {+ XmlConfigFile schemaConf = null;+ InputSource inputSource = new InputSource(is);+ inputSource.setSystemId(SystemIdResolver.createSystemIdFromResourceName(name));+ schemaConf = new XmlConfigFile(loader, SCHEMA, inputSource, "/" + SCHEMA + "/", null);+ return new DataConfigNode(new DOMConfigNode(schemaConf.getDocument().getDocumentElement()));+ }++ public interface ConfigResource {++ ConfigNode get() throws Exception;+}public static class VersionedConfig {diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.javaindex ddabd252363..d1387adc841 100644--- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java+++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java@@ -67,7 +67,6 @@ import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SolrNamedThreadFactory;-import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.SolrConfig;import org.apache.solr.core.SolrResourceLoader;import org.apache.solr.rest.schema.FieldTypeXmlAdapter;@@ -103,7 +102,7 @@ public final class ManagedIndexSchema extends IndexSchema {* By default, this follows the normal config path directory searching rules.* @see org.apache.solr.core.SolrResourceLoader#openResource*/- ManagedIndexSchema(SolrConfig solrConfig, String name, ConfigSetService.ConfigResource is, boolean isMutable,+ ManagedIndexSchema(SolrConfig solrConfig, String name, IndexSchemaFactory.ConfigResource is, boolean isMutable,String managedSchemaResourceName, int schemaZkVersion, Object schemaUpdateLock) {super(name, is, solrConfig.luceneMatchVersion, solrConfig.getResourceLoader(), solrConfig.getSubstituteProperties());this.isMutable = isMutable;diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java b/solr/core/src/java/org/apache/solr/schema/SchemaManager.javaindex fe2db9a62eb..80d2cc2fbec 100644--- a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java+++ b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java@@ -33,7 +33,6 @@ import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.util.CommandOperation;import org.apache.solr.common.util.TimeSource;-import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.CoreDescriptor;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrResourceLoader;@@ -454,7 +453,7 @@ public class SchemaManager {if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {int version = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion();log.info("managed schema loaded . version : {} ", version);- return new ManagedIndexSchema(core.getSolrConfig(), name, () -> ConfigSetService.getParsedSchema(in, zkLoader, core.getLatestSchema().getResourceName()), true, name, version,+ return new ManagedIndexSchema(core.getSolrConfig(), name, () -> IndexSchemaFactory.getParsedSchema(in, zkLoader, core.getLatestSchema().getResourceName()), true, name, version,core.getLatestSchema().getSchemaUpdateLock());} else {return (ManagedIndexSchema) core.getLatestSchema();diff --git a/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java b/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.javaindex bc69630272b..e20a62c3362 100644--- a/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java+++ b/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java@@ -25,7 +25,6 @@ import org.apache.solr.common.cloud.OnReconnect;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.cloud.ZooKeeperException;import org.apache.solr.core.CloseHook;-import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrCore;import org.apache.zookeeper.KeeperException;@@ -175,7 +174,7 @@ public class ZkIndexSchemaReader implements OnReconnect {String resourceName = managedIndexSchemaFactory.getManagedSchemaResourceName();ManagedIndexSchema newSchema = new ManagedIndexSchema(managedIndexSchemaFactory.getConfig(), resourceName,- () -> ConfigSetService.getParsedSchema(new ByteArrayInputStream(data),zkLoader , resourceName), managedIndexSchemaFactory.isMutable(),+ () -> IndexSchemaFactory.getParsedSchema(new ByteArrayInputStream(data),zkLoader , resourceName), managedIndexSchemaFactory.isMutable(),resourceName, stat.getVersion(), oldSchema.getSchemaUpdateLock());managedIndexSchemaFactory.setSchema(newSchema);long stop = System.nanoTime();diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.javaindex f93a5ae8d84..9242b5bea52 100755--- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java+++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java@@ -96,7 +96,6 @@ import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.impl.HttpClientUtil;import org.apache.solr.client.solrj.impl.HttpSolrClient;-import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;import org.apache.solr.client.solrj.response.CollectionAdminResponse;@@ -108,15 +107,16 @@ import org.apache.solr.common.cloud.UrlScheme;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.cloud.ZkCoreNodeProps;import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.cloud.ZkMaintenanceUtils;import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.ContentStreamBase;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.StrUtils;+import org.apache.solr.core.ConfigSetService;import org.apache.solr.security.Sha256AuthenticationProvider;import org.apache.solr.util.configuration.SSLConfigurationsFactory;import org.noggit.CharArr;@@ -1543,12 +1543,14 @@ public class SolrCLI implements CLIO {if (confname == null || "".equals(confname.trim())) {confname = collectionName;}- Path confPath = ZkConfigManager.getConfigsetPath(confdir,+ Path confPath = ConfigSetService.getConfigsetPath(confdir,configsetsDir);echoIfVerbose("Uploading " + confPath.toAbsolutePath().toString() +" for config " + confname + " to ZooKeeper at " + cloudSolrClient.getZkHost(), cli);- ((ZkClientClusterStateProvider) cloudSolrClient.getClusterStateProvider()).uploadConfig(confPath, confname);+ ZkMaintenanceUtils.uploadToZK(cloudSolrClient.getZkStateReader().getZkClient(),+ confPath, ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confname,+ ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN);}// since creating a collection is a heavy-weight operation, check for existence first@@ -1838,12 +1840,13 @@ public class SolrCLI implements CLIO {String confName = cli.getOptionValue("confname");try (SolrZkClient zkClient = new SolrZkClient(zkHost, 30000)) {echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli);- Path confPath = ZkConfigManager.getConfigsetPath(cli.getOptionValue("confdir"), cli.getOptionValue("configsetsDir"));+ Path confPath = ConfigSetService.getConfigsetPath(cli.getOptionValue("confdir"), cli.getOptionValue("configsetsDir"));echo("Uploading " + confPath.toAbsolutePath().toString() +" for config " + cli.getOptionValue("confname") + " to ZooKeeper at " + zkHost);+ ZkMaintenanceUtils.uploadToZK(zkClient, confPath, ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName,+ ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN);- zkClient.upConfig(confPath, confName);} catch (Exception e) {log.error("Could not complete upconfig operation for reason: ", e);throw (e);diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.javaindex 9fc8198af78..5f6bb0ed547 100644--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java@@ -34,7 +34,6 @@ import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.cloud.ClusterStateUtil;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.util.BadHdfsThreadsFilter;import org.junit.AfterClass;@@ -58,8 +57,7 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());- ZkConfigManager configManager = new ZkConfigManager(zkClient());- configManager.uploadConfigDir(configset("cloud-hdfs"), "conf1");+ cluster.uploadConfigSet(configset("cloud-hdfs"), "conf1");}@AfterClassdiff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.javaindex babcd15b8c9..82e600ecb94 100644--- a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java@@ -391,6 +391,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {when(distributedClusterStateUpdater.createStateChangeRecorder(any(), anyBoolean())).thenReturn(stateChangeRecorder);when(coreContainerMock.getUpdateShardHandler()).thenReturn(updateShardHandlerMock);when(coreContainerMock.getPlacementPluginFactory()).thenReturn(placementPluginFactoryMock);+ when(coreContainerMock.getConfigSetService()).thenReturn(new ZkConfigSetService(solrZkClientMock));when(updateShardHandlerMock.getDefaultHttpClient()).thenReturn(httpClientMock);when(zkControllerMock.getSolrCloudManager()).thenReturn(cloudDataProviderMock);diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.javaindex 65b09d56e7d..1af32d27284 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java@@ -74,7 +74,6 @@ import org.apache.solr.client.solrj.response.ConfigSetAdminResponse;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.params.CollectionParams.CollectionAction;import org.apache.solr.common.params.ConfigSetParams;import org.apache.solr.common.params.ConfigSetParams.ConfigSetAction;@@ -83,6 +82,7 @@ import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;import org.apache.solr.core.ConfigSetProperties;+import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.TestSolrConfigHandler;import org.apache.solr.security.AuthenticationPlugin;import org.apache.solr.security.AuthorizationContext;@@ -114,19 +114,19 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- private static ZkConfigManager zkConfigManager;-@BeforeClasspublic static void setUpClass() throws Exception {configureCluster(1).withSecurityJson(getSecurityJson()).configure();- zkConfigManager = new ZkConfigManager(cluster.getZkClient());}@AfterClasspublic static void tearDownClass() throws Exception {- zkConfigManager = null;+ }++ private static ConfigSetService getConfigSetService() {+ return cluster.getOpenOverseer().getCoreContainer().getConfigSetService();}@Override@@ -141,7 +141,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {public void testCreateErrors() throws Exception {final String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString();try (final SolrClient solrClient = getHttpSolrClient(baseUrl)) {- zkConfigManager.uploadConfigDir(configset("configset-2"), "configSet");+ getConfigSetService().uploadConfig("configSet", configset("configset-2"));// no actionCreateNoErrorChecking createNoAction = new CreateNoErrorChecking();@@ -229,7 +229,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {FileUtils.write(new File(tmpConfigDir, ConfigSetProperties.DEFAULT_FILENAME),getConfigSetProps(oldProps), UTF_8);}- zkConfigManager.uploadConfigDir(tmpConfigDir.toPath(), baseConfigSetName);+ getConfigSetService().uploadConfig(baseConfigSetName, tmpConfigDir.toPath());}private void verifyCreate(String baseConfigSetName, String configSetName,@@ -241,12 +241,11 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT, null);try {- ZkConfigManager configManager = new ZkConfigManager(zkClient);- assertFalse(configManager.configExists(configSetName));+ assertFalse(getConfigSetService().checkConfigExists(configSetName));ConfigSetAdminResponse response = createConfigSet(baseConfigSetName, configSetName, newProps, solrClient, username);assertNotNull(response.getResponse());- assertTrue(configManager.configExists(configSetName));+ assertTrue(getConfigSetService().checkConfigExists(configSetName));verifyProperties(configSetName, oldProps, newProps, zkClient);} finally {@@ -293,8 +292,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {private void verifyProperties(String configSetName, Map<String, String> oldProps,Map<String, String> newProps, SolrZkClient zkClient) throws Exception {@SuppressWarnings({"rawtypes"})- NamedList properties = getConfigSetPropertiesFromZk(zkClient,- ZkConfigManager.CONFIGS_ZKNODE + "/" + configSetName + "/" + DEFAULT_FILENAME);+ NamedList properties = getConfigSetPropertiesFromZk(zkClient,ZkConfigSetService.CONFIGS_ZKNODE + "/" + configSetName + "/" + DEFAULT_FILENAME);// let's check without merging the maps, since that's what the MessageHandler does// (since we'd probably repeat any bug in the MessageHandler here)if (oldProps == null && newProps == null) {@@ -882,8 +880,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {private long uploadConfigSet(String configSetName, String suffix, String username,SolrZkClient zkClient, boolean v2) throws IOException {- ZkConfigManager configManager = new ZkConfigManager(zkClient);- assertFalse(configManager.configExists(configSetName + suffix));+ assertFalse(getConfigSetService().checkConfigExists(configSetName + suffix));return uploadConfigSet(configSetName, suffix, username, false, false, v2);}@@ -1118,7 +1115,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {FileUtils.copyDirectory(configDir, tmpConfigDir);FileUtils.write(new File(tmpConfigDir, "configsetprops.json"),getConfigSetProps(ImmutableMap.<String, String>of("immutable", "true")), UTF_8);- zkConfigManager.uploadConfigDir(tmpConfigDir.toPath(), "configSet");+ getConfigSetService().uploadConfig("configSet", tmpConfigDir.toPath());// no ConfigSet nameDeleteNoErrorChecking delete = new DeleteNoErrorChecking();@@ -1150,19 +1147,18 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {final String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString();final SolrClient solrClient = getHttpSolrClient(baseUrl);final String configSet = "testDelete";- zkConfigManager.uploadConfigDir(configset("configset-2"), configSet);+ getConfigSetService().uploadConfig(configSet, configset("configset-2"));SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT, null);try {- ZkConfigManager configManager = new ZkConfigManager(zkClient);- assertTrue(configManager.configExists(configSet));+ assertTrue(getConfigSetService().checkConfigExists(configSet));Delete delete = new Delete();delete.setConfigSetName(configSet);ConfigSetAdminResponse response = delete.process(solrClient);assertNotNull(response.getResponse());- assertFalse(configManager.configExists(configSet));+ assertFalse(getConfigSetService().checkConfigExists(configSet));} finally {zkClient.close();}@@ -1188,7 +1184,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {Set<String> configSets = new HashSet<String>();for (int i = 0; i < 5; ++i) {String configSet = "configSet" + i;- zkConfigManager.uploadConfigDir(configset("configset-2"), configSet);+ getConfigSetService().uploadConfig(configSet, configset("configset-2"));configSets.add(configSet);}response = list.process(solrClient);@@ -1216,7 +1212,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {** @see SolrDispatchFilter#SOLR_DEFAULT_CONFDIR_ATTRIBUTE* @see #setDefaultConfigDirSysPropIfNotSet- * @see ZkController#getDefaultConfigDirPath+ * @see ConfigSetService#getDefaultConfigDirPath*/@Testpublic void testUserAndTestDefaultConfigsetsAreSame() throws IOException {@@ -1224,7 +1220,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {assertTrue("_default dir doesn't exist: " + ExternalPaths.DEFAULT_CONFIGSET, extPath.exists());assertTrue("_default dir isn't a dir: " + ExternalPaths.DEFAULT_CONFIGSET, extPath.isDirectory());- final String zkBootStrap = ZkController.getDefaultConfigDirPath();+ final String zkBootStrap = ConfigSetService.getDefaultConfigDirPath();assertEquals("extPath _default configset dir vs zk bootstrap path",ExternalPaths.DEFAULT_CONFIGSET, zkBootStrap);}diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.javaindex e23f71a7a0a..54a67160202 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java@@ -39,9 +39,9 @@ import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteSolrException;import org.apache.solr.client.solrj.request.ConfigSetAdminRequest.Create;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.util.Utils;import org.apache.solr.core.ConfigSetProperties;+import org.apache.solr.core.ConfigSetService;import org.apache.zookeeper.KeeperException;import org.apache.zookeeper.KeeperException.NoNodeException;import org.apache.zookeeper.Watcher;@@ -60,7 +60,7 @@ import org.junit.After;import org.junit.Before;import org.junit.Test;-import static org.apache.solr.common.cloud.ZkConfigManager.CONFIGS_ZKNODE;+import static org.apache.solr.cloud.ZkConfigSetService.CONFIGS_ZKNODE;/*** Test the ConfigSets API under ZK failure. In particular,@@ -104,6 +104,7 @@ public class TestConfigSetsAPIZkFailure extends SolrTestCaseJ4 {public void testCreateZkFailure() throws Exception {final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString();final SolrClient solrClient = getHttpSolrClient(baseUrl);+ final ConfigSetService configSetService = solrCluster.getOpenOverseer().getCoreContainer().getConfigSetService();final Map<String, String> oldProps = ImmutableMap.of("immutable", "true");setupBaseConfigSet(BASE_CONFIGSET_NAME, oldProps);@@ -111,14 +112,14 @@ public class TestConfigSetsAPIZkFailure extends SolrTestCaseJ4 {SolrZkClient zkClient = new SolrZkClient(solrCluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT, null);try {- ZkConfigManager configManager = new ZkConfigManager(zkClient);- assertFalse(configManager.configExists(CONFIGSET_NAME));++ assertFalse(configSetService.checkConfigExists(CONFIGSET_NAME));Create create = new Create();create.setBaseConfigSetName(BASE_CONFIGSET_NAME).setConfigSetName(CONFIGSET_NAME);RemoteSolrException se = expectThrows(RemoteSolrException.class, () -> create.process(solrClient));// partial creation should have been cleaned up- assertFalse(configManager.configExists(CONFIGSET_NAME));+ assertFalse(configSetService.checkConfigExists(CONFIGSET_NAME));assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, se.code());} finally {zkClient.close();diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.javaindex 9a54dafbfed..547124ec093 100644--- a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java@@ -38,9 +38,9 @@ import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.ClusterProperties;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.core.ConfigSetService;import org.apache.solr.util.ExternalPaths;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;@@ -110,7 +110,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {assertEquals("upconfig", ZkCLI.UPCONFIG);assertEquals("x", ZkCLI.EXCLUDE_REGEX_SHORT);assertEquals("excluderegex", ZkCLI.EXCLUDE_REGEX);- assertEquals(ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX, ZkCLI.EXCLUDE_REGEX_DEFAULT);+ assertEquals(ConfigSetService.UPLOAD_FILENAME_EXCLUDE_REGEX, ZkCLI.EXCLUDE_REGEX_DEFAULT);}@Test@@ -123,7 +123,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {ZkCLI.main(args);- assertTrue(zkClient.exists(chroot + ZkConfigManager.CONFIGS_ZKNODE+ assertTrue(zkClient.exists(chroot + ZkConfigSetService.CONFIGS_ZKNODE+ "/collection1", true));}@@ -245,7 +245,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {}ZkCLI.main(upconfigArgs);- assertTrue(zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, true));+ assertTrue(zkClient.exists(ZkConfigSetService.CONFIGS_ZKNODE + "/" + confsetname, true));// print help// ZkCLI.main(new String[0]);@@ -269,7 +269,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {ZkCLI.main(args);File[] files = confDir.listFiles();- List<String> zkFiles = zkClient.getChildren(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, null, true);+ List<String> zkFiles = zkClient.getChildren(ZkConfigSetService.CONFIGS_ZKNODE + "/" + confsetname, null, true);assertEquals(files.length, zkFiles.size());File sourceConfDir = new File(ExternalPaths.TECHPRODUCTS_CONFIGSET);@@ -279,7 +279,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {int indexOfRelativePath = sourceFile.getAbsolutePath().lastIndexOf("sample_techproducts_configs" + File.separator + "conf");String relativePathofFile = sourceFile.getAbsolutePath().substring(indexOfRelativePath + 33, sourceFile.getAbsolutePath().length());File downloadedFile = new File(confDir,relativePathofFile);- if (ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_PATTERN.matcher(relativePathofFile).matches()) {+ if (ConfigSetService.UPLOAD_FILENAME_EXCLUDE_PATTERN.matcher(relativePathofFile).matches()) {assertFalse(sourceFile.getAbsolutePath() + " exists in ZK, downloaded:" + downloadedFile.getAbsolutePath(), downloadedFile.exists());} else {assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists());diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.javaindex ae9001b7988..09be95f4c7e 100644--- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java@@ -29,7 +29,6 @@ import org.apache.lucene.util.LuceneTestCase.Slow;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.common.cloud.ClusterProperties;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.CollectionParams;@@ -183,7 +182,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 {SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);String actualConfigName = "firstConfig";- zkClient.makePath(ZkConfigManager.CONFIGS_ZKNODE + "/" + actualConfigName, true);+ zkClient.makePath(ZkConfigSetService.CONFIGS_ZKNODE + "/" + actualConfigName, true);Map<String,Object> props = new HashMap<>();props.put("configName", actualConfigName);diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/SimpleCollectionCreateDeleteTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/SimpleCollectionCreateDeleteTest.javaindex 8cd835fe329..782a3c037fc 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/SimpleCollectionCreateDeleteTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/SimpleCollectionCreateDeleteTest.java@@ -20,6 +20,7 @@ import org.apache.solr.client.solrj.embedded.JettySolrRunner;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.cloud.AbstractFullDistribZkTestBase;import org.apache.solr.cloud.OverseerCollectionConfigSetProcessor;+import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.TimeSource;@@ -56,12 +57,12 @@ public class SimpleCollectionCreateDeleteTest extends AbstractFullDistribZkTestBNamedList<Object> request = create.process(cloudClient).getResponse();if (request.get("success") != null) {- assertTrue(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));+ assertTrue(getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection(collectionName);cloudClient.request(delete);- assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));+ assertFalse(getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));// currently, removing a collection does not wait for cores to be unloadedTimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);@@ -113,15 +114,15 @@ public class SimpleCollectionCreateDeleteTest extends AbstractFullDistribZkTestB// config for this collection is '.AUTOCREATED', and exists globallyassertTrue(configName.endsWith(".AUTOCREATED"));- assertTrue(cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));+ assertTrue(getZkClient().exists(ZkStateReader.CONFIGS_ZKNODE + "/" + configName, true));CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection(collectionName);cloudClient.request(delete);// collection has been deleted- assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));+ assertFalse(getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));// ... and so has its autocreated config set- assertFalse("The auto-created config set should have been deleted with its collection", cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));+ assertFalse("The auto-created config set should have been deleted with its collection", getZkClient().exists(ZkStateReader.CONFIGS_ZKNODE + "/" + configName, true));}}@@ -135,13 +136,13 @@ public class SimpleCollectionCreateDeleteTest extends AbstractFullDistribZkTestBif (requestInitial.get("success") != null) {// collection exists now- assertTrue(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameInitial, false));+ assertTrue(getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameInitial, false));String configName = cloudClient.getZkStateReader().readConfigName(collectionNameInitial);// config for this collection is '.AUTOCREATED', and exists globallyassertTrue(configName.endsWith(".AUTOCREATED"));- assertTrue(cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));+ assertTrue(getZkClient().exists(ZkStateReader.CONFIGS_ZKNODE + "/" + configName, true));// create a second collection, sharing the same configSetString collectionNameWithSharedConfig = "SimpleCollectionCreateDeleteTest.collectionSharingAutocreatedConfigSet";@@ -149,7 +150,7 @@ public class SimpleCollectionCreateDeleteTest extends AbstractFullDistribZkTestBNamedList<Object> requestWithSharedConfig = createWithSharedConfig.process(cloudClient).getResponse();assertTrue("The collection with shared config set should have been created", requestWithSharedConfig.get("success") != null);- assertTrue("The new collection should exist after a successful creation", cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameWithSharedConfig, false));+ assertTrue("The new collection should exist after a successful creation", getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameWithSharedConfig, false));String configNameOfSecondCollection = cloudClient.getZkStateReader().readConfigName(collectionNameWithSharedConfig);@@ -160,19 +161,23 @@ public class SimpleCollectionCreateDeleteTest extends AbstractFullDistribZkTestBcloudClient.request(deleteInitialCollection);// initial collection has been deleted- assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameInitial, false));+ assertFalse(getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameInitial, false));// ... but not its autocreated config set, since it is shared with another collection- assertTrue("The auto-created config set should NOT have been deleted. Another collection is using it.", cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));+ assertTrue("The auto-created config set should NOT have been deleted. Another collection is using it.", getZkClient().exists(ZkStateReader.CONFIGS_ZKNODE + "/" + configName, true));// delete the second collection - the config set should now be deleted, since it is no longer shared any other collectionCollectionAdminRequest.Delete deleteSecondCollection = CollectionAdminRequest.deleteCollection(collectionNameWithSharedConfig);cloudClient.request(deleteSecondCollection);// the collection has been deleted- assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameWithSharedConfig, false));+ assertFalse(getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameWithSharedConfig, false));// ... and the config set is now also deleted - once it doesn't get referenced by any collection- assertFalse("The auto-created config set should have been deleted now. No collection is referencing it.", cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));+ assertFalse("The auto-created config set should have been deleted now. No collection is referencing it.", getZkClient().exists(ZkStateReader.CONFIGS_ZKNODE + "/" + configName, true));}}+ public SolrZkClient getZkClient() {+ return cloudClient.getZkStateReader().getZkClient();+ }+}diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.javaindex 15f3b506abc..884ea222fa0 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java@@ -33,6 +33,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.client.solrj.request.V2Request;import org.apache.solr.client.solrj.response.CollectionAdminResponse;+import org.apache.solr.cloud.ZkConfigSetService;import org.apache.solr.cloud.ZkTestServer;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrInputDocument;@@ -41,7 +42,6 @@ import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.params.CollectionParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.ShardParams;@@ -230,7 +230,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {waitForRecoveriesToFinish(collection, false);// Now try deleting the configset and doing a clusterstatus.- String parent = ZkConfigManager.CONFIGS_ZKNODE + "/" + configSet;+ String parent = ZkConfigSetService.CONFIGS_ZKNODE + "/" + configSet;deleteThemAll(client.getZkStateReader().getZkClient(), parent);client.getZkStateReader().forciblyRefreshAllClusterStateSlow();diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.javaindex 67cc6ae5b4e..dae02d4db79 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java@@ -39,9 +39,9 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;import org.apache.solr.SolrIgnoredThreadsFilter;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.cloud.ZkConfigSetService;import org.apache.solr.cloud.hdfs.HdfsTestUtil;import org.apache.solr.common.cloud.DocCollection;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.backup.BackupManager;@@ -141,11 +141,11 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCauseFactory("solr.StandardDirectoryFactory");configureCluster(NUM_SHARDS)// nodes- .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))- .addConfig("confFaulty", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf"))+ .addConfig("confFaulty", TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf")).withSolrXml(SOLR_XML).configure();- cluster.getZkClient().delete(ZkConfigManager.CONFIGS_ZKNODE + Path.SEPARATOR + "confFaulty" + Path.SEPARATOR + "solrconfig.xml", -1, true);+ cluster.getZkClient().delete(ZkConfigSetService.CONFIGS_ZKNODE + Path.SEPARATOR + "confFaulty" + Path.SEPARATOR + "solrconfig.xml", -1, true);}@AfterClassdiff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.javaindex 063dcf7fc66..3c52362f7ee 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java@@ -28,9 +28,9 @@ import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.cloud.MiniSolrCloudCluster;+import org.apache.solr.cloud.ZkConfigSetService;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.core.backup.repository.LocalFileSystemRepository;import org.junit.BeforeClass;import org.junit.Test;@@ -64,7 +64,7 @@ public class TestLocalFSCloudBackupRestore extends AbstractCloudBackupRestoreTes.addConfig("confFaulty", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")).withSolrXml(solrXml).configure();- cluster.getZkClient().delete(ZkConfigManager.CONFIGS_ZKNODE + "/" + "confFaulty" + "/" + "solrconfig.xml", -1, true);+ cluster.getZkClient().delete(ZkConfigSetService.CONFIGS_ZKNODE + "/" + "confFaulty" + "/" + "solrconfig.xml", -1, true);boolean whitespacesInPath = random().nextBoolean();if (whitespacesInPath) {diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.javaindex da66e6f48f4..d88b1d1fbf4 100644--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java@@ -28,7 +28,6 @@ import org.apache.solr.client.solrj.response.QueryResponse;import org.apache.solr.cloud.SolrCloudTestCase;import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.cloud.Replica;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.util.BadHdfsThreadsFilter;import org.junit.AfterClass;import org.junit.BeforeClass;@@ -48,8 +47,7 @@ public class HDFSCollectionsAPITest extends SolrCloudTestCase {dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());- ZkConfigManager configManager = new ZkConfigManager(zkClient());- configManager.uploadConfigDir(configset("cloud-hdfs"), "conf1");+ cluster.uploadConfigSet(configset("cloud-hdfs"), "conf1");}diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.javaindex 056cc9cd8c2..45192fe0dd8 100644--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java@@ -23,7 +23,6 @@ import org.apache.lucene.util.LuceneTestCase.Slow;import org.apache.lucene.util.QuickPatchThreadsFilter;import org.apache.solr.SolrIgnoredThreadsFilter;import org.apache.solr.cloud.RecoveryZkTest;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.util.BadHdfsThreadsFilter;import org.junit.AfterClass;import org.junit.BeforeClass;@@ -42,8 +41,7 @@ public class HdfsRecoveryZkTest extends RecoveryZkTest {public static void setupClass() throws Exception {dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());- ZkConfigManager configManager = new ZkConfigManager(zkClient());- configManager.uploadConfigDir(configset("cloud-hdfs"), "conf");+ cluster.uploadConfigSet(configset("cloud-hdfs"), "conf");}@AfterClassdiff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.javaindex 26c517ed5d8..2d408f2c19e 100644--- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java+++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java@@ -18,6 +18,7 @@ package org.apache.solr.core;import java.io.File;import java.io.FileOutputStream;+import java.io.IOException;import java.nio.file.Path;import java.nio.file.Paths;@@ -459,6 +460,65 @@ public class TestCoreContainer extends SolrTestCaseJ4 {return null;}+ @Override+ public boolean checkConfigExists(String configName) throws IOException {+ return false;+ }++ @Override+ public void deleteConfig(String configName) throws IOException {++ }++ @Override+ public void deleteFilesFromConfig(String configName, List<String> filesToDelete) throws IOException {++ }++ public void copyConfig(String fromConfig, String toConfig) throws IOException {++ }++ @Override+ public void uploadConfig(String configName, Path dir) throws IOException {++ }++ @Override+ public void uploadFileToConfig(String configName, String fileName, byte[] data, boolean overwriteOnExists) throws IOException {++ }++ @Override+ public void setConfigMetadata(String configName, Map<String, Object> data) throws IOException {++ }++ @Override+ public Map<String, Object> getConfigMetadata(String configName) throws IOException {+ return null;+ }++ @Override+ public void downloadConfig(String configName, Path dir) throws IOException {++ }++ @Override+ public List<String> listConfigs() throws IOException {+ return null;+ }++ @Override+ public byte[] downloadFileFromConfig(String configName, String filePath) throws IOException {+ return new byte[0];+ }++ @Override+ public List<String> getAllConfigFiles(String configName) throws IOException {+ return null;+ }+@Overrideprotected SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd){return null;diff --git a/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java b/solr/core/src/test/org/apache/solr/handler/TestConfigReload.javaindex ff195010eb3..c9efd4972ee 100644--- a/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java+++ b/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java@@ -28,13 +28,13 @@ import org.apache.http.HttpEntity;import org.apache.http.client.methods.HttpGet;import org.apache.http.util.EntityUtils;import org.apache.solr.cloud.AbstractFullDistribZkTestBase;+import org.apache.solr.cloud.ZkConfigSetService;import org.apache.solr.common.LinkedHashMapWriter;import org.apache.solr.common.MapWriter;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.StrUtils;import org.apache.solr.common.util.Utils;@@ -67,7 +67,7 @@ public class TestConfigReload extends AbstractFullDistribZkTestBase {if (log.isInfoEnabled()) {log.info("live_nodes_count : {}", cloudClient.getZkStateReader().getClusterState().getLiveNodes());}- String confPath = ZkConfigManager.CONFIGS_ZKNODE+"/conf1/";+ String confPath = ZkConfigSetService.CONFIGS_ZKNODE+"/conf1/";// checkConfReload(client, confPath + ConfigOverlay.RESOURCE_NAME, "overlay");checkConfReload(client, confPath + SolrConfig.DEFAULT_CONF_FILE,"config", "/config");diff --git a/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java b/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.javaindex a42ebaa35db..83d79b875c1 100644--- a/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java+++ b/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java@@ -26,7 +26,6 @@ import java.util.Locale;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.embedded.JettySolrRunner;import org.apache.solr.client.solrj.impl.CloudSolrClient;-import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.UpdateRequest;import org.apache.solr.client.solrj.response.QueryResponse;@@ -141,7 +140,7 @@ public class CrossCollectionJoinQueryTest extends SolrCloudTestCase {try {// now we need to re-upload our config , now that we know a valid solr url for the cluster.CloudSolrClient client = cluster.getSolrClient();- ((ZkClientClusterStateProvider) client.getClusterStateProvider()).uploadConfig(configset("ccjoin"), "ccjoin");+ cluster.uploadConfigSet(configset("ccjoin"), "ccjoin");// reload the cores with the updated allowSolrUrls config.CollectionAdminRequest.Reload.reloadCollection("products").process(client);CollectionAdminRequest.Reload.reloadCollection("parts").process(client);@@ -221,7 +220,7 @@ public class CrossCollectionJoinQueryTest extends SolrCloudTestCase {try {// now we need to re-upload our config , now that we know a valid solr url for the cluster.CloudSolrClient client = cluster.getSolrClient();- ((ZkClientClusterStateProvider) client.getClusterStateProvider()).uploadConfig(configset("ccjoin"), "ccjoin");+ cluster.uploadConfigSet(configset("ccjoin"), "ccjoin");// reload the cores with the updated allowSolrUrls config.CollectionAdminRequest.Reload.reloadCollection("products").process(client);CollectionAdminRequest.Reload.reloadCollection("parts").process(client);diff --git a/solr/solr-ref-guide/src/format-of-solr-xml.adoc b/solr/solr-ref-guide/src/format-of-solr-xml.adocindex 459e49ad452..93f8475db7a 100644--- a/solr/solr-ref-guide/src/format-of-solr-xml.adoc+++ b/solr/solr-ref-guide/src/format-of-solr-xml.adoc@@ -68,7 +68,7 @@ This attribute does not need to be set.+If used, this attribute should be set to the FQN (Fully qualified name) of a class that inherits from ConfigSetService , and you must provide a constructor with one param which the type is `org.apache.solr.core.CoreContainer` . For example, `<str name="configSetService">com.myorg.CustomConfigSetService</str>`.+-If this attribute isn't set, Solr uses the default configSetService , with zookeeper aware of `org.apache.solr.cloud.CloudConfigSetService`, without zookeeper aware of `org.apache.solr.core.ConfigSetService.Standalone`.+If this attribute isn't set, Solr uses the default configSetService , with zookeeper aware of `org.apache.solr.cloud.ZkConfigSetService`, without zookeeper aware of `org.apache.solr.core.FileSystemConfigSetService`.`adminHandler`::This attribute does not need to be set.diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.javaindex 3020c1aced5..3f101db6fcb 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java@@ -19,7 +19,6 @@ package org.apache.solr.client.solrj.impl;import java.io.IOException;import java.lang.invoke.MethodHandles;-import java.nio.file.Path;import java.util.Collection;import java.util.Collections;import java.util.List;@@ -130,31 +129,6 @@ public class ZkClientClusterStateProvider implements ClusterStateProvider {return state == null || state.get() == null ? null : (String) state.get().getProperties().get("policy");}- /**- * Download a named config from Zookeeper to a location on the filesystem- * @param configName the name of the config- * @param downloadPath the path to write config files to- * @throws IOException if an I/O exception occurs- */- public void downloadConfig(String configName, Path downloadPath) throws IOException {- getZkStateReader().getConfigManager().downloadConfigDir(configName, downloadPath);- }-- /**- * Upload a set of config files to Zookeeper and give it a name- *- * NOTE: You should only allow trusted users to upload configs. If you- * are allowing client access to zookeeper, you should protect the- * /configs node against unauthorised write access.- *- * @param configPath {@link java.nio.file.Path} to the config files- * @param configName the name of the config- * @throws IOException if an IO error occurs- */- public void uploadConfig(Path configPath, String configName) throws IOException {- getZkStateReader().getConfigManager().uploadConfigDir(configPath, configName);- }-@Overridepublic void connect() {// Esentially a No-Op, but force a check that we're not closed and the ZkStateReader is available...diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.javaindex 01078b89f43..47aee058fd5 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java@@ -837,7 +837,7 @@ public class SolrZkClient implements Closeable {}public void upConfig(Path confPath, String confName) throws IOException {- ZkMaintenanceUtils.upConfig(this, confPath, confName);+ ZkMaintenanceUtils.uploadToZK(this, confPath, ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName, ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN);}public String listZnode(String path, Boolean recurse) throws KeeperException, InterruptedException, SolrServerException {@@ -845,7 +845,7 @@ public class SolrZkClient implements Closeable {}public void downConfig(String confName, Path confPath) throws IOException {- ZkMaintenanceUtils.downConfig(this, confName, confPath);+ ZkMaintenanceUtils.downloadFromZK(this, ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName, confPath);}public void zkTransfer(String src, Boolean srcIsZk,diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.javadeleted file mode 100644index 72dacf22776..00000000000--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java+++ /dev/null@@ -1,230 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.common.cloud;--import org.apache.zookeeper.KeeperException;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.nio.file.Files;-import java.nio.file.Path;-import java.nio.file.Paths;-import java.util.Collections;-import java.util.List;-import java.util.Locale;-import java.util.Set;-import java.util.regex.Pattern;--/**- * Class that manages named configs in Zookeeper- */-public class ZkConfigManager {-- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- /** ZkNode where named configs are stored */- public static final String CONFIGS_ZKNODE = "/configs";-- public static final String UPLOAD_FILENAME_EXCLUDE_REGEX = "^\\..*$";- public static final Pattern UPLOAD_FILENAME_EXCLUDE_PATTERN = Pattern.compile(UPLOAD_FILENAME_EXCLUDE_REGEX);-- private final SolrZkClient zkClient;-- /**- * Creates a new ZkConfigManager- * @param zkClient the {@link SolrZkClient} to use- */- public ZkConfigManager(SolrZkClient zkClient) { this.zkClient = zkClient; }---- /**- * Upload files from a given path to a config in Zookeeper- * @param dir {@link java.nio.file.Path} to the files- * @param configName the name to give the config- * @throws IOException- * if an I/O error occurs or the path does not exist- */- public void uploadConfigDir(Path dir, String configName) throws IOException {- zkClient.uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, UPLOAD_FILENAME_EXCLUDE_PATTERN);- }-- /**- * Upload matching files from a given path to a config in Zookeeper- * @param dir {@link java.nio.file.Path} to the files- * @param configName the name to give the config- * @param filenameExclusions files matching this pattern will not be uploaded- * @throws IOException- * if an I/O error occurs or the path does not exist- */- public void uploadConfigDir(Path dir, String configName,- Pattern filenameExclusions) throws IOException {- zkClient.uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, filenameExclusions);- }-- /**- * Download a config from Zookeeper and write it to the filesystem- * @param configName the config to download- * @param dir the {@link Path} to write files under- * @throws IOException- * if an I/O error occurs or the config does not exist- */- public void downloadConfigDir(String configName, Path dir) throws IOException {- zkClient.downloadFromZK(CONFIGS_ZKNODE + "/" + configName, dir);- }-- public List<String> listConfigs() throws IOException {- try {- return zkClient.getChildren(ZkConfigManager.CONFIGS_ZKNODE, null, true);- }- catch (KeeperException.NoNodeException e) {- return Collections.emptyList();- }- catch (KeeperException | InterruptedException e) {- throw new IOException("Error listing configs", SolrZkClient.checkInterrupted(e));- }- }-- /**- * Check whether a config exists in Zookeeper- *- * @param configName the config to check existance on- * @return whether the config exists or not- * @throws IOException if an I/O error occurs- */- public Boolean configExists(String configName) throws IOException {- try {- return zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true);- } catch (KeeperException | InterruptedException e) {- throw new IOException("Error checking whether config exists",- SolrZkClient.checkInterrupted(e));- }- }-- /**- * Delete a config in ZooKeeper- *- * @param configName the config to delete- * @throws IOException if an I/O error occurs- */- public void deleteConfigDir(String configName) throws IOException {- try {- zkClient.clean(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName);- } catch (KeeperException | InterruptedException e) {- throw new IOException("Error checking whether config exists",- SolrZkClient.checkInterrupted(e));- }- }-- private void copyConfigDirFromZk(String fromZkPath, String toZkPath, Set<String> copiedToZkPaths) throws IOException {- try {- List<String> files = zkClient.getChildren(fromZkPath, null, true);- for (String file : files) {- List<String> children = zkClient.getChildren(fromZkPath + "/" + file, null, true);- if (children.size() == 0) {- copyData(copiedToZkPaths, fromZkPath + "/" + file, toZkPath + "/" + file);- } else {- copyConfigDirFromZk(fromZkPath + "/" + file, toZkPath + "/" + file, copiedToZkPaths);- }- }- } catch (KeeperException | InterruptedException e) {- throw new IOException("Error copying nodes from zookeeper path " + fromZkPath + " to " + toZkPath,- SolrZkClient.checkInterrupted(e));- }- }-- private void copyData(Set<String> copiedToZkPaths, String fromZkFilePath, String toZkFilePath) throws KeeperException, InterruptedException {- log.info("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);- byte[] data = zkClient.getData(fromZkFilePath, null, null, true);- zkClient.makePath(toZkFilePath, data, true);- if (copiedToZkPaths != null) copiedToZkPaths.add(toZkFilePath);- }-- /**- * Copy a config in ZooKeeper- *- * @param fromConfig the config to copy from- * @param toConfig the config to copy to- * @throws IOException if an I/O error occurs- */- public void copyConfigDir(String fromConfig, String toConfig) throws IOException {- copyConfigDir(fromConfig, toConfig, null);- }-- /**- * Copy a config in ZooKeeper- *- * @param fromConfig the config to copy from- * @param toConfig the config to copy to- * @param copiedToZkPaths should be an empty Set, will be filled in by function- with the paths that were actually copied to.- * @throws IOException if an I/O error occurs- */- public void copyConfigDir(String fromConfig, String toConfig, Set<String> copiedToZkPaths) throws IOException {- String fromConfigPath = CONFIGS_ZKNODE + "/" + fromConfig;- String toConfigPath = CONFIGS_ZKNODE + "/" + toConfig;- try {- copyData(copiedToZkPaths, fromConfigPath, toConfigPath);- } catch (KeeperException | InterruptedException e) {- throw new IOException("Error config " + fromConfig + " to " + toConfig,- SolrZkClient.checkInterrupted(e));- }- copyConfigDirFromZk(fromConfigPath, toConfigPath, copiedToZkPaths);- }-- // This method is used by configSetUploadTool and CreateTool to resolve the configset directory.- // Check several possibilities:- // 1> confDir/solrconfig.xml exists- // 2> confDir/conf/solrconfig.xml exists- // 3> configSetDir/confDir/conf/solrconfig.xml exists (canned configs)-- // Order is important here since "confDir" may be- // 1> a full path to the parent of a solrconfig.xml or parent of /conf/solrconfig.xml- // 2> one of the canned config sets only, e.g. _default- // and trying to assemble a path for configsetDir/confDir is A Bad Idea. if confDir is a full path.-- public static Path getConfigsetPath(String confDir, String configSetDir) throws IOException {-- // A local path to the source, probably already includes "conf".- Path ret = Paths.get(confDir, "solrconfig.xml").normalize();- if (Files.exists(ret)) {- return Paths.get(confDir).normalize();- }-- // a local path to the parent of a "conf" directory- ret = Paths.get(confDir, "conf", "solrconfig.xml").normalize();- if (Files.exists(ret)) {- return Paths.get(confDir, "conf").normalize();- }-- // one of the canned configsets.- ret = Paths.get(configSetDir, confDir, "conf", "solrconfig.xml").normalize();- if (Files.exists(ret)) {- return Paths.get(configSetDir, confDir, "conf").normalize();- }--- throw new IllegalArgumentException(String.format(Locale.ROOT,- "Could not find solrconfig.xml at %s, %s or %s",- Paths.get(configSetDir, "solrconfig.xml").normalize().toAbsolutePath().toString(),- Paths.get(configSetDir, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString(),- Paths.get(configSetDir, confDir, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString()- ));- }-}diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.javaindex 04e80ac1947..5d874a64a27 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java@@ -45,6 +45,11 @@ import org.slf4j.LoggerFactory;* in bin/solr it made sense to keep the individual transfer methods in a central place, so here it is.*/public class ZkMaintenanceUtils {+ /** ZkNode where named configs are stored */+ public static final String CONFIGS_ZKNODE = "/configs";+ public static final String UPLOAD_FILENAME_EXCLUDE_REGEX = "^\\..*$";+ /** files matching this pattern will not be uploaded to ZkNode /configs */+ public static final Pattern UPLOAD_FILENAME_EXCLUDE_PATTERN = Pattern.compile(UPLOAD_FILENAME_EXCLUDE_REGEX);private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());private static final String ZKNODE_DATA_FILE = "zknode.data";@@ -217,21 +222,6 @@ public class ZkMaintenanceUtils {}}- // This not just a copy operation since the config manager takes care of construction the znode path to configsets- public static void downConfig(SolrZkClient zkClient, String confName, Path confPath) throws IOException {- ZkConfigManager manager = new ZkConfigManager(zkClient);-- // Try to download the configset- manager.downloadConfigDir(confName, confPath);- }-- // This not just a copy operation since the config manager takes care of construction the znode path to configsets- public static void upConfig(SolrZkClient zkClient, Path confPath, String confName) throws IOException {- ZkConfigManager manager = new ZkConfigManager(zkClient);-- // Try to download the configset- manager.uploadConfigDir(confPath, confName);- }// yeah, it's recursive :(public static void clean(SolrZkClient zkClient, String path) throws InterruptedException, KeeperException {diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.javaindex baae22bbed0..b26183cea05 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java@@ -185,8 +185,6 @@ public class ZkStateReader implements SolrCloseable {private volatile Map<String, Object> clusterProperties = Collections.emptyMap();- private final ZkConfigManager configManager;-private ConfigData securityData;private final Runnable securityNodeListener;@@ -287,7 +285,6 @@ public class ZkStateReader implements SolrCloseable {public ZkStateReader(SolrZkClient zkClient, Runnable securityNodeListener) {this.zkClient = zkClient;- this.configManager = new ZkConfigManager(zkClient);this.closeClient = false;this.securityNodeListener = securityNodeListener;assert ObjectReleaseTracker.track(this);@@ -313,17 +310,12 @@ public class ZkStateReader implements SolrCloseable {}}});- this.configManager = new ZkConfigManager(zkClient);this.closeClient = true;this.securityNodeListener = null;assert ObjectReleaseTracker.track(this);}- public ZkConfigManager getConfigManager() {- return configManager;- }-/*** Forcibly refresh cluster state from ZK. Do this only to avoid race conditions because it's expensive.* <p>diff --git a/solr/solrj/src/test/org/apache/solr/client/ref_guide_examples/ZkConfigFilesTest.java b/solr/solrj/src/test/org/apache/solr/client/ref_guide_examples/ZkConfigFilesTest.javaindex dcb11118872..0c597e0c3fa 100644--- a/solr/solrj/src/test/org/apache/solr/client/ref_guide_examples/ZkConfigFilesTest.java+++ b/solr/solrj/src/test/org/apache/solr/client/ref_guide_examples/ZkConfigFilesTest.java@@ -22,8 +22,7 @@ import java.nio.file.Paths;import java.util.List;import org.apache.solr.cloud.SolrCloudTestCase;-import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;+import org.apache.solr.core.ConfigSetService;import org.apache.solr.util.ExternalPaths;import org.junit.After;import org.junit.Before;@@ -37,14 +36,16 @@ import org.junit.Test;*/public class ZkConfigFilesTest extends SolrCloudTestCase {- private static final int ZK_TIMEOUT_MILLIS = 10000;-@BeforeClasspublic static void setUpCluster() throws Exception {configureCluster(1).configure();}+ private static ConfigSetService getConfigSetService() {+ return cluster.getOpenOverseer().getCoreContainer().getConfigSetService();+ }+@Beforepublic void clearConfigsBefore() throws Exception {clearConfigs();@@ -56,25 +57,20 @@ public class ZkConfigFilesTest extends SolrCloudTestCase {}private void clearConfigs() throws Exception {- ZkConfigManager manager = new ZkConfigManager(cluster.getZkClient());- List<String> configs = manager.listConfigs();+ List<String> configs = getConfigSetService().listConfigs();for (String config : configs) {- manager.deleteConfigDir(config);+ getConfigSetService().deleteConfig(config);}}@Testpublic void testCanUploadConfigToZk() throws Exception {- final String zkConnectionString = cluster.getZkClient().getZkServerAddress();final String localConfigSetDirectory = new File(ExternalPaths.TECHPRODUCTS_CONFIGSET).getAbsolutePath();assertConfigsContainOnly();// tag::zk-configset-upload[]- try (SolrZkClient zkClient = new SolrZkClient(zkConnectionString, ZK_TIMEOUT_MILLIS)) {- ZkConfigManager manager = new ZkConfigManager(zkClient);- manager.uploadConfigDir(Paths.get(localConfigSetDirectory), "nameForConfigset");- }+ getConfigSetService().uploadConfig("nameForConfigset", Paths.get(localConfigSetDirectory));// end::zk-configset-upload[]assertConfigsContainOnly("nameForConfigset");@@ -83,8 +79,7 @@ public class ZkConfigFilesTest extends SolrCloudTestCase {private void assertConfigsContainOnly(String... expectedConfigs) throws Exception {final int expectedSize = expectedConfigs.length;- ZkConfigManager manager = new ZkConfigManager(cluster.getZkClient());- List<String> actualConfigs = manager.listConfigs();+ List<String> actualConfigs = getConfigSetService().listConfigs();assertEquals(expectedSize, actualConfigs.size());for (String expectedConfig : expectedConfigs) {diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/TestCloudSolrClientConnections.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/TestCloudSolrClientConnections.javaindex 4289ebffcaa..006575c6e15 100644--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/TestCloudSolrClientConnections.java+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/TestCloudSolrClientConnections.java@@ -24,7 +24,6 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.cloud.MiniSolrCloudCluster;import org.apache.solr.common.AlreadyClosedException;import org.apache.solr.common.SolrException;-import org.apache.solr.common.cloud.ZkConfigManager;import org.apache.solr.common.cloud.ZkStateReader;import org.junit.Test;@@ -66,7 +65,7 @@ public class TestCloudSolrClientConnections extends SolrTestCaseJ4 {try {CloudSolrClient client = cluster.getSolrClient();SolrException e = expectThrows(SolrException.class, () -> {- ((ZkClientClusterStateProvider)client.getClusterStateProvider()).uploadConfig(configPath, "testconfig");+ cluster.getZkClient().upConfig(configPath, "testconfig");});assertTrue("Unexpected message: " + e.getMessage(), e.getMessage().contains("cluster not found/not ready"));@@ -74,10 +73,9 @@ public class TestCloudSolrClientConnections extends SolrTestCaseJ4 {cluster.waitForAllNodes(30);client.connect(20, TimeUnit.SECONDS);- ((ZkClientClusterStateProvider)client.getClusterStateProvider()).uploadConfig(configPath, "testconfig");+ cluster.getZkClient().upConfig(configPath, "testconfig");- ZkConfigManager configManager = new ZkConfigManager(client.getZkStateReader().getZkClient());- assertTrue("List of uploaded configs does not contain 'testconfig'", configManager.listConfigs().contains("testconfig"));+ assertTrue("List of uploaded configs does not contain 'testconfig'", cluster.getZkClient().exists(ZkStateReader.CONFIGS_ZKNODE + "/" + "testconfig", true));} finally {cluster.shutdown();diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java b/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.javasimilarity index 82%rename from solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.javarename to solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.javaindex 8c75b5b9531..58facb53d3b 100644--- a/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java+++ b/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java@@ -18,7 +18,9 @@ package org.apache.solr.common.cloud;import com.google.common.base.Throwables;import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.cloud.ZkConfigSetService;import org.apache.solr.cloud.ZkTestServer;+import org.apache.solr.core.ConfigSetService;import org.apache.zookeeper.KeeperException;import org.apache.zookeeper.ZooDefs;import org.apache.zookeeper.data.ACL;@@ -37,7 +39,7 @@ import java.util.ArrayList;import java.util.Collection;import java.util.List;-public class TestZkConfigManager extends SolrTestCaseJ4 {+public class TestZkConfigSetService extends SolrTestCaseJ4 {private static ZkTestServer zkServer;@@ -57,8 +59,8 @@ public class TestZkConfigManager extends SolrTestCaseJ4 {@Testpublic void testConstants() throws Exception {- assertEquals("/configs", ZkConfigManager.CONFIGS_ZKNODE);- assertEquals("^\\..*$", ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX);+ assertEquals("/configs", ZkConfigSetService.CONFIGS_ZKNODE);+ assertEquals("^\\..*$", ConfigSetService.UPLOAD_FILENAME_EXCLUDE_REGEX);}@Test@@ -67,9 +69,9 @@ public class TestZkConfigManager extends SolrTestCaseJ4 {zkServer.ensurePathExists("/solr");try (SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress("/solr"), 10000)) {+ ConfigSetService configSetService = new ZkConfigSetService(zkClient);- ZkConfigManager configManager = new ZkConfigManager(zkClient);- assertEquals(0, configManager.listConfigs().size());+ assertEquals(0, configSetService.listConfigs().size());byte[] testdata = "test data".getBytes(StandardCharsets.UTF_8);@@ -83,16 +85,16 @@ public class TestZkConfigManager extends SolrTestCaseJ4 {Files.createDirectory(tempConfig.resolve(".ignoreddir"));Files.createFile(tempConfig.resolve(".ignoreddir").resolve("ignored"));- configManager.uploadConfigDir(tempConfig, "testconfig");+ configSetService.uploadConfig("testconfig", tempConfig);// uploading a directory creates a new config- List<String> configs = configManager.listConfigs();+ List<String> configs = configSetService.listConfigs();assertEquals(1, configs.size());assertEquals("testconfig", configs.get(0));// check downloadingPath downloadPath = createTempDir("download");- configManager.downloadConfigDir("testconfig", downloadPath);+ configSetService.downloadConfig( "testconfig", downloadPath);assertTrue(Files.exists(downloadPath.resolve("file1")));assertTrue(Files.exists(downloadPath.resolve("file2")));assertTrue(Files.isDirectory(downloadPath.resolve("subdir")));@@ -106,22 +108,22 @@ public class TestZkConfigManager extends SolrTestCaseJ4 {// uploading to the same config overwritesbyte[] overwritten = "new test data".getBytes(StandardCharsets.UTF_8);Files.write(tempConfig.resolve("file1"), overwritten);- configManager.uploadConfigDir(tempConfig, "testconfig");+ configSetService.uploadConfig("testconfig", tempConfig);- assertEquals(1, configManager.listConfigs().size());+ assertEquals(1, configSetService.listConfigs().size());Path download2 = createTempDir("download2");- configManager.downloadConfigDir("testconfig", download2);+ configSetService.downloadConfig( "testconfig", download2);byte[] checkdata2 = Files.readAllBytes(download2.resolve("file1"));assertArrayEquals(overwritten, checkdata2);// uploading same files to a new name creates a new config- configManager.uploadConfigDir(tempConfig, "config2");- assertEquals(2, configManager.listConfigs().size());+ configSetService.uploadConfig("config2", tempConfig);+ assertEquals(2, configSetService.listConfigs().size());// Test copying a config works in both flavors- configManager.copyConfigDir("config2", "config2copy");- configManager.copyConfigDir("config2", "config2copy2", null);- configs = configManager.listConfigs();+ configSetService.copyConfig("config2", "config2copy");+ configSetService.copyConfig( "config2", "config2copy2");+ configs = configSetService.listConfigs();assertTrue("config2copy should exist", configs.contains("config2copy"));assertTrue("config2copy2 should exist", configs.contains("config2copy2"));}@@ -175,16 +177,16 @@ public class TestZkConfigManager extends SolrTestCaseJ4 {// Start with all-access clienttry (SolrZkClient client = buildZkClient(zkServer.getZkAddress("/acl"), aclProvider, writeable)) {- ZkConfigManager configManager = new ZkConfigManager(client);- configManager.uploadConfigDir(configPath, "acltest");- assertEquals(1, configManager.listConfigs().size());+ ConfigSetService configSetService = new ZkConfigSetService(client);+ configSetService.uploadConfig("acltest", configPath);+ assertEquals(1, configSetService.listConfigs().size());}// Readonly access client can get the list of configs, but can't uploadtry (SolrZkClient client = buildZkClient(zkServer.getZkAddress("/acl"), aclProvider, readonly)) {- ZkConfigManager configManager = new ZkConfigManager(client);- assertEquals(1, configManager.listConfigs().size());- configManager.uploadConfigDir(configPath, "acltest2");+ ConfigSetService configSetService = new ZkConfigSetService(client);+ assertEquals(1, configSetService.listConfigs().size());+ configSetService.uploadConfig("acltest2", configPath);fail ("Should have thrown an ACL exception");}catch (IOException e) {@@ -193,8 +195,8 @@ public class TestZkConfigManager extends SolrTestCaseJ4 {// Client with no auth whatsoever can't even get the list of configstry (SolrZkClient client = new SolrZkClient(zkServer.getZkAddress("/acl"), 10000)) {- ZkConfigManager configManager = new ZkConfigManager(client);- configManager.listConfigs();+ ConfigSetService configSetService = new ZkConfigSetService(client);+ configSetService.listConfigs();fail("Should have thrown an ACL exception");}catch (IOException e) {diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.javaindex c248db82d8f..f14b6f2b1da 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java@@ -63,7 +63,7 @@ import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkConfigManager;+import org.apache.solr.common.cloud.ZkMaintenanceUtils;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.common.util.IOUtils;@@ -543,8 +543,7 @@ public class MiniSolrCloudCluster {public void uploadConfigSet(Path configDir, String configName) throws IOException, KeeperException, InterruptedException {try(SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(),AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT, null)) {- ZkConfigManager manager = new ZkConfigManager(zkClient);- manager.uploadConfigDir(configDir, configName);+ ZkMaintenanceUtils.uploadToZK(zkClient, configDir, ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + configName, ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN);}}@@ -611,7 +610,7 @@ public class MiniSolrCloudCluster {}try {// cleanup any property before removing the configset- getZkClient().delete(ZkConfigManager.CONFIGS_ZKNODE + "/" + configSet + "/" + DEFAULT_FILENAME, -1, true);+ getZkClient().delete(ZkConfigSetService.CONFIGS_ZKNODE + "/" + configSet + "/" + DEFAULT_FILENAME, -1, true);} catch (KeeperException.NoNodeException nne) { }new ConfigSetAdminRequest.Delete().setConfigSetName(configSet)@@ -752,7 +751,7 @@ public class MiniSolrCloudCluster {return getOpenOverseer(overseers);}-+public static Overseer getOpenOverseer(List<Overseer> overseers) {ArrayList<Overseer> shuffledOverseers = new ArrayList<Overseer>(overseers);Collections.shuffle(shuffledOverseers, LuceneTestCase.random());diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.javaindex 35c87e72d05..afb6d4dc339 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java@@ -43,7 +43,6 @@ import org.apache.solr.client.solrj.embedded.JettyConfig;import org.apache.solr.client.solrj.embedded.JettySolrRunner;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.impl.HttpSolrClient;-import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;import org.apache.solr.client.solrj.request.CoreAdminRequest;import org.apache.solr.client.solrj.request.CoreStatus;import org.apache.solr.common.cloud.ClusterProperties;@@ -276,7 +275,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {null, securityJson, trackJettyMetrics);CloudSolrClient client = cluster.getSolrClient();for (Config config : configs) {- ((ZkClientClusterStateProvider)client.getClusterStateProvider()).uploadConfig(config.path, config.name);+ cluster.uploadConfigSet(config.path, config.name);}if (clusterProperties.size() > 0) {