Apache Solr open-source search software
SOLR-17418: Streamline configSet code
Houston Putman8ec3876dc6d0
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 626a090abfc..ec1d7b2520a 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -215,6 +215,8 @@ Improvementssupport docValues will have them enabled by default. This behavior can be reverted by setting'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below. (Houston Putman)+* SOLR-17418: Streamline ConfigSet modification logic. (Houston Putman, Liu Huajin)+Optimizations---------------------* SOLR-17257: Both Minimize Cores and the Affinity replica placement strategies would over-gatherdiff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.javaindex e678e44d725..8bbdc8ac098 100644--- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java+++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java@@ -41,7 +41,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.CoreAdminRequest;import org.apache.solr.client.solrj.request.GenericSolrRequest;import org.apache.solr.client.solrj.response.CoreAdminResponse;-import org.apache.solr.common.cloud.ZkMaintenanceUtils;+import org.apache.solr.cloud.ZkConfigSetService;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.params.CommonParams;@@ -311,7 +311,10 @@ public class CreateTool extends ToolBase {confName = collectionName;}+ // TODO: This should be done using the configSet APIfinal Path configsetsDirPath = SolrCLI.getConfigSetsDir(solrInstallDirPath);+ ConfigSetService configSetService =+ new ZkConfigSetService(ZkStateReader.from(cloudSolrClient).getZkClient());Path confPath = ConfigSetService.getConfigsetPath(confDir, configsetsDirPath.toString());echoIfVerbose(@@ -322,11 +325,8 @@ public class CreateTool extends ToolBase {+ " to ZooKeeper at "+ cloudSolrClient.getClusterStateProvider().getQuorumHosts(),cli);- ZkMaintenanceUtils.uploadToZK(- ZkStateReader.from(cloudSolrClient).getZkClient(),- confPath,- ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName,- ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN);+ // We will trust the config since we have the Zookeeper Address+ configSetService.uploadConfig(confName, confPath, true);}// since creating a collection is a heavy-weight operation, check for existence firstdiff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.javaindex 5057e62613f..a0b53ea88e8 100644--- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java+++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java@@ -167,7 +167,7 @@ public class ZkConfigSetService extends ConfigSetService {}@Override- public void uploadConfig(String configName, Path dir) throws IOException {+ protected void uploadConfig(String configName, Path dir) throws IOException {zkClient.uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, ConfigSetService.UPLOAD_FILENAME_EXCLUDE_PATTERN);}@@ -230,6 +230,8 @@ public class ZkConfigSetService extends ConfigSetService {(Map<String, Object>)Utils.fromJSON(zkClient.getData(CONFIGS_ZKNODE + "/" + configName, null, null, true));return data;+ } catch (KeeperException.NoNodeException e) {+ return Collections.emptyMap();} catch (KeeperException | InterruptedException e) {throw new IOException("Error getting config metadata", SolrZkClient.checkInterrupted(e));}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.javaindex 55d1725e6ec..9a4b99514d0 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java@@ -150,6 +150,7 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {final URI location;final URI backupPath;final List<String> nodeList;+ final boolean requestIsTrusted;final CoreContainer container;final BackupRepository repository;@@ -165,6 +166,7 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {this.asyncId = message.getStr(ASYNC);this.repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);this.backupId = message.getInt(CoreAdminParams.BACKUP_ID, -1);+ this.requestIsTrusted = message.getBool(CoreAdminParams.TRUSTED, false);this.container = ccc.getCoreContainer();this.repository = this.container.newBackupRepository(repo);@@ -228,7 +230,8 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {rc.backupProperties.getConfigName(),rc.restoreConfigName,rc.backupManager,- rc.container.getConfigSetService());+ rc.container.getConfigSetService(),+ rc.requestIsTrusted);log.info("Starting restore into collection={} with backup_name={} at location={}",@@ -293,7 +296,8 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {String configName,String restoreConfigName,BackupManager backupMgr,- ConfigSetService configSetService)+ ConfigSetService configSetService,+ boolean requestIsTrusted)throws IOException {if (configSetService.checkConfigExists(restoreConfigName)) {log.info(@@ -305,7 +309,8 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {"Config with name {} does not already exist in ZooKeeper. Will restore from Backup.",restoreConfigName);- backupMgr.uploadConfigDir(configName, restoreConfigName, configSetService);+ backupMgr.uploadConfigDir(+ configName, restoreConfigName, configSetService, requestIsTrusted);}}diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.javaindex 32432d9f18b..53160cd75ee 100644--- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java+++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java@@ -23,6 +23,7 @@ import java.lang.invoke.MethodHandles;import java.lang.reflect.Constructor;import java.nio.file.Files;import java.nio.file.Path;+import java.util.HashMap;import java.util.List;import java.util.Locale;import java.util.Map;@@ -116,7 +117,7 @@ public abstract class ConfigSetService {"intended to be the default. Current 'solr.default.confdir' value:",System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE));} else {- this.uploadConfig(ConfigSetsHandler.DEFAULT_CONFIGSET_NAME, configDirPath);+ this.uploadConfig(ConfigSetsHandler.DEFAULT_CONFIGSET_NAME, configDirPath, true);}}}@@ -131,7 +132,7 @@ public abstract class ConfigSetService {String confName =System.getProperty(ZkController.COLLECTION_PARAM_PREFIX + ZkController.CONFIGNAME_PROP, "configuration1");- this.uploadConfig(confName, configPath);+ this.uploadConfig(confName, configPath, true);}/**@@ -215,7 +216,7 @@ public abstract class ConfigSetService {if (StrUtils.isNullOrEmpty(confName)) confName = coreName;Path udir = cd.getInstanceDir().resolve("conf");log.info("Uploading directory {} with name {} for solrCore {}", udir, confName, coreName);- cc.getConfigSetService().uploadConfig(confName, udir);+ cc.getConfigSetService().uploadConfig(confName, udir, true);}}@@ -244,6 +245,18 @@ public abstract class ConfigSetService {return (flags == null || flags.get("trusted") == null || flags.getBooleanArg("trusted"));}+ /**+ * Change the trust of the given configSet.+ *+ * @param name name of the configSet+ * @param isTrusted whether the given configSet should be trusted or not+ */+ public void setConfigSetTrust(String name, boolean isTrusted) throws IOException {+ Map<String, Object> contentMap = new HashMap<>(getConfigMetadata(name));+ contentMap.put("trusted", isTrusted);+ setConfigMetadata(name, contentMap);+ }+/*** Load the ConfigSet for a core*@@ -406,7 +419,20 @@ public abstract class ConfigSetService {* @param dir {@link Path} to the files* @throws IOException if an I/O error occurs or the path does not exist*/- public abstract void uploadConfig(String configName, Path dir) throws IOException;+ protected abstract void uploadConfig(String configName, Path dir) throws IOException;++ /**+ * Upload files from a given path to config, which will explicitly be trusted or not.+ *+ * @param configName the config name+ * @param dir {@link Path} to the files+ * @param isTrusted whether the config being uploaded is trusted+ * @throws IOException if an I/O error occurs or the path does not exist+ */+ public void uploadConfig(String configName, Path dir, boolean isTrusted) throws IOException {+ setConfigSetTrust(configName, isTrusted);+ uploadConfig(configName, dir);+ }/*** Upload a file to config If file does not exist, it will be uploaded If overwriteOnExists is set@@ -479,9 +505,25 @@ public abstract class ConfigSetService {* @param configName the config name* @param data the metadata to be set on config*/- public abstract void setConfigMetadata(String configName, Map<String, Object> data)+ protected abstract void setConfigMetadata(String configName, Map<String, Object> data)throws IOException;+ /**+ * Set the config metadata If config does not exist, it will be created and set metadata on it+ * Else metadata will be replaced with the provided metadata. This will preserve the trust that+ * the configSet already has if trust is not included in the metadata.+ *+ * @param configName the config name+ * @param data the metadata to be set on config+ */+ public void setConfigMetadataWithTrust(String configName, Map<String, Object> data)+ throws IOException {+ if (!data.containsKey("trusted")) {+ data.put("trusted", isConfigSetTrusted(configName));+ }+ setConfigMetadata(configName, data);+ }+/*** Get the config metadata (mutable, non-null)*diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.javaindex c4686195d64..da01cc57620 100644--- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java+++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java@@ -139,7 +139,7 @@ public class FileSystemConfigSetService extends ConfigSetService {}@Override- public void uploadConfig(String configName, Path source) throws IOException {+ protected void uploadConfig(String configName, Path source) throws IOException {Path dest = getConfigDir(configName);copyRecursively(source, dest);}@@ -169,7 +169,11 @@ public class FileSystemConfigSetService extends ConfigSetService {@Overridepublic void setConfigMetadata(String configName, Map<String, Object> data) throws IOException {// store metadata in .metadata.json file- Path metadataPath = getConfigDir(configName).resolve(METADATA_FILE);+ Path configDir = getConfigDir(configName);+ if (!Files.exists(configDir)) {+ Files.createDirectory(configDir);+ }+ Path metadataPath = configDir.resolve(METADATA_FILE);Files.write(metadataPath, Utils.toJSON(data));}diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.javaindex e846da51659..df3353a7b31 100644--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java@@ -255,12 +255,16 @@ public class BackupManager {* @throws IOException in case of I/O errors.*/public void uploadConfigDir(- String sourceConfigName, String targetConfigName, ConfigSetService configSetService)+ String sourceConfigName,+ String targetConfigName,+ ConfigSetService configSetService,+ boolean requestIsTrusted)throws IOException {URI source = repository.resolveDirectory(getZkStateDir(), CONFIG_STATE_DIR, sourceConfigName);if (!repository.exists(source)) {throw new IllegalArgumentException("Configset expected at " + source + " does not exist");}+ configSetService.setConfigSetTrust(targetConfigName, requestIsTrusted);uploadConfigToSolrCloud(configSetService, source, targetConfigName, "");}diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.javaindex e1e071e6418..ff8d9ac9690 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java@@ -31,6 +31,7 @@ import static org.apache.solr.common.params.CommonParams.NAME;import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID;import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION;import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY;+import static org.apache.solr.common.params.CoreAdminParams.TRUSTED;import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT;import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM;@@ -57,6 +58,7 @@ import org.apache.solr.common.params.CollectionParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.core.CoreContainer;import org.apache.solr.handler.admin.CollectionsHandler;+import org.apache.solr.handler.configsets.ConfigSetAPIBase;import org.apache.solr.jersey.JacksonReflectMapWriter;import org.apache.solr.jersey.PermissionName;import org.apache.solr.request.SolrQueryRequest;@@ -155,7 +157,7 @@ public class RestoreCollectionAPI extends BackupAPIBase {return response;}- public static ZkNodeProps createRemoteMessage(+ public ZkNodeProps createRemoteMessage(String backupName, RestoreCollectionRequestBody requestBody) {final Map<String, Object> remoteMessage = requestBody.toMap(new HashMap<>());@@ -181,6 +183,10 @@ public class RestoreCollectionAPI extends BackupAPIBase {if (requestBody.backupId != null) remoteMessage.put(BACKUP_ID, requestBody.backupId);if (requestBody.repository != null)remoteMessage.put(BACKUP_REPOSITORY, requestBody.repository);+ remoteMessage.put(+ TRUSTED,+ ConfigSetAPIBase.isTrusted(+ solrQueryRequest.getUserPrincipal(), coreContainer.getAuthenticationPlugin()));return new ZkNodeProps(remoteMessage);}diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/ConfigSetAPIBase.java b/solr/core/src/java/org/apache/solr/handler/configsets/ConfigSetAPIBase.javaindex 87c60fda94a..3f401e31bd8 100644--- a/solr/core/src/java/org/apache/solr/handler/configsets/ConfigSetAPIBase.java+++ b/solr/core/src/java/org/apache/solr/handler/configsets/ConfigSetAPIBase.java@@ -24,7 +24,6 @@ import java.io.IOException;import java.io.InputStream;import java.lang.invoke.MethodHandles;import java.security.Principal;-import java.util.Collections;import java.util.Iterator;import java.util.Map;import java.util.Optional;@@ -124,8 +123,6 @@ public class ConfigSetAPIBase {boolean requestIsTrusted,String configName)throws IOException {- Map<String, Object> metadata = Collections.singletonMap("trusted", requestIsTrusted);-if (overwritesExisting) {if (!requestIsTrusted) {ensureOverwritingUntrustedConfigSet(configName);@@ -133,7 +130,7 @@ public class ConfigSetAPIBase {// If the request is trusted and cleanup=true, then the configSet will be set to trusted after// the overwriting has been done.} else {- configSetService.setConfigMetadata(configName, metadata);+ configSetService.setConfigSetTrust(configName, requestIsTrusted);}}diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetAPI.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetAPI.javaindex f9bde932cb0..79d1b34d5ca 100644--- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetAPI.java+++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetAPI.java@@ -25,7 +25,6 @@ import java.lang.invoke.MethodHandles;import java.nio.charset.StandardCharsets;import java.util.Collections;import java.util.List;-import java.util.Map;import java.util.zip.ZipEntry;import java.util.zip.ZipInputStream;import org.apache.solr.api.EndPoint;@@ -110,8 +109,7 @@ public class UploadConfigSetAPI extends ConfigSetAPIBase {&& requestIsTrusted&& overwritesExisting&& !configSetService.isConfigSetTrusted(configSetName)) {- Map<String, Object> metadata = Collections.singletonMap("trusted", true);- configSetService.setConfigMetadata(configSetName, metadata);+ configSetService.setConfigSetTrust(configSetName, true);}}diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.javaindex db35a805c51..0125e7988d8 100644--- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java+++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java@@ -1231,8 +1231,7 @@ class SchemaDesignerConfigSetHelper implements SchemaDesignerConstants {public void removeConfigSetTrust(String configSetName) {try {- Map<String, Object> metadata = Collections.singletonMap("trusted", false);- cc.getConfigSetService().setConfigMetadata(configSetName, metadata);+ cc.getConfigSetService().setConfigSetTrust(configSetName, false);} catch (IOException e) {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.javaindex a45232a3af9..68142392df1 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java@@ -147,7 +147,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {public void testCreateErrors() throws Exception {final String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString();try (final SolrClient solrClient = getHttpSolrClient(baseUrl)) {- getConfigSetService().uploadConfig("configSet", configset("configset-2"));+ getConfigSetService().uploadConfig("configSet", configset("configset-2"), false);// no actionCreateNoErrorChecking createNoAction = new CreateNoErrorChecking();@@ -298,7 +298,9 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {getConfigSetProps(oldProps),UTF_8);}- getConfigSetService().uploadConfig(baseConfigSetName, tmpConfigDir);+ getConfigSetService()+ .uploadConfig(+ baseConfigSetName, tmpConfigDir, isTrusted(zkClient(), baseConfigSetName, ""));}private void verifyCreate(@@ -1350,12 +1352,15 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {throws KeeperException, InterruptedException {String configSetZkPath =String.format(Locale.ROOT, "/configs/%s%s", configsetName, configsetSuffix);- byte[] configSetNodeContent = zkClient.getData(configSetZkPath, null, null, true);- ;+ try {+ byte[] configSetNodeContent = zkClient.getData(configSetZkPath, null, null, true);- @SuppressWarnings("unchecked")- Map<Object, Object> contentMap = (Map<Object, Object>) Utils.fromJSON(configSetNodeContent);- return (boolean) contentMap.getOrDefault("trusted", true);+ @SuppressWarnings("unchecked")+ Map<Object, Object> contentMap = (Map<Object, Object>) Utils.fromJSON(configSetNodeContent);+ return (boolean) contentMap.getOrDefault("trusted", true);+ } catch (KeeperException.NoNodeException e) {+ return true;+ }}private int getConfigZNodeVersion(@@ -1928,7 +1933,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {tmpConfigDir.resolve("configsetprops.json"),getConfigSetProps(Map.of("immutable", "true")),UTF_8);- getConfigSetService().uploadConfig("configSet", tmpConfigDir);+ getConfigSetService().uploadConfig("configSet", tmpConfigDir, false);// no ConfigSet nameDeleteNoErrorChecking delete = new DeleteNoErrorChecking();@@ -1956,7 +1961,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {final String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString();final SolrClient solrClient = getHttpSolrClient(baseUrl);final String configSet = "testDelete";- getConfigSetService().uploadConfig(configSet, configset("configset-2"));+ getConfigSetService().uploadConfig(configSet, configset("configset-2"), false);assertDelete(solrClient, configSet, true);assertDelete(solrClient, "configSetBogus", false);solrClient.close();@@ -2005,7 +2010,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {Set<String> configSets = new HashSet<String>();for (int i = 0; i < 5; ++i) {String configSet = "configSet" + i;- getConfigSetService().uploadConfig(configSet, configset("configset-2"));+ getConfigSetService().uploadConfig(configSet, configset("configset-2"), false);configSets.add(configSet);}response = list.process(solrClient);diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigSetService.java b/solr/core/src/test/org/apache/solr/core/TestConfigSetService.javaindex e1f3787fc88..726ad98e8db 100644--- a/solr/core/src/test/org/apache/solr/core/TestConfigSetService.java+++ b/solr/core/src/test/org/apache/solr/core/TestConfigSetService.java@@ -24,6 +24,7 @@ import java.nio.file.Files;import java.nio.file.Path;import java.util.Arrays;import java.util.Collections;+import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.function.Supplier;@@ -78,7 +79,7 @@ public class TestConfigSetService extends SolrTestCaseJ4 {Files.createDirectory(configDir.resolve("subdir"));Files.createFile(configDir.resolve("subdir").resolve("file3"));- configSetService.uploadConfig(configName, configDir);+ configSetService.uploadConfig(configName, configDir, false);assertTrue(configSetService.checkConfigExists(configName));assertFalse(configSetService.checkConfigExists("dummyConfig"));@@ -92,14 +93,15 @@ public class TestConfigSetService extends SolrTestCaseJ4 {assertArrayEquals(configSetService.downloadFileFromConfig(configName, "subdir/file4"), data);Map<String, Object> metadata = configSetService.getConfigMetadata(configName);- assertTrue(metadata.isEmpty());+ assertFalse(metadata.isEmpty());+ assertFalse(configSetService.isConfigSetTrusted(configName));- configSetService.setConfigMetadata(configName, Collections.singletonMap("trusted", true));- metadata = configSetService.getConfigMetadata(configName);- assertTrue(metadata.containsKey("trusted"));+ configSetService.setConfigSetTrust(configName, true);+ assertTrue(configSetService.isConfigSetTrusted(configName));- configSetService.setConfigMetadata(configName, Collections.singletonMap("foo", true));- assertFalse(configSetService.getConfigMetadata(configName).containsKey("trusted"));+ configSetService.setConfigMetadataWithTrust(+ configName, new HashMap<>(Collections.singletonMap("foo", true)));+ assertTrue(configSetService.isConfigSetTrusted(configName));assertTrue(configSetService.getConfigMetadata(configName).containsKey("foo"));List<String> configFiles = configSetService.getAllConfigFiles(configName);diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.javaindex b8405fba6b0..4aaea27c857 100644--- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java+++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java@@ -568,14 +568,14 @@ public class TestCoreContainer extends SolrTestCaseJ4 {public void copyConfig(String fromConfig, String toConfig) {}@Override- public void uploadConfig(String configName, Path dir) {}+ protected void uploadConfig(String configName, Path dir) {}@Overridepublic void uploadFileToConfig(String configName, String fileName, byte[] data, boolean overwriteOnExists) {}@Override- public void setConfigMetadata(String configName, Map<String, Object> data) {}+ protected void setConfigMetadata(String configName, Map<String, Object> data) {}@Overridepublic Map<String, Object> getConfigMetadata(String configName) {diff --git a/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java b/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.javaindex 7b0e26d47cc..f9695a990e5 100644--- a/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java+++ b/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java@@ -22,6 +22,7 @@ import java.io.IOException;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;+import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.stream.Collectors;@@ -61,12 +62,13 @@ public class TestFileSystemConfigSetService extends SolrTestCaseJ4 {fileSystemConfigSetService.uploadFileToConfig(configName, "testfile", testdata, true);// metadata is stored in .metadata.json- fileSystemConfigSetService.setConfigMetadata(configName, Map.of("key1", "val1"));+ fileSystemConfigSetService.setConfigMetadataWithTrust(+ configName, new HashMap<>(Map.of("key1", "val1")));Map<String, Object> metadata = fileSystemConfigSetService.getConfigMetadata(configName);- assertEquals(metadata.toString(), "{key1=val1}");+ assertEquals("{key1=val1, trusted=true}", metadata.toString());List<String> allConfigFiles = fileSystemConfigSetService.getAllConfigFiles(configName);- assertEquals(allConfigFiles.toString(), "[schema.xml, solrconfig.xml, testfile]");+ assertEquals("[schema.xml, solrconfig.xml, testfile]", allConfigFiles.toString());fileSystemConfigSetService.deleteFilesFromConfig(configName, List.of(METADATA_FILE, "testfile"));@@ -74,19 +76,19 @@ public class TestFileSystemConfigSetService extends SolrTestCaseJ4 {assertTrue(metadata.isEmpty());allConfigFiles = fileSystemConfigSetService.getAllConfigFiles(configName);- assertEquals(allConfigFiles.toString(), "[schema.xml, solrconfig.xml]");+ assertEquals("[schema.xml, solrconfig.xml]", allConfigFiles.toString());fileSystemConfigSetService.copyConfig(configName, "copytestconfig");assertEquals(fileSystemConfigSetService.listConfigs().size(), 2);allConfigFiles = fileSystemConfigSetService.getAllConfigFiles("copytestconfig");- assertEquals(allConfigFiles.toString(), "[schema.xml, solrconfig.xml]");+ assertEquals("[schema.xml, solrconfig.xml]", allConfigFiles.toString());Path downloadConfig = createTempDir("downloadConfig");fileSystemConfigSetService.downloadConfig(configName, downloadConfig);List<String> configs = getFileList(downloadConfig);- assertEquals(configs.toString(), "[schema.xml, solrconfig.xml]");+ assertEquals("[schema.xml, solrconfig.xml]", configs.toString());Exception ex =assertThrows(diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperReadAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperReadAPITest.javaindex 39d494e177c..1a580953c5e 100644--- a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperReadAPITest.java+++ b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperReadAPITest.java@@ -72,10 +72,10 @@ public class ZookeeperReadAPITest extends SolrCloudTestCase {assertNotNull(o);o = Utils.executeGET(client.getHttpClient(), basezkls + "/configs", Utils.JSONCONSUMER);assertEquals(- "0",+ "16",String.valueOf(getObjectByPath(o, true, split(":/configs:_default:dataLength", ':'))));assertEquals(- "0", String.valueOf(getObjectByPath(o, true, split(":/configs:conf:dataLength", ':'))));+ "16", String.valueOf(getObjectByPath(o, true, split(":/configs:conf:dataLength", ':'))));assertEquals("0", String.valueOf(getObjectByPath(o, true, split("/stat/version", '/'))));o = Utils.executeGET(client.getHttpClient(), basezk + "/configs", Utils.JSONCONSUMER);diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.javaindex 17416f95786..922cc989726 100644--- a/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java+++ b/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java@@ -30,6 +30,7 @@ import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID;import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION;import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY;import static org.apache.solr.common.params.CoreAdminParams.NAME;+import static org.apache.solr.common.params.CoreAdminParams.TRUSTED;import static org.hamcrest.Matchers.containsString;import java.util.List;@@ -38,11 +39,28 @@ import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.client.api.model.CreateCollectionRequestBody;import org.apache.solr.common.SolrException;import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.NodeConfig;+import org.apache.solr.request.LocalSolrQueryRequest;+import org.junit.BeforeClass;import org.junit.Test;/** Unit tests for {@link RestoreCollectionAPI} */public class RestoreCollectionAPITest extends SolrTestCaseJ4 {+ private static RestoreCollectionAPI restoreCollectionAPI;++ @BeforeClass+ public static void setUpApi() {+ restoreCollectionAPI =+ new RestoreCollectionAPI(+ new CoreContainer(+ new NodeConfig.NodeConfigBuilder("testnode", createTempDir()).build()),+ new LocalSolrQueryRequest(null, new NamedList<>()),+ null);+ }+@Testpublic void testReportsErrorIfBackupNameMissing() {final var requestBody = new RestoreCollectionAPI.RestoreCollectionRequestBody();@@ -51,8 +69,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 {expectThrows(SolrException.class,() -> {- final var api = new RestoreCollectionAPI(null, null, null);- api.restoreCollection(null, requestBody);+ restoreCollectionAPI.restoreCollection(null, requestBody);});assertEquals(400, thrown.code());@@ -65,8 +82,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 {expectThrows(SolrException.class,() -> {- final var api = new RestoreCollectionAPI(null, null, null);- api.restoreCollection("someBackupName", null);+ restoreCollectionAPI.restoreCollection("someBackupName", null);});assertEquals(400, thrown.code());@@ -81,8 +97,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 {expectThrows(SolrException.class,() -> {- final var api = new RestoreCollectionAPI(null, null, null);- api.restoreCollection("someBackupName", requestBody);+ restoreCollectionAPI.restoreCollection("someBackupName", requestBody);});assertEquals(400, thrown.code());@@ -97,8 +112,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 {expectThrows(SolrException.class,() -> {- final var api = new RestoreCollectionAPI(null, null, null);- api.restoreCollection("someBackupName", requestBody);+ restoreCollectionAPI.restoreCollection("someBackupName", requestBody);});assertEquals(400, thrown.code());@@ -116,9 +130,9 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 {requestBody.async = "someAsyncId";final var remoteMessage =- RestoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties();+ restoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties();- assertEquals(7, remoteMessage.size());+ assertEquals(8, remoteMessage.size());assertEquals("restore", remoteMessage.get(QUEUE_OPERATION));assertEquals("someCollectionName", remoteMessage.get(COLLECTION));assertEquals("/some/location/path", remoteMessage.get(BACKUP_LOCATION));@@ -126,6 +140,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 {assertEquals("someRepositoryName", remoteMessage.get(BACKUP_REPOSITORY));assertEquals("someAsyncId", remoteMessage.get(ASYNC));assertEquals("someBackupName", remoteMessage.get(NAME));+ assertFalse((Boolean) remoteMessage.get(TRUSTED));}@Test@@ -146,9 +161,9 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 {createParams.properties = Map.of("foo", "bar");final var remoteMessage =- RestoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties();+ restoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties();- assertEquals(14, remoteMessage.size());+ assertEquals(15, remoteMessage.size());assertEquals("restore", remoteMessage.get(QUEUE_OPERATION));assertEquals("someCollectionName", remoteMessage.get(COLLECTION));assertEquals("/some/location/path", remoteMessage.get(BACKUP_LOCATION));@@ -162,6 +177,7 @@ public class RestoreCollectionAPITest extends SolrTestCaseJ4 {assertEquals(Integer.valueOf(456), remoteMessage.get(TLOG_REPLICAS));assertEquals(Integer.valueOf(789), remoteMessage.get(PULL_REPLICAS));assertEquals("node1,node2", remoteMessage.get(CREATE_NODE_SET_PARAM));+ assertFalse((Boolean) remoteMessage.get(TRUSTED));assertEquals("bar", remoteMessage.get("property.foo"));}diff --git a/solr/solr-ref-guide/modules/deployment-guide/examples/ZkConfigFilesTest.java b/solr/solr-ref-guide/modules/deployment-guide/examples/ZkConfigFilesTest.javaindex f3d9c41fb1e..f2fb303a7dd 100644--- a/solr/solr-ref-guide/modules/deployment-guide/examples/ZkConfigFilesTest.java+++ b/solr/solr-ref-guide/modules/deployment-guide/examples/ZkConfigFilesTest.java@@ -70,7 +70,8 @@ public class ZkConfigFilesTest extends SolrCloudTestCase {assertConfigsContainOnly();// tag::zk-configset-upload[]- getConfigSetService().uploadConfig("nameForConfigset", Paths.get(localConfigSetDirectory));+ getConfigSetService()+ .uploadConfig("nameForConfigset", Paths.get(localConfigSetDirectory), false);// end::zk-configset-upload[]assertConfigsContainOnly("nameForConfigset");diff --git a/solr/solrj-zookeeper/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java b/solr/solrj-zookeeper/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.javaindex 5375f62cdeb..8a2ff9f84cf 100644--- a/solr/solrj-zookeeper/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java+++ b/solr/solrj-zookeeper/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java@@ -94,13 +94,14 @@ public class TestZkConfigSetService extends SolrTestCaseJ4 {Files.createDirectory(tempConfig.resolve(".ignoreddir"));Files.createFile(tempConfig.resolve(".ignoreddir").resolve("ignored"));- configSetService.uploadConfig("testconfig", tempConfig);+ configSetService.uploadConfig("testconfig", tempConfig, true);// uploading a directory creates a new configList<String> configs = configSetService.listConfigs();assertEquals(1, configs.size());assertEquals("testconfig", configs.get(0));assertTrue(configSetService.checkConfigExists("testconfig"));+ assertTrue(configSetService.isConfigSetTrusted("testconfig"));// check downloadingPath downloadPath = createTempDir("download");@@ -118,16 +119,17 @@ public class TestZkConfigSetService extends SolrTestCaseJ4 {// uploading to the same config overwritesbyte[] overwritten = "new test data".getBytes(StandardCharsets.UTF_8);Files.write(tempConfig.resolve("file1"), overwritten);- configSetService.uploadConfig("testconfig", tempConfig);+ configSetService.uploadConfig("testconfig", tempConfig, false);assertEquals(1, configSetService.listConfigs().size());Path download2 = createTempDir("download2");configSetService.downloadConfig("testconfig", download2);byte[] checkdata2 = Files.readAllBytes(download2.resolve("file1"));assertArrayEquals(overwritten, checkdata2);+ assertFalse(configSetService.isConfigSetTrusted("testconfig"));// uploading same files to a new name creates a new config- configSetService.uploadConfig("config2", tempConfig);+ configSetService.uploadConfig("config2", tempConfig, true);assertEquals(2, configSetService.listConfigs().size());// Test copying a config works in both flavors@@ -136,6 +138,7 @@ public class TestZkConfigSetService extends SolrTestCaseJ4 {configs = configSetService.listConfigs();assertTrue("config2copy should exist", configs.contains("config2copy"));assertTrue("config2copy2 should exist", configs.contains("config2copy2"));+ assertTrue(configSetService.isConfigSetTrusted("config2"));}}@@ -210,7 +213,7 @@ public class TestZkConfigSetService extends SolrTestCaseJ4 {try (SolrZkClient client =buildZkClient(zkServer.getZkAddress("/acl"), aclProvider, writeable)) {ConfigSetService configSetService = new ZkConfigSetService(client);- configSetService.uploadConfig("acltest", configPath);+ configSetService.uploadConfig("acltest", configPath, false);assertEquals(1, configSetService.listConfigs().size());}@@ -221,7 +224,8 @@ public class TestZkConfigSetService extends SolrTestCaseJ4 {assertEquals(1, configSetService.listConfigs().size());IOException ioException =assertThrows(- IOException.class, () -> configSetService.uploadConfig("acltest2", configPath));+ IOException.class,+ () -> configSetService.uploadConfig("acltest2", configPath, false));assertEquals(KeeperException.NoAuthException.class, ioException.getCause().getClass());}diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.javaindex a85b3fe9ead..15d4397bd39 100644--- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java+++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java@@ -159,6 +159,9 @@ public abstract class CoreAdminParams {*/public static final String REPLICA_TYPE = "replicaType";+ /** Whether the request that generated the admin command is trusted */+ public static final String TRUSTED = "trusted";+public enum CoreAdminAction {STATUS(true),UNLOAD,diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.javaindex 5e0461f5240..26f33c7c60d 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java@@ -70,7 +70,6 @@ import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.cloud.ZkMaintenanceUtils;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.util.ExecutorUtil;@@ -568,11 +567,7 @@ public class MiniSolrCloudCluster {.withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS).withConnTimeOut(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS).build()) {- ZkMaintenanceUtils.uploadToZK(- zkClient,- configDir,- ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + configName,- ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN);+ new ZkConfigSetService(zkClient).uploadConfig(configName, configDir, true);}}